From 8020171dbdfdde0b42fd4eeb9f38f26c2d38a8df Mon Sep 17 00:00:00 2001 From: Alex Bocharov Date: Tue, 9 Jul 2024 11:38:01 -0500 Subject: [PATCH 1/5] Added hasher template parameter `H` for `Synchronizer` with `WyHash` as default. This will speed up checksum calculation during writes. --- Cargo.lock | 11 ++++++++++- Cargo.toml | 2 +- src/synchronizer.rs | 25 ++++++++++++++----------- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 573259e..fddef37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,8 +222,8 @@ dependencies = [ "proptest", "rand", "rkyv", - "seahash", "thiserror", + "wyhash", ] [[package]] @@ -629,6 +629,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "wyhash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295" +dependencies = [ + "rand_core", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index c8fccf5..7a23f4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ categories = ["os", "filesystem", "concurrency", "data-structures", "memory-mana bytecheck = { version = "~0.6.8", default-features = false } memmap2 = "0.7.0" rkyv = { version = "0.7.40", features = ["validation", "strict"] } -seahash = "4.1.0" thiserror = "1.0.30" +wyhash = "0.5.0" [dev-dependencies] proptest = "1.0.0" diff --git a/src/synchronizer.rs b/src/synchronizer.rs index ebba229..1540d47 100644 --- a/src/synchronizer.rs +++ b/src/synchronizer.rs @@ -3,17 +3,17 @@ //! The `Synchronizer` offers a simple interface for reading and writing data from/to shared memory. It uses memory-mapped files and wait-free synchronization to provide high concurrency wait-free reads over a single writer instance. This design is inspired by the [Left-Right concurrency control technique](https://github.com/pramalhe/ConcurrencyFreaks/blob/master/papers/left-right-2014.pdf), allowing for efficient and flexible inter-process communication. //! //! Furthermore, with the aid of the [rkyv](https://rkyv.org/) library, `Synchronizer` can perform zero-copy deserialization, reducing time and memory usage when accessing data. +use std::ffi::OsStr; +use std::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use std::time::Duration; use bytecheck::CheckBytes; use rkyv::ser::serializers::AllocSerializer; use rkyv::ser::Serializer; use rkyv::validation::validators::DefaultValidator; use rkyv::{archived_root, check_archived_root, Archive, Serialize}; -use seahash::SeaHasher; -use std::ffi::OsStr; -use std::hash::Hasher; -use std::time::Duration; use thiserror::Error; +use wyhash::WyHash; use crate::data::DataContainer; use crate::guard::{ReadGuard, ReadResult}; @@ -24,11 +24,13 @@ use crate::synchronizer::SynchronizerError::*; /// `Synchronizer` is a concurrency primitive that manages data access between a single writer process and multiple reader processes. /// /// It coordinates the access to two data files that store the shared data. A state file, also memory-mapped, stores the index of the current data file and the number of active readers for each index, updated via atomic instructions. -pub struct Synchronizer { +pub struct Synchronizer { /// Container storing state mmap state_container: StateContainer, /// Container storing data mmap data_container: DataContainer, + /// Hasher used for checksum calculation + build_hasher: BuildHasherDefault, } /// `SynchronizerError` enumerates all possible errors returned by this library. @@ -68,6 +70,7 @@ impl Synchronizer { Synchronizer { state_container: StateContainer::new(path_prefix), data_container: DataContainer::new(path_prefix), + build_hasher: BuildHasherDefault::default(), } } @@ -111,7 +114,7 @@ impl Synchronizer { let state = self.state_container.state(true)?; // calculate data checksum - let mut hasher = SeaHasher::new(); + let mut hasher = self.build_hasher.build_hasher(); hasher.write(&data); let checksum = hasher.finish(); @@ -142,7 +145,7 @@ impl Synchronizer { let state = self.state_container.state(true)?; // calculate data checksum - let mut hasher = SeaHasher::new(); + let mut hasher = self.build_hasher.build_hasher(); hasher.write(data); let checksum = hasher.finish(); @@ -296,7 +299,7 @@ mod tests { assert!(!Path::new(&data_path_1).exists()); assert_eq!( reader.version().unwrap(), - InstanceVersion(15768700985330904896) + InstanceVersion(8817430144856633152) ); // check that first time scoped `read` works correctly and switches the data @@ -315,7 +318,7 @@ mod tests { assert!(Path::new(&data_path_1).exists()); assert_eq!( reader.version().unwrap(), - InstanceVersion(7331894278219651425) + InstanceVersion(1441050725688826209) ); // check that another scoped `read` works correctly and switches the data @@ -328,7 +331,7 @@ mod tests { assert_eq!(reset, false); assert_eq!( reader.version().unwrap(), - InstanceVersion(9949249822303202528) + InstanceVersion(14058099486534675680) ); let entity = entity_generator.gen(200); @@ -337,7 +340,7 @@ mod tests { assert_eq!(reset, false); assert_eq!( reader.version().unwrap(), - InstanceVersion(16072265150643592177) + InstanceVersion(18228729609619266545) ); fetch_and_assert_entity(&mut reader, &entity, true); From 6b24b1e4ddb3631c8486380788d05e7d79bf0c9f Mon Sep 17 00:00:00 2001 From: Alex Bocharov Date: Tue, 9 Jul 2024 12:39:25 -0500 Subject: [PATCH 2/5] Added read/write benchmarks from `Synchronizer`. --- Cargo.lock | 1088 +++++++++++++++++++++++++++++++++++++-- Cargo.toml | 6 + benches/synchronizer.rs | 64 +++ 3 files changed, 1116 insertions(+), 42 deletions(-) create mode 100644 benches/synchronizer.rs diff --git a/Cargo.lock b/Cargo.lock index fddef37..7cbf323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -13,12 +28,59 @@ dependencies = [ "version_check", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -40,6 +102,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + [[package]] name = "bitvec" version = "1.0.1" @@ -52,6 +120,12 @@ dependencies = [ "wyz", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "bytecheck" version = "0.6.11" @@ -74,12 +148,30 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.79" @@ -92,6 +184,129 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags 1.3.2", + "textwrap", + "unicode-width", +] + +[[package]] +name = "cpp_demangle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "debugid" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" +dependencies = [ + "uuid 0.8.2", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -100,7 +315,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -122,6 +337,24 @@ dependencies = [ "instant", ] +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fnv" version = "1.0.7" @@ -145,6 +378,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "half" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" + [[package]] name = "hashbrown" version = "0.12.3" @@ -154,12 +399,83 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "inferno" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3886428c6400486522cf44b8626e7b94ad794c14390290f2a274dcf728a58f" +dependencies = [ + "ahash", + "atty", + "indexmap 1.9.3", + "itoa", + "lazy_static", + "log", + "num-format", + "quick-xml", + "rgb", + "str_stack", +] + [[package]] name = "instant" version = "0.1.12" @@ -175,9 +491,33 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", ] [[package]] @@ -188,9 +528,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" @@ -204,6 +544,43 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memmap2" version = "0.7.0" @@ -213,12 +590,32 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + [[package]] name = "mmap-sync" version = "1.0.2" dependencies = [ "bytecheck", - "memmap2", + "criterion", + "memmap2 0.7.0", + "pprof", "proptest", "rand", "rkyv", @@ -226,6 +623,35 @@ dependencies = [ "wyhash", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "nix" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", + "memoffset", +] + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -236,12 +662,115 @@ dependencies = [ "libm", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.2.6", +] + +[[package]] +name = "plotters" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" + +[[package]] +name = "plotters-svg" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "pprof" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55f35f865aa964be21fcde114cbd1cfbd9bf8a471460ed965b0f84f96c711401" +dependencies = [ + "backtrace", + "cfg-if", + "criterion", + "findshlibs", + "inferno", + "lazy_static", + "libc", + "log", + "nix", + "parking_lot", + "prost", + "prost-build", + "prost-derive", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -249,32 +778,85 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] -name = "proc-macro2" -version = "1.0.60" +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" +dependencies = [ + "bit-set", + "bitflags 1.3.2", + "byteorder", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.6.29", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" dependencies = [ - "unicode-ident", + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] -name = "proptest" -version = "1.2.0" +name = "prost-types" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" dependencies = [ - "bit-set", - "bitflags", - "byteorder", - "lazy_static", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "rusty-fork", - "tempfile", - "unarray", + "bytes", + "prost", ] [[package]] @@ -303,6 +885,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.28" @@ -357,13 +948,65 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.4", ] [[package]] @@ -372,6 +1015,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + [[package]] name = "rend" version = "0.4.0" @@ -381,6 +1030,15 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "rgb" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aee83dc281d5a3200d37b299acd13b81066ea126a7f16f0eae70fc9aed241d9" +dependencies = [ + "bytemuck", +] + [[package]] name = "rkyv" version = "0.7.42" @@ -389,13 +1047,13 @@ checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" dependencies = [ "bitvec", "bytecheck", - "hashbrown", + "hashbrown 0.12.3", "ptr_meta", "rend", "rkyv_derive", "seahash", "tinyvec", - "uuid", + "uuid 1.3.3", ] [[package]] @@ -409,18 +1067,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustix" version = "0.37.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys 0.4.14", + "windows-sys 0.48.0", ] [[package]] @@ -435,18 +1112,118 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "serde" +version = "1.0.185" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" + +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "serde_json" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "simdutf8" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + +[[package]] +name = "symbolic-common" +version = "8.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f551f902d5642e58039aee6a9021a61037926af96e071816361644983966f540" +dependencies = [ + "debugid", + "memmap2 0.5.10", + "stable_deref_trait", + "uuid 0.8.2", +] + +[[package]] +name = "symbolic-demangle" +version = "8.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4564ca7b4e6eb14105aa8bbbce26e080f6b5d9c4373e67167ab31f7b86443750" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -484,9 +1261,18 @@ dependencies = [ "autocfg", "cfg-if", "fastrand", - "redox_syscall", - "rustix", - "windows-sys", + "redox_syscall 0.3.5", + "rustix 0.37.20", + "windows-sys 0.48.0", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", ] [[package]] @@ -509,6 +1295,16 @@ dependencies = [ "syn 2.0.18", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -536,6 +1332,24 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + [[package]] name = "uuid" version = "1.3.3" @@ -557,19 +1371,145 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.18", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.25", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -578,13 +1518,29 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -593,42 +1549,90 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wyhash" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 7a23f4c..36f534e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,11 @@ thiserror = "1.0.30" wyhash = "0.5.0" [dev-dependencies] +criterion = { version = "0.3.5", features = ["html_reports"] } +pprof = { version = "0.6.2", features = ["flamegraph", "protobuf", "criterion"] } proptest = "1.0.0" rand = "0.8.5" + +[[bench]] +name = "synchronizer" +harness = false diff --git a/benches/synchronizer.rs b/benches/synchronizer.rs new file mode 100644 index 0000000..5c040a8 --- /dev/null +++ b/benches/synchronizer.rs @@ -0,0 +1,64 @@ +use std::time::Duration; + +use bytecheck::CheckBytes; +use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; +use mmap_sync::synchronizer::Synchronizer; +use pprof::criterion::PProfProfiler; +use rkyv::{Archive, Deserialize, Serialize}; + +/// Example data-structure shared between writer and reader(s) +#[derive(Archive, Deserialize, Serialize, Debug, PartialEq)] +#[archive_attr(derive(CheckBytes))] +pub struct HelloWorld { + pub version: u32, + pub messages: Vec, +} + +pub fn bench_synchronizer(c: &mut Criterion) { + let mut synchronizer = Synchronizer::new("/dev/shm/hello_world".as_ref()); + let data = HelloWorld { + version: 7, + messages: vec!["Hello".to_string(), "World".to_string(), "!".to_string()], + }; + let bytes = rkyv::to_bytes::(&data).unwrap(); + + let mut group = c.benchmark_group("synchronizer"); + group.throughput(Throughput::Elements(1)); + + group.bench_function("write", |b| { + b.iter(|| { + synchronizer + .write(black_box(&data), Duration::from_nanos(10)) + .expect("failed to write data"); + }) + }); + + group.bench_function("write_raw", |b| { + b.iter(|| { + synchronizer + .write_raw::(black_box(&bytes), Duration::from_nanos(10)) + .expect("failed to write data"); + }) + }); + + group.bench_function("read/check_bytes_true", |b| { + b.iter(|| { + let archived = unsafe { synchronizer.read::(true).unwrap() }; + assert_eq!(archived.version, data.version); + }) + }); + + group.bench_function("read/check_bytes_false", |b| { + b.iter(|| { + let archived = unsafe { synchronizer.read::(false).unwrap() }; + assert_eq!(archived.version, data.version); + }) + }); +} + +criterion_group! { + name = benches; + config = Criterion::default().with_profiler(PProfProfiler::new(100, pprof::criterion::Output::Protobuf)); + targets = bench_synchronizer +} +criterion_main!(benches); From 4a3fdba32c2ef367129dbb69928f753e924bc530 Mon Sep 17 00:00:00 2001 From: Alex Bocharov Date: Tue, 9 Jul 2024 12:50:36 -0500 Subject: [PATCH 3/5] Optimize `Synchronizer::write` and `write_raw` methods to re-use opened maps. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmarks: ``` synchronizer/write time: [17.183 µs 17.387 µs 17.647 µs] thrpt: [56.666 Kelem/s 57.515 Kelem/s 58.195 Kelem/s] change: time: [-47.134% -45.506% -43.898%] (p = 0.00 < 0.05) thrpt: [+78.247% +83.508% +89.156%] Performance has improved. synchronizer/write_raw time: [147.12 ns 147.76 ns 148.57 ns] thrpt: [6.7310 Melem/s 6.7676 Melem/s 6.7970 Melem/s] change: time: [-98.463% -98.442% -98.422%] (p = 0.00 < 0.05) thrpt: [+6236.1% +6320.0% +6405.5%] Performance has improved. synchronizer/read/check_bytes_true time: [41.194 ns 41.271 ns 41.369 ns] thrpt: [24.172 Melem/s 24.230 Melem/s 24.275 Melem/s] change: time: [-0.3344% +0.8124% +1.6734%] (p = 0.12 > 0.05) thrpt: [-1.6458% -0.8058% +0.3355%] No change in performance detected. synchronizer/read/check_bytes_false time: [26.658 ns 26.668 ns 26.685 ns] thrpt: [37.475 Melem/s 37.498 Melem/s 37.513 Melem/s] change: time: [-4.4520% -3.5968% -2.8378%] (p = 0.00 < 0.05) thrpt: [+2.9206% +3.7309% +4.6595%] Performance has improved. ``` --- src/data.rs | 52 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/data.rs b/src/data.rs index 4d8d3b3..1d38ef1 100644 --- a/src/data.rs +++ b/src/data.rs @@ -17,7 +17,9 @@ pub(crate) struct DataContainer { /// Reader's current local instance version version: Option, /// Read-only memory mapped files storing data - idx_mmaps: [Option; 2], + read_mmaps: [Option; 2], + /// Write-only memory mapped files storing data + write_mmaps: [Option; 2], } impl DataContainer { @@ -26,7 +28,8 @@ impl DataContainer { DataContainer { path_prefix: path_prefix.into(), version: None, - idx_mmaps: [None, None], + read_mmaps: [None, None], + write_mmaps: [None, None], } } @@ -36,27 +39,38 @@ impl DataContainer { data: &[u8], version: InstanceVersion, ) -> Result { - let mut opts = OpenOptions::new(); - opts.read(true).write(true).create(true); + let mmap = &mut self.write_mmaps[version.idx()]; + let data_size = version.size(); - // Only add mode on Unix-based systems - #[cfg(unix)] - opts.mode(0o640); + // only open and mmap data file in the following cases: + // * if it never was opened/mapped before + // * if current mmap size is smaller than requested data size + if mmap.is_none() || mmap.as_ref().unwrap().len() < data_size { + let mut opts = OpenOptions::new(); + opts.read(true).write(true).create(true); - let data_file = opts - .open(version.path(&self.path_prefix)) - .map_err(FailedDataWrite)?; + // Only add mode on Unix-based systems + #[cfg(unix)] + opts.mode(0o640); - // grow data file when its current length exceeded - let data_len = data.len() as u64; - if data_len > data_file.metadata().map_err(FailedDataWrite)?.len() { - data_file.set_len(data_len).map_err(FailedDataWrite)?; + let data_file = opts + .open(version.path(&self.path_prefix)) + .map_err(FailedDataWrite)?; + + // grow data file when its current length exceeded + let data_len = data.len() as u64; + if data_len > data_file.metadata().map_err(FailedDataWrite)?.len() { + data_file.set_len(data_len).map_err(FailedDataWrite)?; + } + + *mmap = Some(unsafe { MmapMut::map_mut(&data_file).map_err(FailedDataWrite)? }); } - // copy data to mapped file and ensure it's been flushed - let mut mmap = unsafe { MmapMut::map_mut(&data_file).map_err(FailedDataWrite)? }; - mmap[..data.len()].copy_from_slice(data); - mmap.flush().map_err(FailedDataWrite)?; + if let Some(mmap) = mmap { + // copy data to mapped file and ensure it's been flushed + mmap[..data.len()].copy_from_slice(data); + mmap.flush().map_err(FailedDataWrite)?; + } Ok(data.len()) } @@ -67,7 +81,7 @@ impl DataContainer { &mut self, version: InstanceVersion, ) -> Result<(&[u8], bool), SynchronizerError> { - let mmap = &mut self.idx_mmaps[version.idx()]; + let mmap = &mut self.read_mmaps[version.idx()]; let data_size = version.size(); // only open and mmap data file in the following cases: From 86e132d1173ba5723aaa7bf6f8b9d681de2f2479 Mon Sep 17 00:00:00 2001 From: Alex Bocharov Date: Tue, 9 Jul 2024 16:23:51 -0500 Subject: [PATCH 4/5] Speed up writes further with re-usable buffer backed by `AlignedVec`. Also added template parameter `N` for scratch space size. Benchmarks: ``` synchronizer/write time: [250.71 ns 251.42 ns 252.41 ns] thrpt: [3.9619 Melem/s 3.9774 Melem/s 3.9887 Melem/s] change: time: [-99.152% -99.147% -99.142%] (p = 0.00 < 0.05) thrpt: [+11559% +11627% +11699%] Performance has improved. synchronizer/write_raw time: [145.25 ns 145.53 ns 145.92 ns] thrpt: [6.8531 Melem/s 6.8717 Melem/s 6.8849 Melem/s] change: time: [-98.508% -98.471% -98.443%] (p = 0.00 < 0.05) thrpt: [+6322.7% +6441.2% +6602.9%] Performance has improved. synchronizer/read/check_bytes_true time: [40.114 ns 40.139 ns 40.186 ns] thrpt: [24.884 Melem/s 24.914 Melem/s 24.929 Melem/s] change: time: [-0.1031% -0.0200% +0.0753%] (p = 0.69 > 0.05) thrpt: [-0.0752% +0.0200% +0.1032%] No change in performance detected. synchronizer/read/check_bytes_false time: [26.658 ns 26.673 ns 26.696 ns] thrpt: [37.458 Melem/s 37.491 Melem/s 37.512 Melem/s] change: time: [-0.9845% -0.9083% -0.7959%] (p = 0.00 < 0.05) thrpt: [+0.8023% +0.9167% +0.9943%] Change within noise threshold. ``` --- src/synchronizer.rs | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/synchronizer.rs b/src/synchronizer.rs index 1540d47..f6d8890 100644 --- a/src/synchronizer.rs +++ b/src/synchronizer.rs @@ -8,10 +8,10 @@ use std::hash::{BuildHasher, BuildHasherDefault, Hasher}; use std::time::Duration; use bytecheck::CheckBytes; -use rkyv::ser::serializers::AllocSerializer; +use rkyv::ser::serializers::{AlignedSerializer, AllocSerializer}; use rkyv::ser::Serializer; use rkyv::validation::validators::DefaultValidator; -use rkyv::{archived_root, check_archived_root, Archive, Serialize}; +use rkyv::{archived_root, check_archived_root, AlignedVec, Archive, Serialize}; use thiserror::Error; use wyhash::WyHash; @@ -24,13 +24,15 @@ use crate::synchronizer::SynchronizerError::*; /// `Synchronizer` is a concurrency primitive that manages data access between a single writer process and multiple reader processes. /// /// It coordinates the access to two data files that store the shared data. A state file, also memory-mapped, stores the index of the current data file and the number of active readers for each index, updated via atomic instructions. -pub struct Synchronizer { +pub struct Synchronizer { /// Container storing state mmap state_container: StateContainer, /// Container storing data mmap data_container: DataContainer, /// Hasher used for checksum calculation build_hasher: BuildHasherDefault, + /// Re-usable buffer for serialization + serialize_buffer: Option, } /// `SynchronizerError` enumerates all possible errors returned by this library. @@ -61,16 +63,21 @@ pub enum SynchronizerError { InvalidInstanceVersionParams, } -/// Default serializer with 1 MB scratch space allocated on the heap. -type DefaultSerializer = AllocSerializer<1_000_000>; - impl Synchronizer { - /// Create new instance of `Synchronizer` using given `path_prefix` - pub fn new(path_prefix: &OsStr) -> Synchronizer { + /// Create new instance of `Synchronizer` using given `path_prefix` and default template parameters + pub fn new(path_prefix: &OsStr) -> Self { + Self::with_params(path_prefix) + } +} + +impl Synchronizer { + /// Create new instance of `Synchronizer` using given `path_prefix` and template parameters + pub fn with_params(path_prefix: &OsStr) -> Self { Synchronizer { state_container: StateContainer::new(path_prefix), data_container: DataContainer::new(path_prefix), build_hasher: BuildHasherDefault::default(), + serialize_buffer: Some(AlignedVec::new()), } } @@ -97,11 +104,18 @@ impl Synchronizer { grace_duration: Duration, ) -> Result<(usize, bool), SynchronizerError> where - T: Serialize, + T: Serialize>, T::Archived: for<'b> CheckBytes>, { + let mut buf = self.serialize_buffer.take().ok_or(FailedEntityWrite)?; + buf.clear(); + // serialize given entity into bytes - let mut serializer = DefaultSerializer::default(); + let mut serializer = AllocSerializer::new( + AlignedSerializer::new(buf), + Default::default(), + Default::default(), + ); let _ = serializer .serialize_value(entity) .map_err(|_| FailedEntityWrite)?; @@ -126,6 +140,9 @@ impl Synchronizer { // switch readers to new version state.switch_version(new_version); + // Restore buffer for potential reuse + self.serialize_buffer.replace(data); + Ok((size, reset)) } @@ -138,7 +155,7 @@ impl Synchronizer { grace_duration: Duration, ) -> Result<(usize, bool), SynchronizerError> where - T: Serialize, + T: Serialize>, T::Archived: for<'b> CheckBytes>, { // fetch current state from mapped memory From c170c270ae3ecce21078598a6873945a098720d9 Mon Sep 17 00:00:00 2001 From: Alex Bocharov Date: Wed, 10 Jul 2024 11:57:21 -0500 Subject: [PATCH 5/5] Update README, Cargo dependencies and release new `mmap-sync` version `1.0.3`. --- Cargo.lock | 486 +++++++++++++++++++++-------------------------------- Cargo.toml | 8 +- README.md | 43 ++++- 3 files changed, 233 insertions(+), 304 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cbf323..fc53d2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,19 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -38,27 +51,22 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.86" +name = "anes" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] -name = "arrayvec" -version = "0.7.4" +name = "anstyle" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] -name = "atty" -version = "0.2.14" +name = "arrayvec" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "autocfg" @@ -160,12 +168,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" - [[package]] name = "cast" version = "0.3.0" @@ -184,46 +186,87 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" -version = "2.34.0" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ - "bitflags 1.3.2", - "textwrap", - "unicode-width", + "clap_builder", ] +[[package]] +name = "clap_builder" +version = "4.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + [[package]] name = "cpp_demangle" -version = "0.3.5" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" dependencies = [ "cfg-if", ] [[package]] name = "criterion" -version = "0.3.6" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ - "atty", + "anes", "cast", + "ciborium", "clap", "criterion-plot", - "csv", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", - "serde_cbor", "serde_derive", "serde_json", "tinytemplate", @@ -232,9 +275,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.4.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools", @@ -266,33 +309,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] -name = "csv" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.11" +name = "crunchy" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" -dependencies = [ - "memchr", -] +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "debugid" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid 0.8.2", + "uuid", ] [[package]] @@ -349,12 +377,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "fnv" version = "1.0.7" @@ -386,9 +408,13 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "half" -version = "1.8.3" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] name = "hashbrown" @@ -396,7 +422,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.6", ] [[package]] @@ -405,49 +431,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.2.6" @@ -460,17 +449,17 @@ dependencies = [ [[package]] name = "inferno" -version = "0.10.12" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3886428c6400486522cf44b8626e7b94ad794c14390290f2a274dcf728a58f" +checksum = "321f0f839cd44a4686e9504b0a62b4d69a50b62072144c71c68f5873c167b8d9" dependencies = [ - "ahash", - "atty", - "indexmap 1.9.3", + "ahash 0.8.11", + "indexmap", + "is-terminal", "itoa", - "lazy_static", "log", "num-format", + "once_cell", "quick-xml", "rgb", "str_stack", @@ -491,11 +480,22 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "libc", "windows-sys 0.48.0", ] +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -544,12 +544,6 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" -[[package]] -name = "linux-raw-sys" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" - [[package]] name = "lock_api" version = "0.4.12" @@ -574,31 +568,13 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - -[[package]] -name = "memmap2" -version = "0.7.0" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180d4b35be83d33392d1d1bfbd2ae1eca7ff5de1a94d3fc87faaa99a069e7cbd" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - [[package]] name = "miniz_oxide" version = "0.7.4" @@ -610,11 +586,11 @@ dependencies = [ [[package]] name = "mmap-sync" -version = "1.0.2" +version = "1.0.3" dependencies = [ "bytecheck", "criterion", - "memmap2 0.7.0", + "memmap2", "pprof", "proptest", "rand", @@ -623,23 +599,15 @@ dependencies = [ "wyhash", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "nix" -version = "0.23.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", - "cc", "cfg-if", "libc", - "memoffset", ] [[package]] @@ -685,37 +653,25 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ - "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.6" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", - "instant", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.5.2", "smallvec", - "winapi", -] - -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset", - "indexmap 2.2.6", + "windows-targets 0.52.6", ] [[package]] @@ -748,23 +704,22 @@ dependencies = [ [[package]] name = "pprof" -version = "0.6.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f35f865aa964be21fcde114cbd1cfbd9bf8a471460ed965b0f84f96c711401" +checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" dependencies = [ "backtrace", "cfg-if", "criterion", "findshlibs", "inferno", - "lazy_static", "libc", "log", "nix", + "once_cell", "parking_lot", - "prost", - "prost-build", - "prost-derive", + "protobuf", + "protobuf-codegen-pure", "smallvec", "symbolic-demangle", "tempfile", @@ -779,9 +734,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -807,56 +762,28 @@ dependencies = [ ] [[package]] -name = "prost" -version = "0.9.0" +name = "protobuf" +version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive", -] +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] -name = "prost-build" -version = "0.9.0" +name = "protobuf-codegen" +version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost", - "prost-types", - "regex", - "tempfile", - "which", + "protobuf", ] [[package]] -name = "prost-derive" -version = "0.9.0" +name = "protobuf-codegen-pure" +version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865" dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost", + "protobuf", + "protobuf-codegen", ] [[package]] @@ -887,9 +814,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.22.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" dependencies = [ "memchr", ] @@ -970,20 +897,20 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", ] [[package]] @@ -1053,7 +980,7 @@ dependencies = [ "rkyv_derive", "seahash", "tinyvec", - "uuid 1.3.3", + "uuid", ] [[package]] @@ -1083,20 +1010,7 @@ dependencies = [ "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "0.38.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" -dependencies = [ - "bitflags 2.6.0", - "errno", - "libc", - "linux-raw-sys 0.4.14", + "linux-raw-sys", "windows-sys 0.48.0", ] @@ -1141,18 +1055,11 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "serde" -version = "1.0.185" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" - -[[package]] -name = "serde_cbor" -version = "0.11.2" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ - "half", - "serde", + "serde_derive", ] [[package]] @@ -1163,14 +1070,14 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] name = "serde_json" -version = "1.0.109" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -1203,21 +1110,21 @@ checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" [[package]] name = "symbolic-common" -version = "8.8.0" +version = "12.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f551f902d5642e58039aee6a9021a61037926af96e071816361644983966f540" +checksum = "71297dc3e250f7dbdf8adb99e235da783d690f5819fdeb4cce39d9cfb0aca9f1" dependencies = [ "debugid", - "memmap2 0.5.10", + "memmap2", "stable_deref_trait", - "uuid 0.8.2", + "uuid", ] [[package]] name = "symbolic-demangle" -version = "8.8.0" +version = "12.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4564ca7b4e6eb14105aa8bbbce26e080f6b5d9c4373e67167ab31f7b86443750" +checksum = "424fa2c9bf2c862891b9cfd354a752751a6730fd838a4691e7f6c2c7957b9daf" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -1237,9 +1144,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", @@ -1262,19 +1169,10 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.20", + "rustix", "windows-sys 0.48.0", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "thiserror" version = "1.0.40" @@ -1292,7 +1190,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.43", ] [[package]] @@ -1332,24 +1230,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" -[[package]] -name = "unicode-segmentation" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" - -[[package]] -name = "unicode-width" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" - -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" - [[package]] name = "uuid" version = "1.3.3" @@ -1408,7 +1288,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.43", "wasm-bindgen-shared", ] @@ -1430,7 +1310,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.43", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1451,18 +1331,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.25", -] - [[package]] name = "winapi" version = "0.3.9" @@ -1650,3 +1518,23 @@ checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ "tap", ] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] diff --git a/Cargo.toml b/Cargo.toml index 36f534e..33dc2c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mmap-sync" -version = "1.0.2" +version = "1.0.3" edition = "2021" authors = ["Alex Bocharov "] description = "A Rust package allowing sharing of data between processes in a wait-free and zero-copy fashion from mapped memory." @@ -12,14 +12,14 @@ categories = ["os", "filesystem", "concurrency", "data-structures", "memory-mana [dependencies] bytecheck = { version = "~0.6.8", default-features = false } -memmap2 = "0.7.0" +memmap2 = "0.9.4" rkyv = { version = "0.7.40", features = ["validation", "strict"] } thiserror = "1.0.30" wyhash = "0.5.0" [dev-dependencies] -criterion = { version = "0.3.5", features = ["html_reports"] } -pprof = { version = "0.6.2", features = ["flamegraph", "protobuf", "criterion"] } +criterion = { version = "0.5.1", features = ["html_reports"] } +pprof = { version = "0.13.0", features = ["flamegraph", "protobuf-codec", "criterion"] } proptest = "1.0.0" rand = "0.8.5" diff --git a/README.md b/README.md index 601f709..3803de6 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The templated type `T` for `Synchronizer` can be any Rust struct implementing sp To use `mmap-sync`, add it to your `Cargo.toml` under `[dependencies]`: ```toml [dependencies] -mmap-sync = "1.0.0" +mmap-sync = "1" ``` Then, import `mmap-sync` in your Rust program: ```rust @@ -86,3 +86,44 @@ $ stat -c '%A %s %n' /tmp/hello_world_* ``` With these steps, you can start utilizing `mmap-sync` in your Rust applications for efficient concurrent data access across processes. + +## Tuning performance +Using `tmpfs` volume will reduce the disk I/O latency since it operates directly on RAM, offering faster read and write capabilities compared to conventional disk-based storage: +```rust +let mut synchronizer = Synchronizer::new("/dev/shm/hello_world".as_ref()); +``` +This change points the synchronizer to use a shared memory object located in a `tmpfs` filesystem, which is typically mounted at `/dev/shm` on most Linux systems. This should help alleviate some of the bottlenecks associated with disk I/O. +If `/dev/shm` does not provide enough space or if you want to create a dedicated `tmpfs` instance, you can set up your own with the desired size. For example, to create a 1GB `tmpfs` volume, you can use the following command: +```shell +sudo mount -t tmpfs -o size=1G tmpfs /mnt/mytmpfs +``` + +## Benchmarks +To run benchmarks you first need to install `cargo-criterion` binary: +```shell +cargo install cargo-criterion +``` + +Then you'll be able to run benchmarks with the following command: +```shell +cargo criterion --bench synchronizer +``` + +Benchmarks presented below are executed on Linux laptop with `13th Gen Intel(R) Core(TM) i7-13800H` processor and compiler flags set to `RUSTFLAGS=-C target-cpu=native`. +```shell +synchronizer/write + time: [250.71 ns 251.42 ns 252.41 ns] + thrpt: [3.9619 Melem/s 3.9774 Melem/s 3.9887 Melem/s] + +synchronizer/write_raw + time: [145.25 ns 145.53 ns 145.92 ns] + thrpt: [6.8531 Melem/s 6.8717 Melem/s 6.8849 Melem/s] + +synchronizer/read/check_bytes_true + time: [40.114 ns 40.139 ns 40.186 ns] + thrpt: [24.884 Melem/s 24.914 Melem/s 24.929 Melem/s] + +synchronizer/read/check_bytes_false + time: [26.658 ns 26.673 ns 26.696 ns] + thrpt: [37.458 Melem/s 37.491 Melem/s 37.512 Melem/s] +``` \ No newline at end of file