From d44385da2de58ffdb0e98cc3f326eed09afe1111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 17 Sep 2024 11:38:51 +0200 Subject: [PATCH 01/52] WIP: make len atomic, add and use methods using atomics This is a very early draft. method names and semantic may change. --- src/lib.rs | 74 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3aabad4..0918d7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,12 +10,13 @@ use core::{ ops::{Deref, DerefMut, Index, IndexMut}, ptr, slice::SliceIndex, + sync::atomic::{AtomicUsize, Ordering}, }; struct HeaderVecHeader { head: H, capacity: usize, - len: usize, + len: AtomicUsize, } /// A vector with a header of your choosing, behind a thin pointer @@ -78,19 +79,62 @@ impl HeaderVec { unsafe { core::ptr::write(&mut header.head, head) }; // These primitive types don't have drop implementations. header.capacity = capacity; - header.len = 0; + header.len = AtomicUsize::new(0); this } + /// Get the length of the vector from a mutable reference. + #[inline(always)] + pub fn len_from_mut(&mut self) -> usize { + *self.header_mut().len.get_mut() + } + + /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is + /// properly synchronized after it got atomically updated. + #[inline(always)] + pub fn len_atomic_acquire(&self) -> usize { + self.header().len.load(Ordering::Acquire) + } + + /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't + /// need exact synchronization semantic. + #[inline(always)] + pub fn len_atomic_relaxed(&self) -> usize { + self.header().len.load(Ordering::Relaxed) + } + + /// Alias for [`HeaderVec::len_atomic_relaxed`]. This gives always a valid view of the + /// length when used in isolation. #[inline(always)] pub fn len(&self) -> usize { - self.header().len + self.len_atomic_relaxed() + } + + /// Add `n` to the length of the vector atomically with `Ordering::Release`. + /// + /// # Safety + /// + /// Before incrementing the length of the vector, you must ensure that new elements are + /// properly initialized. + #[inline(always)] + pub unsafe fn len_atomic_add_release(&self, n: usize) -> usize { + self.header().len.fetch_add(n, Ordering::Release) + } + + #[inline(always)] + pub fn is_empty_from_mut(&mut self) -> bool { + self.len_from_mut() == 0 } #[inline(always)] pub fn is_empty(&self) -> bool { - self.len() == 0 + self.len_atomic_relaxed() == 0 + } + + #[inline(always)] + pub fn is_empty_atomic_acquire(&self) -> bool { + self.len_atomic_acquire() == 0 } #[inline(always)] @@ -100,12 +144,17 @@ impl HeaderVec { #[inline(always)] pub fn as_slice(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len()) } + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_relaxed()) } + } + + #[inline(always)] + pub fn as_slice_atomic_acquire(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } } #[inline(always)] pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len()) } + unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_from_mut()) } } /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`]. @@ -196,7 +245,7 @@ impl HeaderVec { /// Returns `true` if the memory was moved to a new location. /// In this case, you are responsible for updating the weak nodes. pub fn push(&mut self, item: T) -> Option<*const ()> { - let old_len = self.len(); + let old_len = self.len_from_mut(); let new_len = old_len + 1; let old_capacity = self.capacity(); // If it isn't big enough. @@ -208,7 +257,7 @@ impl HeaderVec { unsafe { core::ptr::write(self.start_ptr_mut().add(old_len), item); } - self.header_mut().len = new_len; + self.header_mut().len = new_len.into(); previous_pointer } @@ -221,7 +270,7 @@ impl HeaderVec { // This keeps track of the length (and next position) of the contiguous retained elements // at the beginning of the vector. let mut head = 0; - let original_len = self.len(); + let original_len = self.len_from_mut(); // Get the offset of the beginning of the slice. let start_ptr = self.start_ptr_mut(); // Go through each index. @@ -243,7 +292,7 @@ impl HeaderVec { } } // The head now represents the new length of the vector. - self.header_mut().len = head; + self.header_mut().len = head.into(); } /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. @@ -305,7 +354,7 @@ impl Drop for HeaderVec { fn drop(&mut self) { unsafe { ptr::drop_in_place(&mut self.header_mut().head); - for ix in 0..self.len() { + for ix in 0..self.len_from_mut() { ptr::drop_in_place(self.start_ptr_mut().add(ix)); } alloc::alloc::dealloc(self.ptr as *mut u8, Self::layout(self.capacity())); @@ -367,7 +416,8 @@ where T: Clone, { fn clone(&self) -> Self { - let mut new_vec = Self::with_capacity(self.len(), self.header().head.clone()); + let mut new_vec = + Self::with_capacity(self.len_atomic_acquire(), self.header().head.clone()); for e in self.as_slice() { new_vec.push(e.clone()); } From c9d53888b1b0923e03c621a3b2833f2330091b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 17 Sep 2024 16:39:41 +0200 Subject: [PATCH 02/52] rename methods in _exact/_strict, add 'atomic_append' feature, add 'spare_capacity()' method --- Cargo.toml | 4 ++ src/lib.rs | 143 ++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 107 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9d4a58c..4ba66aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,7 @@ keywords = ["header", "heap", "vec", "vector", "graph"] categories = ["no-std"] license = "MIT" readme = "README.md" + +[features] +default = ["atomic_append"] +atomic_append = [] diff --git a/src/lib.rs b/src/lib.rs index 79f5b05..0ca244c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,13 +10,18 @@ use core::{ ops::{Deref, DerefMut, Index, IndexMut}, ptr, slice::SliceIndex, - sync::atomic::{AtomicUsize, Ordering}, }; +#[cfg(feature = "atomic_append")] +use core::sync::atomic::{AtomicUsize, Ordering}; + struct HeaderVecHeader { head: H, capacity: usize, + #[cfg(feature = "atomic_append")] len: AtomicUsize, + #[cfg(not(feature = "atomic_append"))] + len: usize, } /// A vector with a header of your choosing behind a thin pointer @@ -75,62 +80,73 @@ impl HeaderVec { unsafe { core::ptr::write(&mut header.head, head) }; // These primitive types don't have drop implementations. header.capacity = capacity; - header.len = AtomicUsize::new(0); + header.len = 0usize.into(); this } - /// Get the length of the vector from a mutable reference. + /// Get the length of the vector from a mutable reference. When one has a `&mut + /// HeaderVec`, this is the method is always exact and can be slightly faster than the non + /// mutable `len()`. + #[cfg(feature = "atomic_append")] #[inline(always)] - pub fn len_from_mut(&mut self) -> usize { + pub fn len_exact(&mut self) -> usize { *self.header_mut().len.get_mut() } - - /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is - /// properly synchronized after it got atomically updated. + #[cfg(not(feature = "atomic_append"))] #[inline(always)] - pub fn len_atomic_acquire(&self) -> usize { - self.header().len.load(Ordering::Acquire) + pub fn len_exact(&mut self) -> usize { + self.header_mut().len } - /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't - /// need exact synchronization semantic. + /// This gives the length of the `HeaderVec`. This is the non synchronized variant may + /// produce racy results in case another thread atomically appended to + /// `&self`. Nevertheless it is always safe to use. + #[cfg(feature = "atomic_append")] #[inline(always)] - pub fn len_atomic_relaxed(&self) -> usize { - self.header().len.load(Ordering::Relaxed) + pub fn len(&self) -> usize { + self.len_atomic_relaxed() } - - /// Alias for [`HeaderVec::len_atomic_relaxed`]. This gives always a valid view of the - /// length when used in isolation. + #[cfg(not(feature = "atomic_append"))] #[inline(always)] pub fn len(&self) -> usize { - self.len_atomic_relaxed() + self.header().len } - /// Add `n` to the length of the vector atomically with `Ordering::Release`. - /// - /// # Safety - /// - /// Before incrementing the length of the vector, you must ensure that new elements are - /// properly initialized. + /// This gives the length of the `HeaderVec`. With `atomic_append` enabled this gives a + /// exact result *after* another thread atomically appended to this `HeaderVec`. It still + /// requires synchronization because the length may become invalidated when another thread + /// atomically appends data to this `HeaderVec` while we still work with the result of + /// this method. + #[cfg(not(feature = "atomic_append"))] #[inline(always)] - pub unsafe fn len_atomic_add_release(&self, n: usize) -> usize { - self.header().len.fetch_add(n, Ordering::Release) + pub fn len_strict(&self) -> usize { + self.header().len + } + #[cfg(feature = "atomic_append")] + #[inline(always)] + pub fn len_strict(&self) -> usize { + self.len_atomic_acquire() } + /// Check whenever a `HeaderVec` is empty. This uses a `&mut self` reference and is + /// always exact and may be slightly faster than the non mutable variant. #[inline(always)] - pub fn is_empty_from_mut(&mut self) -> bool { - self.len_from_mut() == 0 + pub fn is_empty_exact(&mut self) -> bool { + self.len_exact() == 0 } + /// Check whenever a `HeaderVec` is empty. This uses a `&self` reference and may be racy + /// when another thread atomically appended to this `HeaderVec`. #[inline(always)] pub fn is_empty(&self) -> bool { - self.len_atomic_relaxed() == 0 + self.len() == 0 } + /// Check whenever a `HeaderVec` is empty. see [`len_strict()`] about the exactness guarantees. #[inline(always)] - pub fn is_empty_atomic_acquire(&self) -> bool { - self.len_atomic_acquire() == 0 + pub fn is_empty_strict(&self) -> bool { + self.len_strict() == 0 } #[inline(always)] @@ -138,19 +154,20 @@ impl HeaderVec { self.header().capacity } + /// This is the amount of elements that can be added to the `HeaderVec` without reallocation. #[inline(always)] - pub fn as_slice(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_relaxed()) } + pub fn spare_capacity(&self) -> usize { + self.header().capacity - self.len_strict() } #[inline(always)] - pub fn as_slice_atomic_acquire(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } + pub fn as_slice(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_strict()) } } #[inline(always)] pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_from_mut()) } + unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_exact()) } } /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`]. @@ -241,7 +258,7 @@ impl HeaderVec { /// Returns `true` if the memory was moved to a new location. /// In this case, you are responsible for updating the weak nodes. pub fn push(&mut self, item: T) -> Option<*const ()> { - let old_len = self.len_from_mut(); + let old_len = self.len_exact(); let new_len = old_len + 1; let old_capacity = self.capacity(); // If it isn't big enough. @@ -266,7 +283,7 @@ impl HeaderVec { // This keeps track of the length (and next position) of the contiguous retained elements // at the beginning of the vector. let mut head = 0; - let original_len = self.len_from_mut(); + let original_len = self.len_exact(); // Get the offset of the beginning of the slice. let start_ptr = self.start_ptr_mut(); // Go through each index. @@ -346,11 +363,58 @@ impl HeaderVec { } } +#[cfg(feature = "atomic_append")] +/// The atomic append API is only enabled when the `atomic_append` feature flag is set (which +/// is the default). +impl HeaderVec { + /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is + /// properly synchronized after it got atomically updated. + #[inline(always)] + fn len_atomic_acquire(&self) -> usize { + self.header().len.load(Ordering::Acquire) + } + + /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't + /// need exact synchronization semantic. + #[inline(always)] + fn len_atomic_relaxed(&self) -> usize { + self.header().len.load(Ordering::Relaxed) + } + + /// Add `n` to the length of the vector atomically with `Ordering::Release`. + /// + /// # Safety + /// + /// Before incrementing the length of the vector, you must ensure that new elements are + /// properly initialized. + #[inline(always)] + unsafe fn len_atomic_add_release(&self, n: usize) -> usize { + self.header().len.fetch_add(n, Ordering::Release) + } + + #[inline(always)] + pub fn is_empty_atomic_acquire(&self) -> bool { + self.len_atomic_acquire() == 0 + } + + #[inline(always)] + pub fn as_slice_atomic_acquire(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } + } + + /// Gets the pointer to the end of the slice. This returns a mutable pointer to + /// uninitialized memory behind the last element. + #[inline(always)] + fn end_ptr_atomic_mut(&self) -> *mut T { + unsafe { self.ptr.add(Self::offset()).add(self.len_atomic_acquire()) } + } +} + impl Drop for HeaderVec { fn drop(&mut self) { unsafe { ptr::drop_in_place(&mut self.header_mut().head); - for ix in 0..self.len_from_mut() { + for ix in 0..self.len_exact() { ptr::drop_in_place(self.start_ptr_mut().add(ix)); } alloc::alloc::dealloc(self.ptr as *mut u8, Self::layout(self.capacity())); @@ -412,8 +476,7 @@ where T: Clone, { fn clone(&self) -> Self { - let mut new_vec = - Self::with_capacity(self.len_atomic_acquire(), self.header().head.clone()); + let mut new_vec = Self::with_capacity(self.len_strict(), self.header().head.clone()); for e in self.as_slice() { new_vec.push(e.clone()); } From ecc95544ecd398d54bdb83b0a37cf23f7893435d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 17 Sep 2024 17:20:04 +0200 Subject: [PATCH 03/52] add push_atomic() --- src/lib.rs | 26 ++++++++++++++++++++++++++ tests/atomic_append.rs | 16 ++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tests/atomic_append.rs diff --git a/src/lib.rs b/src/lib.rs index 0ca244c..bbb4fb2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -408,6 +408,32 @@ impl HeaderVec { fn end_ptr_atomic_mut(&self) -> *mut T { unsafe { self.ptr.add(Self::offset()).add(self.len_atomic_acquire()) } } + + /// Atomically adds an item to the end of the list without reallocation. + /// + /// # Errors + /// + /// If the vector is full, the item is returned. + /// + /// # Safety + /// + /// There must be only one thread calling this method at any time. Synchronization has to + /// be provided by the user. + pub unsafe fn push_atomic(&self, item: T) -> Result<(), T> { + // relaxed is good enough here because this should be the only thread calling this method. + let len = self.len_atomic_relaxed(); + if len < self.capacity() { + unsafe { + core::ptr::write(self.end_ptr_atomic_mut(), item); + }; + let len_again = self.len_atomic_add_release(1); + // in debug builds we check for races, the chance to catch these are still pretty minimal + debug_assert_eq!(len_again, len, "len was updated by another thread"); + Ok(()) + } else { + Err(item) + } + } } impl Drop for HeaderVec { diff --git a/tests/atomic_append.rs b/tests/atomic_append.rs new file mode 100644 index 0000000..a1112db --- /dev/null +++ b/tests/atomic_append.rs @@ -0,0 +1,16 @@ +#![cfg(feature = "atomic_append")] +extern crate std; + +use header_vec::*; + +#[test] +fn test_atomic_append() { + let mut hv = HeaderVec::with_capacity(10, ()); + + hv.push(1); + unsafe { hv.push_atomic(2).unwrap() }; + hv.push(3); + + assert_eq!(hv.len(), 3); + assert_eq!(hv.as_slice(), [1, 2, 3]); +} From 993cad03125f1d817c3db192b85065fd7f1760b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 03:50:22 +0200 Subject: [PATCH 04/52] WIP: reserve/shrink API's --- src/lib.rs | 91 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bbb4fb2..fd3e430 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -222,12 +222,83 @@ impl HeaderVec { self.ptr = weak.ptr; } + /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. + #[inline(always)] + pub fn reserve(&mut self, additional: usize) -> Option<*const ()> { + if self.spare_capacity() < additional { + let len = self.len_exact(); + unsafe { self.resize_cold(len + additional, false) } + } else { + None + } + } + + /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. + #[inline] + pub fn reserve_exact(&mut self, additional: usize) -> Option<*const ()> { + if self.spare_capacity() < additional { + let len = self.len_exact(); + unsafe { self.resize_cold(len + additional, true) } + } else { + None + } + } + + /// Shrinks the capacity of the `HeaderVec` to the `min_capacity` or `self.len()`, whichever is larger. + #[inline] + pub fn shrink_to(&mut self, min_capacity: usize) -> Option<*const ()> { + let requested_capacity = self.len_exact().max(min_capacity); + unsafe { self.resize_cold(requested_capacity, true) } + } + + /// Resizes the vector hold exactly `self.len()` elements. + #[inline(always)] + pub fn shrink_to_fit(&mut self) -> Option<*const ()> { + let len = self.len_exact(); + self.shrink_to(len) + } + + /// Resize the vector to have at least room for `additional` more elements. + /// does exact resizing if `exact` is true. + /// + /// Returns `Some(*const ())` if the memory was moved to a new location. + /// + /// # Safety + /// + /// `requested_capacity` must be greater or equal than `self.len()` #[cold] - fn resize_insert(&mut self) -> Option<*const ()> { + unsafe fn resize_cold(&mut self, requested_capacity: usize, exact: bool) -> Option<*const ()> { + // For efficiency we do only a debug_assert here + debug_assert!( + self.len_exact() <= requested_capacity, + "requested capacity is less than current length" + ); let old_capacity = self.capacity(); - let new_capacity = old_capacity * 2; - // Set the new capacity. - self.header_mut().capacity = new_capacity; + debug_assert_ne!(old_capacity, 0, "capacity of 0 not yet supported"); + debug_assert_ne!(requested_capacity, 0, "capacity of 0 not yet supported"); + + let new_capacity = if requested_capacity > old_capacity { + if exact { + // exact growing + requested_capacity + } else if requested_capacity <= old_capacity * 2 { + // doubling the capacity is sufficient + old_capacity * 2 + } else { + // requested more than twice as much space, reserve the next multiple of + // old_capacity that is greater than the requested capacity. This gives headroom + // for new inserts while not doubling the memory requirement with bulk requests + (requested_capacity / old_capacity + 1).saturating_mul(old_capacity) + } + } else if exact { + // exact shrinking + requested_capacity + } else { + unimplemented!() + // or: (has no public API yet) + // // shrink to the next power of two or self.capacity, whichever is smaller + // requested_capacity.next_power_of_two().min(self.capacity()) + }; // Reallocate the pointer. let ptr = unsafe { alloc::alloc::realloc( @@ -249,24 +320,20 @@ impl HeaderVec { }; // Assign the new pointer. self.ptr = ptr; + // And set the new capacity. + self.header_mut().capacity = new_capacity; previous_pointer } /// Adds an item to the end of the list. /// - /// Returns `true` if the memory was moved to a new location. + /// Returns `Some(*const ())` if the memory was moved to a new location. /// In this case, you are responsible for updating the weak nodes. pub fn push(&mut self, item: T) -> Option<*const ()> { let old_len = self.len_exact(); let new_len = old_len + 1; - let old_capacity = self.capacity(); - // If it isn't big enough. - let previous_pointer = if new_len > old_capacity { - self.resize_insert() - } else { - None - }; + let previous_pointer = self.reserve(1); unsafe { core::ptr::write(self.start_ptr_mut().add(old_len), item); } From 4bcab1f6ed2c60cd7afd938d32f7c657e8330eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 14:36:20 +0200 Subject: [PATCH 05/52] FIX: the reseve functions need saturating add as well --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fd3e430..0a25ad7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -227,7 +227,7 @@ impl HeaderVec { pub fn reserve(&mut self, additional: usize) -> Option<*const ()> { if self.spare_capacity() < additional { let len = self.len_exact(); - unsafe { self.resize_cold(len + additional, false) } + unsafe { self.resize_cold(len.saturating_add(additional), false) } } else { None } @@ -238,7 +238,7 @@ impl HeaderVec { pub fn reserve_exact(&mut self, additional: usize) -> Option<*const ()> { if self.spare_capacity() < additional { let len = self.len_exact(); - unsafe { self.resize_cold(len + additional, true) } + unsafe { self.resize_cold(len.saturating_add(additional), true) } } else { None } From decaeed7c7d8bd4660e66082460d500321bdc223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 15:51:14 +0200 Subject: [PATCH 06/52] FIX: Miri, unaligned access * introduces a helper union to fix simplify alignment calculations no need for cmp and phantom data anymore * add simple testcase that triggered the miri issue * change end_ptr_atomic_mut(), using the rewritten start_ptr() --- src/lib.rs | 39 ++++++++++++++++++--------------------- tests/simple.rs | 9 +++++++++ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0a25ad7..5bcfb68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,9 +3,7 @@ extern crate alloc; use core::{ - cmp, fmt::Debug, - marker::PhantomData, mem::{self, ManuallyDrop}, ops::{Deref, DerefMut, Index, IndexMut}, ptr, @@ -24,6 +22,12 @@ struct HeaderVecHeader { len: usize, } +// This union will be properly aligned and sized to store headers followed by T's. +union AlignedHeader { + _header: ManuallyDrop>, + _data: ManuallyDrop<[T; 0]>, +} + /// A vector with a header of your choosing behind a thin pointer /// /// # Example @@ -47,8 +51,7 @@ struct HeaderVecHeader { /// All of the data, like our header `OurHeaderType { a: 2 }`, the length of the vector: `2`, /// and the contents of the vector `['x', 'z']` resides on the other side of the pointer. pub struct HeaderVec { - ptr: *mut T, - _phantom: PhantomData, + ptr: *mut AlignedHeader, } impl HeaderVec { @@ -58,9 +61,9 @@ impl HeaderVec { pub fn with_capacity(capacity: usize, head: H) -> Self { assert!(capacity > 0, "HeaderVec capacity cannot be 0"); - // Allocate the initial memory, which is unititialized. + // Allocate the initial memory, which is uninitialized. let layout = Self::layout(capacity); - let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut T; + let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut AlignedHeader; // Handle out-of-memory. if ptr.is_null() { @@ -68,10 +71,7 @@ impl HeaderVec { } // Create self. - let mut this = Self { - ptr, - _phantom: PhantomData, - }; + let mut this = Self { ptr }; // Set the header. let header = this.header_mut(); @@ -204,10 +204,7 @@ impl HeaderVec { #[inline(always)] pub unsafe fn weak(&self) -> HeaderVecWeak { HeaderVecWeak { - header_vec: ManuallyDrop::new(Self { - ptr: self.ptr, - _phantom: PhantomData, - }), + header_vec: ManuallyDrop::new(Self { ptr: self.ptr }), } } @@ -305,7 +302,7 @@ impl HeaderVec { self.ptr as *mut u8, Self::layout(old_capacity), Self::elems_to_mem_bytes(new_capacity), - ) as *mut T + ) as *mut AlignedHeader }; // Handle out-of-memory. if ptr.is_null() { @@ -377,10 +374,10 @@ impl HeaderVec { /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. #[inline(always)] - fn offset() -> usize { + const fn offset() -> usize { // The first location, in units of size_of::(), that is after the header // It's the end of the header, rounded up to the nearest size_of::() - (mem::size_of::>() + mem::size_of::() - 1) / mem::size_of::() + mem::size_of::>() / mem::size_of::() } /// Compute the number of elements (in units of T) to allocate for a given capacity. @@ -400,7 +397,7 @@ impl HeaderVec { fn layout(capacity: usize) -> alloc::alloc::Layout { alloc::alloc::Layout::from_size_align( Self::elems_to_mem_bytes(capacity), - cmp::max(mem::align_of::(), mem::align_of::()), + mem::align_of::>() ) .expect("unable to produce memory layout with Hrc key type (is it a zero sized type? they are not permitted)") } @@ -408,13 +405,13 @@ impl HeaderVec { /// Gets the pointer to the start of the slice. #[inline(always)] fn start_ptr(&self) -> *const T { - unsafe { self.ptr.add(Self::offset()) } + unsafe { (self.ptr as *const T).add(Self::offset()) } } /// Gets the pointer to the start of the slice. #[inline(always)] fn start_ptr_mut(&mut self) -> *mut T { - unsafe { self.ptr.add(Self::offset()) } + unsafe { (self.ptr as *mut T).add(Self::offset()) } } #[inline(always)] @@ -473,7 +470,7 @@ impl HeaderVec { /// uninitialized memory behind the last element. #[inline(always)] fn end_ptr_atomic_mut(&self) -> *mut T { - unsafe { self.ptr.add(Self::offset()).add(self.len_atomic_acquire()) } + unsafe { self.start_ptr().add(self.len_atomic_acquire()) as *mut T } } /// Atomically adds an item to the end of the list without reallocation. diff --git a/tests/simple.rs b/tests/simple.rs index 2b42d85..2be342f 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -44,3 +44,12 @@ fn test_head_array() { v_orig.as_slice().iter().copied().collect::() ); } + +// This shown a miri error +#[test] +fn test_push() { + let mut hv = HeaderVec::with_capacity(10, ()); + + hv.push(123); + assert_eq!(hv[0], 123); +} From 58d68a14d5b6a67ece1c278ca262422a1a5e4459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 18:33:22 +0200 Subject: [PATCH 07/52] add: extend_from_slice() --- src/lib.rs | 29 +++++++++++++++++++++++++++++ tests/simple.rs | 9 +++++++++ 2 files changed, 38 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 5bcfb68..8fa9440 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -414,6 +414,13 @@ impl HeaderVec { unsafe { (self.ptr as *mut T).add(Self::offset()) } } + /// Gets the pointer to the end of the slice. This returns a mutable pointer to + /// uninitialized memory behind the last element. + #[inline(always)] + fn end_ptr_mut(&mut self) -> *mut T { + unsafe { self.start_ptr_mut().add(self.len_exact()) } + } + #[inline(always)] fn header(&self) -> &HeaderVecHeader { // The beginning of the memory is always the header. @@ -427,6 +434,28 @@ impl HeaderVec { } } +impl HeaderVec { + /// Adds items from a slice to the end of the list. + /// + /// Returns `Some(*const ())` if the memory was moved to a new location. + /// In this case, you are responsible for updating the weak nodes. + pub fn extend_from_slice(&mut self, slice: &[T]) -> Option<*const ()> { + let previous_pointer = self.reserve(slice.len()); + + // copy data + let end_ptr = self.end_ptr_mut(); + for (index, item) in slice.iter().enumerate() { + unsafe { + core::ptr::write(end_ptr.add(index), item.clone()); + } + } + // correct the len + self.header_mut().len = (self.len_exact() + slice.len()).into(); + + previous_pointer + } +} + #[cfg(feature = "atomic_append")] /// The atomic append API is only enabled when the `atomic_append` feature flag is set (which /// is the default). diff --git a/tests/simple.rs b/tests/simple.rs index 2be342f..82aa8f4 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -53,3 +53,12 @@ fn test_push() { hv.push(123); assert_eq!(hv[0], 123); } + +#[test] +fn test_extend_from_slice() { + let mut hv = HeaderVec::new(()); + + hv.extend_from_slice(&[0, 1, 2]); + hv.extend_from_slice(&[3, 4, 5]); + assert_eq!(hv.as_slice(), &[0, 1, 2, 3, 4, 5]); +} From 00be0d8ddb208360d84804d87aff66034cfbfd93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 19:27:33 +0200 Subject: [PATCH 08/52] add: extend_from_slice_atomic() --- src/lib.rs | 35 +++++++++++++++++++++++++++++++++++ tests/atomic_append.rs | 11 +++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 8fa9440..0e6bfe6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -529,6 +529,41 @@ impl HeaderVec { } } +#[cfg(feature = "atomic_append")] +impl HeaderVec { + /// Atomically add items from a slice to the end of the list. without reallocation + /// + /// # Errors + /// + /// If the vector is full, the item is returned. + /// + /// # Safety + /// + /// There must be only one thread calling this method at any time. Synchronization has to + /// be provided by the user. + pub unsafe fn extend_from_slice_atomic<'a>(&self, slice: &'a [T]) -> Result<(), &'a [T]> { + #[cfg(debug_assertions)] // only for the race check later + let len = self.len_atomic_relaxed(); + if self.spare_capacity() >= slice.len() { + // copy data + let end_ptr = self.end_ptr_atomic_mut(); + for (index, item) in slice.iter().enumerate() { + unsafe { + core::ptr::write(end_ptr.add(index), item.clone()); + } + } + // correct the len + let len_again = self.len_atomic_add_release(slice.len()); + // in debug builds we check for races, the chance to catch these are still pretty minimal + #[cfg(debug_assertions)] + debug_assert_eq!(len_again, len, "len was updated by another thread"); + Ok(()) + } else { + Err(slice) + } + } +} + impl Drop for HeaderVec { fn drop(&mut self) { unsafe { diff --git a/tests/atomic_append.rs b/tests/atomic_append.rs index a1112db..42f5535 100644 --- a/tests/atomic_append.rs +++ b/tests/atomic_append.rs @@ -14,3 +14,14 @@ fn test_atomic_append() { assert_eq!(hv.len(), 3); assert_eq!(hv.as_slice(), [1, 2, 3]); } + +#[test] +fn test_extend_from_slice() { + let hv = HeaderVec::with_capacity(6, ()); + + unsafe { + hv.extend_from_slice_atomic(&[0, 1, 2]).unwrap(); + hv.extend_from_slice_atomic(&[3, 4, 5]).unwrap(); + } + assert_eq!(hv.as_slice(), &[0, 1, 2, 3, 4, 5]); +} From 2b4fbf6643bfbeedc65f43c1238a35929ecf6bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 10 Jan 2025 01:55:54 +0100 Subject: [PATCH 09/52] ADD: spare_capacity_mut() and set_len() These methods have the same API/Semantic than the std::Vec methods. They are useful when one wants to extend a HeaderVec in place in some complex way like for example appending chars to a u8 headervec with `encode_utf8()` --- src/lib.rs | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0e6bfe6..a6c4749 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::{ fmt::Debug, - mem::{self, ManuallyDrop}, + mem::{self, ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut, Index, IndexMut}, ptr, slice::SliceIndex, @@ -372,6 +372,42 @@ impl HeaderVec { self.header_mut().len = head.into(); } + /// Returns the remaining spare capacity of the vector as a slice of + /// `MaybeUninit`. + /// + /// The returned slice can be used to fill the vector with data (e.g. by + /// reading from a file) before marking the data as initialized using the + /// [`set_len`] method. + /// + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + unsafe { + core::slice::from_raw_parts_mut( + self.end_ptr_mut() as *mut MaybeUninit, + self.spare_capacity(), + ) + } + } + + /// Forces the length of the headervec to `new_len`. + /// + /// This is a low-level operation that maintains none of the normal + /// invariants of the type. Normally changing the length of a vector + /// is done using one of the safe operations instead. Noteworthy is that + /// this method does not drop any of the elements that are removed when + /// shrinking the vector. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`capacity()`]. + /// - The elements at `old_len..new_len` must be initialized. + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!( + new_len <= self.capacity(), + "new_len is greater than capacity" + ); + self.header_mut().len = new_len.into(); + } + /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. #[inline(always)] const fn offset() -> usize { From d3a5a6f5c68d4314888750995928e2319b8b1c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Sat, 18 Jan 2025 21:43:06 +0100 Subject: [PATCH 10/52] add support for zero length HeaderVec Having a HeaderVec being zero length is mostly useless because it has to reallocated instanly when anything becomes pushed. This clearly should be avoided! Nevertheless supporting zero length takes out a corner-case and a potential panic and removes the burden for users explicitly ensuring zero length HeaderVecs don't happen in practice. Generally improving software reliability. --- src/lib.rs | 8 ++++---- tests/simple.rs | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a6c4749..62034d6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,6 @@ impl HeaderVec { } pub fn with_capacity(capacity: usize, head: H) -> Self { - assert!(capacity > 0, "HeaderVec capacity cannot be 0"); // Allocate the initial memory, which is uninitialized. let layout = Self::layout(capacity); let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut AlignedHeader; @@ -271,8 +270,6 @@ impl HeaderVec { "requested capacity is less than current length" ); let old_capacity = self.capacity(); - debug_assert_ne!(old_capacity, 0, "capacity of 0 not yet supported"); - debug_assert_ne!(requested_capacity, 0, "capacity of 0 not yet supported"); let new_capacity = if requested_capacity > old_capacity { if exact { @@ -281,11 +278,14 @@ impl HeaderVec { } else if requested_capacity <= old_capacity * 2 { // doubling the capacity is sufficient old_capacity * 2 - } else { + } else if old_capacity > 0 { // requested more than twice as much space, reserve the next multiple of // old_capacity that is greater than the requested capacity. This gives headroom // for new inserts while not doubling the memory requirement with bulk requests (requested_capacity / old_capacity + 1).saturating_mul(old_capacity) + } else { + // special case when we start at capacity 0 + requested_capacity } } else if exact { // exact shrinking diff --git a/tests/simple.rs b/tests/simple.rs index 82aa8f4..e530cfa 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -11,6 +11,17 @@ struct TestA { c: usize, } +#[test] +fn test_empty() { + let mut v_empty = HeaderVec::with_capacity(0, TestA { a: 4, b: !0, c: 66 }); + + assert_eq!(0, v_empty.len()); + assert_eq!(0, v_empty.capacity()); + assert_eq!(0, v_empty.as_slice().len()); + + v_empty.extend_from_slice("the quick brown fox jumps over the lazy dog".as_bytes()); +} + #[test] fn test_head_array() { let mut v_orig = HeaderVec::new(TestA { a: 4, b: !0, c: 66 }); From b34e0bd3df2d2d36febb2452d1fec18c4799d115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 20 Jan 2025 14:36:39 +0100 Subject: [PATCH 11/52] DOC: fixing doc for reserve_cold() --- src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0e6bfe6..ffa42b8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -255,8 +255,8 @@ impl HeaderVec { self.shrink_to(len) } - /// Resize the vector to have at least room for `additional` more elements. - /// does exact resizing if `exact` is true. + /// Resize the vector to least `requested_capacity` elements. + /// Does exact resizing if `exact` is true. /// /// Returns `Some(*const ())` if the memory was moved to a new location. /// @@ -265,7 +265,8 @@ impl HeaderVec { /// `requested_capacity` must be greater or equal than `self.len()` #[cold] unsafe fn resize_cold(&mut self, requested_capacity: usize, exact: bool) -> Option<*const ()> { - // For efficiency we do only a debug_assert here + // For efficiency we do only a debug_assert here, this is a internal unsafe function + // it's contract should be already enforced by the caller which is under our control debug_assert!( self.len_exact() <= requested_capacity, "requested capacity is less than current length" From 971332137ce2f40b2df1eb598411c79d1691f158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 20 Jan 2025 15:04:05 +0100 Subject: [PATCH 12/52] Make HeaderVec being a NonNull<> instead a raw pointer Since we always point to a valid allocation we can use NonNull here. This will benefit from the niche optimization: size_of::>() == size_of::>> Also adds a test to verfiy that HeaderVec are always lean and niche optimized. --- src/lib.rs | 35 +++++++++++++++++++---------------- tests/simple.rs | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ffa42b8..ab426b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ use core::{ mem::{self, ManuallyDrop}, ops::{Deref, DerefMut, Index, IndexMut}, ptr, + ptr::NonNull, slice::SliceIndex, }; @@ -51,7 +52,7 @@ union AlignedHeader { /// All of the data, like our header `OurHeaderType { a: 2 }`, the length of the vector: `2`, /// and the contents of the vector `['x', 'z']` resides on the other side of the pointer. pub struct HeaderVec { - ptr: *mut AlignedHeader, + ptr: NonNull>, } impl HeaderVec { @@ -65,10 +66,10 @@ impl HeaderVec { let layout = Self::layout(capacity); let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut AlignedHeader; - // Handle out-of-memory. - if ptr.is_null() { + let Some(ptr) = NonNull::new(ptr) else { + // Handle out-of-memory. alloc::alloc::handle_alloc_error(layout); - } + }; // Create self. let mut this = Self { ptr }; @@ -173,14 +174,14 @@ impl HeaderVec { /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`]. #[inline(always)] pub fn ptr(&self) -> *const () { - self.ptr as *const () + self.ptr.as_ptr() as *const () } /// This is used to check if this is the `HeaderVec` that corresponds to the given pointer. /// This is useful for updating weak references after [`HeaderVec::push`] returns the pointer. #[inline(always)] pub fn is(&self, ptr: *const ()) -> bool { - self.ptr as *const () == ptr + self.ptr.as_ptr() as *const () == ptr } /// Create a (dangerous) weak reference to the `HeaderVec`. This is useful to be able @@ -300,19 +301,21 @@ impl HeaderVec { // Reallocate the pointer. let ptr = unsafe { alloc::alloc::realloc( - self.ptr as *mut u8, + self.ptr.as_ptr() as *mut u8, Self::layout(old_capacity), Self::elems_to_mem_bytes(new_capacity), ) as *mut AlignedHeader }; - // Handle out-of-memory. - if ptr.is_null() { + + let Some(ptr) = NonNull::new(ptr) else { + // Handle out-of-memory. alloc::alloc::handle_alloc_error(Self::layout(new_capacity)); - } + }; + // Check if the new pointer is different than the old one. let previous_pointer = if ptr != self.ptr { // Give the user the old pointer so they can update everything. - Some(self.ptr as *const ()) + Some(self.ptr()) } else { None }; @@ -406,13 +409,13 @@ impl HeaderVec { /// Gets the pointer to the start of the slice. #[inline(always)] fn start_ptr(&self) -> *const T { - unsafe { (self.ptr as *const T).add(Self::offset()) } + unsafe { (self.ptr.as_ptr() as *const T).add(Self::offset()) } } /// Gets the pointer to the start of the slice. #[inline(always)] fn start_ptr_mut(&mut self) -> *mut T { - unsafe { (self.ptr as *mut T).add(Self::offset()) } + unsafe { (self.ptr.as_ptr() as *mut T).add(Self::offset()) } } /// Gets the pointer to the end of the slice. This returns a mutable pointer to @@ -425,13 +428,13 @@ impl HeaderVec { #[inline(always)] fn header(&self) -> &HeaderVecHeader { // The beginning of the memory is always the header. - unsafe { &*(self.ptr as *const HeaderVecHeader) } + unsafe { &*(self.ptr.as_ptr() as *const HeaderVecHeader) } } #[inline(always)] fn header_mut(&mut self) -> &mut HeaderVecHeader { // The beginning of the memory is always the header. - unsafe { &mut *(self.ptr as *mut HeaderVecHeader) } + unsafe { &mut *(self.ptr.as_ptr() as *mut HeaderVecHeader) } } } @@ -572,7 +575,7 @@ impl Drop for HeaderVec { for ix in 0..self.len_exact() { ptr::drop_in_place(self.start_ptr_mut().add(ix)); } - alloc::alloc::dealloc(self.ptr as *mut u8, Self::layout(self.capacity())); + alloc::alloc::dealloc(self.ptr.as_ptr() as *mut u8, Self::layout(self.capacity())); } } } diff --git a/tests/simple.rs b/tests/simple.rs index 82aa8f4..39f79bb 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -11,6 +11,20 @@ struct TestA { c: usize, } +#[test] +fn test_sizeof() { + // assert that HeaderVec is really a single lean pointer + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::<*mut ()>() + ); + // and has space for niche optimization + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::>>() + ); +} + #[test] fn test_head_array() { let mut v_orig = HeaderVec::new(TestA { a: 4, b: !0, c: 66 }); From e2b5f0d4f9d675453df6331f1c8405b56aecb77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 10 Jan 2025 01:55:54 +0100 Subject: [PATCH 13/52] ADD: spare_capacity_mut() and set_len() These methods have the same API/Semantic than the std::Vec methods. They are useful when one wants to extend a HeaderVec in place in some complex way like for example appending chars to a u8 headervec with `encode_utf8()` --- src/lib.rs | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ab426b9..e6875b3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::{ fmt::Debug, - mem::{self, ManuallyDrop}, + mem::{self, ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut, Index, IndexMut}, ptr, ptr::NonNull, @@ -376,6 +376,42 @@ impl HeaderVec { self.header_mut().len = head.into(); } + /// Returns the remaining spare capacity of the vector as a slice of + /// `MaybeUninit`. + /// + /// The returned slice can be used to fill the vector with data (e.g. by + /// reading from a file) before marking the data as initialized using the + /// [`set_len`] method. + /// + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + unsafe { + core::slice::from_raw_parts_mut( + self.end_ptr_mut() as *mut MaybeUninit, + self.spare_capacity(), + ) + } + } + + /// Forces the length of the headervec to `new_len`. + /// + /// This is a low-level operation that maintains none of the normal + /// invariants of the type. Normally changing the length of a vector + /// is done using one of the safe operations instead. Noteworthy is that + /// this method does not drop any of the elements that are removed when + /// shrinking the vector. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`capacity()`]. + /// - The elements at `old_len..new_len` must be initialized. + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!( + new_len <= self.capacity(), + "new_len is greater than capacity" + ); + self.header_mut().len = new_len.into(); + } + /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. #[inline(always)] const fn offset() -> usize { From 3fe809836d776658f0f8ac173357a6134f071469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Sat, 18 Jan 2025 21:43:06 +0100 Subject: [PATCH 14/52] add support for zero length HeaderVec Having a HeaderVec being zero length is mostly useless because it has to reallocated instanly when anything becomes pushed. This clearly should be avoided! Nevertheless supporting zero length takes out a corner-case and a potential panic and removes the burden for users explicitly ensuring zero length HeaderVecs don't happen in practice. Generally improving software reliability. --- src/lib.rs | 8 ++++---- tests/simple.rs | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e6875b3..a19396c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,7 +61,6 @@ impl HeaderVec { } pub fn with_capacity(capacity: usize, head: H) -> Self { - assert!(capacity > 0, "HeaderVec capacity cannot be 0"); // Allocate the initial memory, which is uninitialized. let layout = Self::layout(capacity); let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut AlignedHeader; @@ -273,8 +272,6 @@ impl HeaderVec { "requested capacity is less than current length" ); let old_capacity = self.capacity(); - debug_assert_ne!(old_capacity, 0, "capacity of 0 not yet supported"); - debug_assert_ne!(requested_capacity, 0, "capacity of 0 not yet supported"); let new_capacity = if requested_capacity > old_capacity { if exact { @@ -283,11 +280,14 @@ impl HeaderVec { } else if requested_capacity <= old_capacity * 2 { // doubling the capacity is sufficient old_capacity * 2 - } else { + } else if old_capacity > 0 { // requested more than twice as much space, reserve the next multiple of // old_capacity that is greater than the requested capacity. This gives headroom // for new inserts while not doubling the memory requirement with bulk requests (requested_capacity / old_capacity + 1).saturating_mul(old_capacity) + } else { + // special case when we start at capacity 0 + requested_capacity } } else if exact { // exact shrinking diff --git a/tests/simple.rs b/tests/simple.rs index 39f79bb..092f498 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -25,6 +25,17 @@ fn test_sizeof() { ); } +#[test] +fn test_empty() { + let mut v_empty = HeaderVec::with_capacity(0, TestA { a: 4, b: !0, c: 66 }); + + assert_eq!(0, v_empty.len()); + assert_eq!(0, v_empty.capacity()); + assert_eq!(0, v_empty.as_slice().len()); + + v_empty.extend_from_slice("the quick brown fox jumps over the lazy dog".as_bytes()); +} + #[test] fn test_head_array() { let mut v_orig = HeaderVec::new(TestA { a: 4, b: !0, c: 66 }); From 4cde654cf773f8f8c660f985052e9d8d711a26b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 20 Jan 2025 17:36:02 +0100 Subject: [PATCH 15/52] add a `std` feature flag For backwards compatibility this is not (yet) enabled by default. Will be used for all API's that require functionality provided by the rust stdlib. --- Cargo.toml | 1 + README.md | 6 ++++++ src/lib.rs | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4ba66aa..e14e79a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,4 @@ readme = "README.md" [features] default = ["atomic_append"] atomic_append = [] +std = [] \ No newline at end of file diff --git a/README.md b/README.md index 909497e..6a113df 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,9 @@ Allows one to store a header struct and a vector all inline in the same memory on the heap and share weak versions for minimizing random lookups in data structures If you use this without creating a weak ptr, it is safe. It is unsafe to create a weak pointer because you now have aliasing. + +## Features + +* `std` + Enables API's that requires stdlib features. Provides more compatibility to `Vec`. + This feature is not enabled by default. diff --git a/src/lib.rs b/src/lib.rs index a19396c..bb327d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![no_std] +#![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; From 9f18d10b1764d4ad997a43d45c752d9f8018f36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Wed, 22 Jan 2025 21:15:40 +0100 Subject: [PATCH 16/52] rename and export start_ptr()/mut to as_ptr()/mut This makes it comply with the std Vec's API --- src/lib.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bb327d8..80e668a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -162,12 +162,12 @@ impl HeaderVec { #[inline(always)] pub fn as_slice(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_strict()) } + unsafe { core::slice::from_raw_parts(self.as_ptr(), self.len_strict()) } } #[inline(always)] pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_exact()) } + unsafe { core::slice::from_raw_parts_mut(self.as_mut_ptr(), self.len_exact()) } } /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`]. @@ -180,7 +180,7 @@ impl HeaderVec { /// This is useful for updating weak references after [`HeaderVec::push`] returns the pointer. #[inline(always)] pub fn is(&self, ptr: *const ()) -> bool { - self.ptr.as_ptr() as *const () == ptr + self.ptr() == ptr } /// Create a (dangerous) weak reference to the `HeaderVec`. This is useful to be able @@ -301,7 +301,7 @@ impl HeaderVec { // Reallocate the pointer. let ptr = unsafe { alloc::alloc::realloc( - self.ptr.as_ptr() as *mut u8, + self.ptr() as *mut u8, Self::layout(old_capacity), Self::elems_to_mem_bytes(new_capacity), ) as *mut AlignedHeader @@ -336,7 +336,7 @@ impl HeaderVec { let new_len = old_len + 1; let previous_pointer = self.reserve(1); unsafe { - core::ptr::write(self.start_ptr_mut().add(old_len), item); + core::ptr::write(self.as_mut_ptr().add(old_len), item); } self.header_mut().len = new_len.into(); previous_pointer @@ -353,7 +353,7 @@ impl HeaderVec { let mut head = 0; let original_len = self.len_exact(); // Get the offset of the beginning of the slice. - let start_ptr = self.start_ptr_mut(); + let start_ptr = self.as_mut_ptr(); // Go through each index. for index in 0..original_len { unsafe { @@ -444,33 +444,33 @@ impl HeaderVec { /// Gets the pointer to the start of the slice. #[inline(always)] - fn start_ptr(&self) -> *const T { - unsafe { (self.ptr.as_ptr() as *const T).add(Self::offset()) } + pub fn as_ptr(&self) -> *const T { + unsafe { (self.ptr() as *const T).add(Self::offset()) } } /// Gets the pointer to the start of the slice. #[inline(always)] - fn start_ptr_mut(&mut self) -> *mut T { - unsafe { (self.ptr.as_ptr() as *mut T).add(Self::offset()) } + pub fn as_mut_ptr(&mut self) -> *mut T { + unsafe { (self.ptr() as *mut T).add(Self::offset()) } } /// Gets the pointer to the end of the slice. This returns a mutable pointer to /// uninitialized memory behind the last element. #[inline(always)] fn end_ptr_mut(&mut self) -> *mut T { - unsafe { self.start_ptr_mut().add(self.len_exact()) } + unsafe { self.as_mut_ptr().add(self.len_exact()) } } #[inline(always)] fn header(&self) -> &HeaderVecHeader { // The beginning of the memory is always the header. - unsafe { &*(self.ptr.as_ptr() as *const HeaderVecHeader) } + unsafe { &*(self.ptr() as *const HeaderVecHeader) } } #[inline(always)] fn header_mut(&mut self) -> &mut HeaderVecHeader { // The beginning of the memory is always the header. - unsafe { &mut *(self.ptr.as_ptr() as *mut HeaderVecHeader) } + unsafe { &mut *(self.ptr() as *mut HeaderVecHeader) } } } @@ -532,14 +532,14 @@ impl HeaderVec { #[inline(always)] pub fn as_slice_atomic_acquire(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } + unsafe { core::slice::from_raw_parts(self.as_ptr(), self.len_atomic_acquire()) } } /// Gets the pointer to the end of the slice. This returns a mutable pointer to /// uninitialized memory behind the last element. #[inline(always)] fn end_ptr_atomic_mut(&self) -> *mut T { - unsafe { self.start_ptr().add(self.len_atomic_acquire()) as *mut T } + unsafe { self.as_ptr().add(self.len_atomic_acquire()) as *mut T } } /// Atomically adds an item to the end of the list without reallocation. @@ -609,9 +609,9 @@ impl Drop for HeaderVec { unsafe { ptr::drop_in_place(&mut self.header_mut().head); for ix in 0..self.len_exact() { - ptr::drop_in_place(self.start_ptr_mut().add(ix)); + ptr::drop_in_place(self.as_mut_ptr().add(ix)); } - alloc::alloc::dealloc(self.ptr.as_ptr() as *mut u8, Self::layout(self.capacity())); + alloc::alloc::dealloc(self.ptr() as *mut u8, Self::layout(self.capacity())); } } } From 47af90f11c3b12d97ade1cc90f504e3c54c25fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Wed, 22 Jan 2025 22:05:18 +0100 Subject: [PATCH 17/52] FIX: Handle the case when resize_cold() would be a no-op This is a safety measure, normally resize_cold() wouldn't been called when here is nothing to do. But we want to ensure that if this ever happens we don't run into a panicking corner case. --- src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 80e668a..04d70ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -273,6 +273,11 @@ impl HeaderVec { ); let old_capacity = self.capacity(); + // Shortcut when nothing is to be done. + if requested_capacity == old_capacity { + return None; + } + let new_capacity = if requested_capacity > old_capacity { if exact { // exact growing From c2026c22beb1ff3d6e509807d6ea34c7689b5b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 23 Jan 2025 01:13:45 +0100 Subject: [PATCH 18/52] WIP: introduce WeakFixupFn, start removing Option<*const ()> This introduce a breaking change starting to modifying all methods that indicate a realloc by returning a Option(*const()) into stdlib compatible signatures and variants taking a closure for the fixup. This commits starts with reserve and shrink methods, more in the following commits. The `WeakFixupFn` is required for Drain and other iterators which do significant work in the destructor (possibly reallocating the headervec there). The closure can be passed along and called when required. Compatibility note: When it is not trivially possible to refactor fixup functionality to a closure then old code can be migrated by: let maybe_realloc: Option<*const ()> = { let mut opt_ptr = None; hv.reserve_with_weakfix(&mut self, 10000, |ptr| opt_ptr = Some(ptr)); opt_ptr }; // now maybe_realloc is like the old Option<*const ()> return if let Some(ptr) = maybe_realloc { // fixup code using ptr } Note 2: do we want legacy wrapper functions that have the old behavior eg. #[deprecated("upgrade to the new API")] pub fn reserve_legacy(&mut self, additional: usize) -> Option<*const ()> { let mut opt_ptr = None; self.reserve_with_weakfix(additional, |ptr| opt_ptr = Some(ptr)); opt_ptr }; --- src/lib.rs | 89 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 04d70ba..275369c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,10 @@ use core::{ #[cfg(feature = "atomic_append")] use core::sync::atomic::{AtomicUsize, Ordering}; +/// A closure that becomes called when a `HeaderVec` becomes reallocated. +/// This is closure is responsible for updating weak nodes. +pub type WeakFixupFn<'a> = &'a mut dyn FnMut(*const ()); + struct HeaderVecHeader { head: H, capacity: usize, @@ -220,39 +224,72 @@ impl HeaderVec { } /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. - #[inline(always)] - pub fn reserve(&mut self, additional: usize) -> Option<*const ()> { - if self.spare_capacity() < additional { - let len = self.len_exact(); - unsafe { self.resize_cold(len.saturating_add(additional), false) } - } else { - None - } + #[inline] + pub fn reserve(&mut self, additional: usize) { + self.reserve_intern(additional, false, None); + } + + /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. + #[inline] + pub fn reserve_with_weakfix(&mut self, additional: usize, weak_fixup: WeakFixupFn) { + self.reserve_intern(additional, false, Some(weak_fixup)); + } + + /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. + #[inline] + pub fn reserve_exact(&mut self, additional: usize) { + self.reserve_intern(additional, true, None); } /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. #[inline] - pub fn reserve_exact(&mut self, additional: usize) -> Option<*const ()> { + pub fn reserve_exact_with_weakfix(&mut self, additional: usize, weak_fixup: WeakFixupFn) { + self.reserve_intern(additional, true, Some(weak_fixup)); + } + + /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. + #[inline(always)] + fn reserve_intern(&mut self, additional: usize, exact: bool, weak_fixup: Option) { if self.spare_capacity() < additional { let len = self.len_exact(); - unsafe { self.resize_cold(len.saturating_add(additional), true) } - } else { - None + // using saturating_add here ensures that we get a allocation error instead wrapping over and + // allocating a total wrong size + unsafe { self.resize_cold(len.saturating_add(additional), exact, weak_fixup) }; } } /// Shrinks the capacity of the `HeaderVec` to the `min_capacity` or `self.len()`, whichever is larger. #[inline] - pub fn shrink_to(&mut self, min_capacity: usize) -> Option<*const ()> { + pub fn shrink_to(&mut self, min_capacity: usize) { let requested_capacity = self.len_exact().max(min_capacity); - unsafe { self.resize_cold(requested_capacity, true) } + unsafe { self.resize_cold(requested_capacity, true, None) }; + } + + /// Shrinks the capacity of the `HeaderVec` to the `min_capacity` or `self.len()`, whichever is larger. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. + #[inline] + pub fn shrink_to_with_weakfix(&mut self, min_capacity: usize, weak_fixup: WeakFixupFn) { + let requested_capacity = self.len_exact().max(min_capacity); + unsafe { self.resize_cold(requested_capacity, true, Some(weak_fixup)) }; + } + + /// Resizes the vector hold exactly `self.len()` elements. + #[inline(always)] + pub fn shrink_to_fit(&mut self) { + self.shrink_to(0); } /// Resizes the vector hold exactly `self.len()` elements. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. #[inline(always)] - pub fn shrink_to_fit(&mut self) -> Option<*const ()> { - let len = self.len_exact(); - self.shrink_to(len) + pub fn shrink_to_fit_with_weakfix(&mut self, weak_fixup: WeakFixupFn) { + self.shrink_to_with_weakfix(0, weak_fixup); } /// Resize the vector to least `requested_capacity` elements. @@ -264,7 +301,12 @@ impl HeaderVec { /// /// `requested_capacity` must be greater or equal than `self.len()` #[cold] - unsafe fn resize_cold(&mut self, requested_capacity: usize, exact: bool) -> Option<*const ()> { + unsafe fn resize_cold( + &mut self, + requested_capacity: usize, + exact: bool, + weak_fixup: Option, + ) { // For efficiency we do only a debug_assert here, this is a internal unsafe function // it's contract should be already enforced by the caller which is under our control debug_assert!( @@ -275,7 +317,7 @@ impl HeaderVec { // Shortcut when nothing is to be done. if requested_capacity == old_capacity { - return None; + return; } let new_capacity = if requested_capacity > old_capacity { @@ -319,7 +361,7 @@ impl HeaderVec { // Check if the new pointer is different than the old one. let previous_pointer = if ptr != self.ptr { - // Give the user the old pointer so they can update everything. + // Store old pointer for weak_fixup. Some(self.ptr()) } else { None @@ -329,7 +371,8 @@ impl HeaderVec { // And set the new capacity. self.header_mut().capacity = new_capacity; - previous_pointer + // Finally run the weak_fixup closure when provided + previous_pointer.map(|ptr| weak_fixup.map(|weak_fixup| weak_fixup(ptr))); } /// Adds an item to the end of the list. @@ -344,7 +387,7 @@ impl HeaderVec { core::ptr::write(self.as_mut_ptr().add(old_len), item); } self.header_mut().len = new_len.into(); - previous_pointer + todo!("weak_fixup transformartion") // previous_pointer } /// Retains only the elements specified by the predicate. @@ -497,7 +540,7 @@ impl HeaderVec { // correct the len self.header_mut().len = (self.len_exact() + slice.len()).into(); - previous_pointer + todo!("weak_fixup transformartion") // previous_pointer } } From ca899afa7c672e47c013ca015169ae07d5b5d754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 23 Jan 2025 17:31:57 +0100 Subject: [PATCH 19/52] CHANGE: modify push() using WeakFixupFn, add push_with_weakfix() --- src/lib.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 275369c..a12c68c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -376,18 +376,26 @@ impl HeaderVec { } /// Adds an item to the end of the list. - /// - /// Returns `Some(*const ())` if the memory was moved to a new location. - /// In this case, you are responsible for updating the weak nodes. - pub fn push(&mut self, item: T) -> Option<*const ()> { + pub fn push(&mut self, item: T) { + self.push_intern(item, None); + } + + /// Adds an item to the end of the list. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. + pub fn push_with_weakfix(&mut self, item: T, weak_fixup: WeakFixupFn) { + self.push_intern(item, Some(weak_fixup)); + } + + #[inline(always)] + fn push_intern(&mut self, item: T, weak_fixup: Option) { let old_len = self.len_exact(); let new_len = old_len + 1; - let previous_pointer = self.reserve(1); + self.reserve_intern(1, false, weak_fixup); unsafe { core::ptr::write(self.as_mut_ptr().add(old_len), item); } self.header_mut().len = new_len.into(); - todo!("weak_fixup transformartion") // previous_pointer } /// Retains only the elements specified by the predicate. From 9775e597c495ce527dabd0d0110330368f8d0f89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 23 Jan 2025 17:41:24 +0100 Subject: [PATCH 20/52] CHANGE: WeakFixupFn for extend_from_slice() --- src/lib.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a12c68c..8a5e0bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -532,11 +532,20 @@ impl HeaderVec { impl HeaderVec { /// Adds items from a slice to the end of the list. - /// - /// Returns `Some(*const ())` if the memory was moved to a new location. - /// In this case, you are responsible for updating the weak nodes. - pub fn extend_from_slice(&mut self, slice: &[T]) -> Option<*const ()> { - let previous_pointer = self.reserve(slice.len()); + pub fn extend_from_slice(&mut self, slice: &[T]) { + self.extend_from_slice_intern(slice, None) + } + + /// Adds items from a slice to the end of the list. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. + pub fn extend_from_slice_with_weakfix(&mut self, slice: &[T], weak_fixup: WeakFixupFn) { + self.extend_from_slice_intern(slice, Some(weak_fixup)); + } + + #[inline(always)] + fn extend_from_slice_intern(&mut self, slice: &[T], weak_fixup: Option) { + self.reserve_intern(slice.len(), false, weak_fixup); // copy data let end_ptr = self.end_ptr_mut(); @@ -547,8 +556,6 @@ impl HeaderVec { } // correct the len self.header_mut().len = (self.len_exact() + slice.len()).into(); - - todo!("weak_fixup transformartion") // previous_pointer } } From 71f42a1de92f536a376c19f2bc08d5b4258a9521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 24 Jan 2025 00:02:40 +0100 Subject: [PATCH 21/52] refactor: put HeaderVecWeak into its own module Things becoming bigger ;) --- src/lib.rs | 27 +++------------------------ src/weak.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 24 deletions(-) create mode 100644 src/weak.rs diff --git a/src/lib.rs b/src/lib.rs index 8a5e0bb..c1845b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,9 @@ use core::{ slice::SliceIndex, }; + +mod weak; +pub use weak::HeaderVecWeak; #[cfg(feature = "atomic_append")] use core::sync::atomic::{AtomicUsize, Ordering}; @@ -753,27 +756,3 @@ where .finish() } } - -pub struct HeaderVecWeak { - header_vec: ManuallyDrop>, -} - -impl Deref for HeaderVecWeak { - type Target = HeaderVec; - - fn deref(&self) -> &Self::Target { - &self.header_vec - } -} - -impl DerefMut for HeaderVecWeak { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.header_vec - } -} - -impl Debug for HeaderVecWeak { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("HeaderVecWeak").finish() - } -} diff --git a/src/weak.rs b/src/weak.rs new file mode 100644 index 0000000..6d1dd1f --- /dev/null +++ b/src/weak.rs @@ -0,0 +1,33 @@ +//! Weak reference to a `HeaderVec`. + +use core::{ + fmt::Debug, + mem::ManuallyDrop, + ops::{Deref, DerefMut}, +}; + +use crate::HeaderVec; + +pub struct HeaderVecWeak { + pub(crate) header_vec: ManuallyDrop>, +} + +impl Deref for HeaderVecWeak { + type Target = HeaderVec; + + fn deref(&self) -> &Self::Target { + &self.header_vec + } +} + +impl DerefMut for HeaderVecWeak { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.header_vec + } +} + +impl Debug for HeaderVecWeak { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("HeaderVecWeak").finish() + } +} From 0fd6d9a7b622ebc3394e72858f97af371a164de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 24 Jan 2025 00:12:04 +0100 Subject: [PATCH 22/52] ADD: a lot `impl From`'s These are mostly the std Vec compatible From impls. By default this creates `HeaderVec<(), T>` Additionally when one wants to pass a header one can do that by a `WithHeader(H,T)` tuple struct. The later is mostly for convenience. I took my liberty to introduce my xmacro crate here. The macro expands to ~120 lines of code. When this dependency is not wanted it could be replaced with the expanded code. --- Cargo.toml | 5 ++++- src/lib.rs | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e14e79a..ea56852 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,4 +14,7 @@ readme = "README.md" [features] default = ["atomic_append"] atomic_append = [] -std = [] \ No newline at end of file +std = [] + +[dependencies] +xmacro = "0.1.2" diff --git a/src/lib.rs b/src/lib.rs index c1845b7..4b94204 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,10 @@ use core::{ slice::SliceIndex, }; +#[cfg(feature = "std")] +use std::{ + borrow::Cow +}; mod weak; pub use weak::HeaderVecWeak; @@ -756,3 +760,38 @@ where .finish() } } + +/// A helper struct for using the `HeaderVec::from(WithHeader(H, T))` +pub struct WithHeader(pub H, pub T); + +xmacro::xmacro! { + // Generates a lot `impl From` for `HeaderVec<(), T>` and `HeaderVec` + // The later variant is initialized from a tuple (H,T). + $[ + from: lt: generics: where: conv: + (&[T]) () () () () + (&mut [T]) () () () () + (&[T; N]) () (const N: usize) () () + (&mut[T; N]) () (const N: usize) () () + ([T; N]) () (const N: usize) () (.as_ref()) + (Cow<'a, [T]>) ('a,) () (where [T]: ToOwned) (.as_ref()) + (Box<[T]>) () () () (.as_ref()) + (Vec) () () () (.as_ref()) + ] + + impl<$lt T: Clone, $generics> From<$from> for HeaderVec<(), T> $where { + fn from(from: $from) -> Self { + let mut hv = HeaderVec::new(()); + hv.extend_from_slice(from $conv); + hv + } + } + + impl<$lt H, T: Clone, $generics> From> for HeaderVec $where { + fn from(from: WithHeader) -> Self { + let mut hv = HeaderVec::new(from.0); + hv.extend_from_slice(from.1 $conv); + hv + } + } +} From 30430e6d7d13fe019b1a8cfc9f9bfaa82c9ae6fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 24 Jan 2025 00:29:49 +0100 Subject: [PATCH 23/52] Make `extemd_from_slice()` generic over AsRef<[T]> This simplifies the `From` impls and should generally be more useful. --- src/lib.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4b94204..87dd081 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ use core::{ fmt::Debug, mem::{self, ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut, Index, IndexMut}, + convert::{From, AsRef}, ptr, ptr::NonNull, slice::SliceIndex, @@ -539,15 +540,15 @@ impl HeaderVec { impl HeaderVec { /// Adds items from a slice to the end of the list. - pub fn extend_from_slice(&mut self, slice: &[T]) { - self.extend_from_slice_intern(slice, None) + pub fn extend_from_slice(&mut self, slice: impl AsRef<[T]>) { + self.extend_from_slice_intern(slice.as_ref(), None) } /// Adds items from a slice to the end of the list. /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for /// updating the weak references as additional parameter. - pub fn extend_from_slice_with_weakfix(&mut self, slice: &[T], weak_fixup: WeakFixupFn) { - self.extend_from_slice_intern(slice, Some(weak_fixup)); + pub fn extend_from_slice_with_weakfix(&mut self, slice: impl AsRef<[T]>, weak_fixup: WeakFixupFn) { + self.extend_from_slice_intern(slice.as_ref(), Some(weak_fixup)); } #[inline(always)] @@ -768,29 +769,29 @@ xmacro::xmacro! { // Generates a lot `impl From` for `HeaderVec<(), T>` and `HeaderVec` // The later variant is initialized from a tuple (H,T). $[ - from: lt: generics: where: conv: - (&[T]) () () () () - (&mut [T]) () () () () - (&[T; N]) () (const N: usize) () () - (&mut[T; N]) () (const N: usize) () () - ([T; N]) () (const N: usize) () (.as_ref()) - (Cow<'a, [T]>) ('a,) () (where [T]: ToOwned) (.as_ref()) - (Box<[T]>) () () () (.as_ref()) - (Vec) () () () (.as_ref()) + from: lt: generics: where: + (&[T]) () () () + (&mut [T]) () () () + (&[T; N]) () (const N: usize) () + (&mut[T; N]) () (const N: usize) () + ([T; N]) () (const N: usize) () + (Cow<'a, [T]>) ('a,) () (where [T]: ToOwned) + (Box<[T]>) () () () + (Vec) () () () ] impl<$lt T: Clone, $generics> From<$from> for HeaderVec<(), T> $where { fn from(from: $from) -> Self { let mut hv = HeaderVec::new(()); - hv.extend_from_slice(from $conv); + hv.extend_from_slice(from); hv } } - impl<$lt H, T: Clone, $generics> From> for HeaderVec $where { - fn from(from: WithHeader) -> Self { + impl<$lt H, T: Clone, $generics> From> for HeaderVec $where { + fn from(from: WithHeader) -> Self { let mut hv = HeaderVec::new(from.0); - hv.extend_from_slice(from.1 $conv); + hv.extend_from_slice(from.1); hv } } From f00e8ad29ccb61d0b2cc617fc758f6ea6188d5bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 24 Jan 2025 00:47:27 +0100 Subject: [PATCH 24/52] FIX: enable From Cow/Vec/Box only when std is enabled We could enabled these in non-std envorinments since HeaderVec depends on alloc, but i delay the decision about this for now. --- src/lib.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 87dd081..e7d7acc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -769,17 +769,22 @@ xmacro::xmacro! { // Generates a lot `impl From` for `HeaderVec<(), T>` and `HeaderVec` // The later variant is initialized from a tuple (H,T). $[ - from: lt: generics: where: - (&[T]) () () () - (&mut [T]) () () () - (&[T; N]) () (const N: usize) () - (&mut[T; N]) () (const N: usize) () - ([T; N]) () (const N: usize) () - (Cow<'a, [T]>) ('a,) () (where [T]: ToOwned) - (Box<[T]>) () () () - (Vec) () () () + attr: + from: lt: generics: where: + ()(&[T]) () () () + ()(&mut [T]) () () () + ()(&[T; N]) () (const N: usize) () + ()(&mut[T; N]) () (const N: usize) () + ()([T; N]) () (const N: usize) () + (#[cfg(feature = "std")]) + (Cow<'a, [T]>) ('a,) () (where [T]: ToOwned) + (#[cfg(feature = "std")]) + (Box<[T]>) () () () + (#[cfg(feature = "std")]) + (Vec) () () () ] + $attr impl<$lt T: Clone, $generics> From<$from> for HeaderVec<(), T> $where { fn from(from: $from) -> Self { let mut hv = HeaderVec::new(()); @@ -788,6 +793,7 @@ xmacro::xmacro! { } } + $attr impl<$lt H, T: Clone, $generics> From> for HeaderVec $where { fn from(from: WithHeader) -> Self { let mut hv = HeaderVec::new(from.0); From 20a54c7cb7e5e34265b2a7105f8b21973f769555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 24 Jan 2025 00:50:22 +0100 Subject: [PATCH 25/52] ADD: missing From<&str> for HeaderVec<.., u8> --- src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index e7d7acc..1192954 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -802,3 +802,19 @@ xmacro::xmacro! { } } } + +impl From<&str> for HeaderVec<(), u8> { + fn from(from: &str) -> Self { + let mut hv = HeaderVec::new(()); + hv.extend_from_slice(from.as_bytes()); + hv + } +} + +impl From> for HeaderVec { + fn from(from: WithHeader) -> Self { + let mut hv = HeaderVec::new(from.0); + hv.extend_from_slice(from.1.as_bytes()); + hv + } +} From 09d6827ffc27d368c0bacf97bb0d9336a03f7676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 24 Jan 2025 13:27:21 +0100 Subject: [PATCH 26/52] Add HeaderVec::from_header_slice(), simplify the From impl This removes the xmacro in favor of a generic From implementation for `H: Default` and data constructed from `AsRef<[T]>` --- Cargo.toml | 2 -- src/lib.rs | 79 +++++++++++++------------------------------------ tests/simple.rs | 13 ++++++++ 3 files changed, 34 insertions(+), 60 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ea56852..232f34c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,5 +16,3 @@ default = ["atomic_append"] atomic_append = [] std = [] -[dependencies] -xmacro = "0.1.2" diff --git a/src/lib.rs b/src/lib.rs index 1192954..fef44c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,19 +3,17 @@ extern crate alloc; use core::{ + convert::{AsRef, From}, fmt::Debug, mem::{self, ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut, Index, IndexMut}, - convert::{From, AsRef}, ptr, ptr::NonNull, slice::SliceIndex, }; #[cfg(feature = "std")] -use std::{ - borrow::Cow -}; +use std::{}; mod weak; pub use weak::HeaderVecWeak; @@ -539,6 +537,14 @@ impl HeaderVec { } impl HeaderVec { + /// Creates a new `HeaderVec` with the given header from some data. + pub fn from_header_slice(header: H, slice: impl AsRef<[T]>) -> Self { + let slice = slice.as_ref(); + let mut hv = Self::with_capacity(slice.len(), header); + hv.extend_from_slice_intern(slice, None); + hv + } + /// Adds items from a slice to the end of the list. pub fn extend_from_slice(&mut self, slice: impl AsRef<[T]>) { self.extend_from_slice_intern(slice.as_ref(), None) @@ -547,7 +553,11 @@ impl HeaderVec { /// Adds items from a slice to the end of the list. /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for /// updating the weak references as additional parameter. - pub fn extend_from_slice_with_weakfix(&mut self, slice: impl AsRef<[T]>, weak_fixup: WeakFixupFn) { + pub fn extend_from_slice_with_weakfix( + &mut self, + slice: impl AsRef<[T]>, + weak_fixup: WeakFixupFn, + ) { self.extend_from_slice_intern(slice.as_ref(), Some(weak_fixup)); } @@ -762,59 +772,12 @@ where } } -/// A helper struct for using the `HeaderVec::from(WithHeader(H, T))` -pub struct WithHeader(pub H, pub T); - -xmacro::xmacro! { - // Generates a lot `impl From` for `HeaderVec<(), T>` and `HeaderVec` - // The later variant is initialized from a tuple (H,T). - $[ - attr: - from: lt: generics: where: - ()(&[T]) () () () - ()(&mut [T]) () () () - ()(&[T; N]) () (const N: usize) () - ()(&mut[T; N]) () (const N: usize) () - ()([T; N]) () (const N: usize) () - (#[cfg(feature = "std")]) - (Cow<'a, [T]>) ('a,) () (where [T]: ToOwned) - (#[cfg(feature = "std")]) - (Box<[T]>) () () () - (#[cfg(feature = "std")]) - (Vec) () () () - ] - - $attr - impl<$lt T: Clone, $generics> From<$from> for HeaderVec<(), T> $where { - fn from(from: $from) -> Self { - let mut hv = HeaderVec::new(()); - hv.extend_from_slice(from); - hv - } - } - - $attr - impl<$lt H, T: Clone, $generics> From> for HeaderVec $where { - fn from(from: WithHeader) -> Self { - let mut hv = HeaderVec::new(from.0); - hv.extend_from_slice(from.1); - hv - } - } -} - -impl From<&str> for HeaderVec<(), u8> { - fn from(from: &str) -> Self { - let mut hv = HeaderVec::new(()); - hv.extend_from_slice(from.as_bytes()); - hv +impl From for HeaderVec +where + U: AsRef<[T]>, +{ + fn from(from: U) -> Self { + HeaderVec::from_header_slice(H::default(), from) } } -impl From> for HeaderVec { - fn from(from: WithHeader) -> Self { - let mut hv = HeaderVec::new(from.0); - hv.extend_from_slice(from.1.as_bytes()); - hv - } -} diff --git a/tests/simple.rs b/tests/simple.rs index 092f498..0ae1cc4 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -87,3 +87,16 @@ fn test_extend_from_slice() { hv.extend_from_slice(&[3, 4, 5]); assert_eq!(hv.as_slice(), &[0, 1, 2, 3, 4, 5]); } + +#[test] +fn test_from() { + assert_eq!(HeaderVec::<(), i32>::from(&[1, 2, 3]).as_slice(), [1, 2, 3]); +} + +#[test] +fn test_from_str() { + assert_eq!( + HeaderVec::<(), u8>::from("test").as_slice(), + "test".as_bytes() + ); +} From d8c717e2bbc86e4ac5945fec875cd623442babf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 30 Jan 2025 16:21:52 +0100 Subject: [PATCH 27/52] ADD: truncate() and clear() Code is taken from the stdlib Vec and adapted to HeaderVec --- src/lib.rs | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index fef44c9..f8f4bed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -474,6 +474,94 @@ impl HeaderVec { self.header_mut().len = new_len.into(); } + /// Shortens a `HeaderVec`, keeping the first `len` elements and dropping + /// the rest. + /// + /// If `len` is greater or equal to the vector's current length, this has + /// no effect. + /// + /// The [`drain`] method can emulate `truncate`, but causes the excess + /// elements to be returned instead of dropped. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// Truncating a five element `HeaderVec` to two elements: + /// + /// ``` + /// use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), _> = HeaderVec::from([1, 2, 3, 4, 5]); + /// hv.truncate(2); + /// assert_eq!(hv.as_slice(), [1, 2]); + /// ``` + /// + /// No truncation occurs when `len` is greater than the vector's current + /// length: + /// + /// ``` + /// use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), _> = HeaderVec::from([1, 2, 3]); + /// hv.truncate(8); + /// assert_eq!(hv.as_slice(), [1, 2, 3]); + /// ``` + /// + /// Truncating when `len == 0` is equivalent to calling the [`clear`] + /// method. + /// + /// ``` + /// use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), _> = HeaderVec::from([1, 2, 3]); + /// hv.truncate(0); + /// assert_eq!(hv.as_slice(), []); + /// ``` + /// + /// [`clear`]: HeaderVec::clear + pub fn truncate(&mut self, len: usize) { + unsafe { + let old_len = self.len_exact(); + if len > old_len { + return; + } + let remaining_len = old_len - len; + let s = ptr::slice_from_raw_parts_mut(self.as_mut_ptr().add(len), remaining_len); + self.header_mut().len = len.into(); + ptr::drop_in_place(s); + } + } + + /// Clears a `HeaderVec`, removing all values. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// ``` + /// use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), _> = HeaderVec::from([1, 2, 3]); + /// + /// hv.clear(); + /// + /// assert!(hv.is_empty()); + /// ``` + #[inline] + pub fn clear(&mut self) { + let elems: *mut [T] = self.as_mut_slice(); + + // SAFETY: + // - `elems` comes directly from `as_mut_slice` and is therefore valid. + // - Setting the length before calling `drop_in_place` means that, + // if an element's `Drop` impl panics, the vector's `Drop` impl will + // do nothing (leaking the rest of the elements) instead of dropping + // some twice. + unsafe { + self.set_len(0); + ptr::drop_in_place(elems); + } + } + /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. #[inline(always)] const fn offset() -> usize { From efc96c02c64bc98ef32a02892c46e3ffd5da8de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 30 Jan 2025 16:57:04 +0100 Subject: [PATCH 28/52] CHANGE: make the `std` feature default (breaks no_std) Software that uses this crate in a no_std environment will now have to clear the `std` flag manually. Rationale: Creating documentation and testing things that require `std` would require a lot conditional compilation tricks. Things would easily go under the Radar when buildiong doc and running tests with `no_std` being the default. Most users likely expect the `std` feature to be enabled by default. Still if it this is a problem we can keep `no_std` being the default with some effort. --- Cargo.toml | 4 ++-- README.md | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 232f34c..aaa236c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ license = "MIT" readme = "README.md" [features] -default = ["atomic_append"] -atomic_append = [] +default = ["std", "atomic_append"] std = [] +atomic_append = [] diff --git a/README.md b/README.md index 6a113df..da6227e 100644 --- a/README.md +++ b/README.md @@ -8,4 +8,8 @@ If you use this without creating a weak ptr, it is safe. It is unsafe to create * `std` Enables API's that requires stdlib features. Provides more compatibility to `Vec`. - This feature is not enabled by default. + This feature is enabled by default. +* `atomic_append` + Enables the `atomic_push` API's that allows extending a `HeaderVec` with interior + mutability from a single thread by a immutable handle. + This feature is enabled by default. From a421e3fd322e8b060e0ece9916e86e7f58f33aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 30 Jan 2025 17:08:46 +0100 Subject: [PATCH 29/52] ADD: `drain()` method and `Drain` iterator This add the drain functionality similar to std Vec's drain to HeaderVec. The `with_weakfix()` things are not needed for Drain (I was wrong in a earlier commit message) but they will be required for upcoming Splice functionality. Since vec::Drain depends on a few nightly features internally but we want to stay compatible with stable a few things are backported from nightly in `future_slice`. OTOH we can already stabilize Drain::keep_rest(). Most code was taken from std::vec and minimally adapted to work for HeaderVec. --- src/drain.rs | 242 ++++++++++++++++++++++++++++++++++++++++++++ src/future_slice.rs | 61 +++++++++++ src/lib.rs | 87 +++++++++++++++- tests/simple.rs | 11 ++ 4 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 src/drain.rs create mode 100644 src/future_slice.rs diff --git a/src/drain.rs b/src/drain.rs new file mode 100644 index 0000000..1542b7e --- /dev/null +++ b/src/drain.rs @@ -0,0 +1,242 @@ +#![cfg(feature = "std")] + +use core::{ + any::type_name, + fmt, + mem::{self}, + ptr::{self, NonNull}, +}; + +use std::{iter::FusedIterator, mem::ManuallyDrop, slice}; + +use crate::HeaderVec; + +/// A draining iterator for `HeaderVec`. +/// +/// This `struct` is created by [`HeaderVec::drain`]. +/// See its documentation for more. +/// +/// # Feature compatibility +/// +/// The `drain()` API and [`Drain`] iterator are only available when the `std` feature is +/// enabled. +/// +/// # Example +/// +/// ``` +/// # use header_vec::HeaderVec; +/// let mut hv: HeaderVec<(), _> = HeaderVec::from([0, 1, 2]); +/// let iter: header_vec::Drain<'_, _, _> = hv.drain(..); +/// ``` +pub struct Drain<'a, H, T> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +impl fmt::Debug for Drain<'_, H, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(&format!( + "Drain<{}, {}>", + type_name::(), + type_name::() + )) + .field("header", unsafe { self.vec.as_ref() }) + .field("iter", &self.iter.as_slice()) + .finish() + } +} + +impl Drain<'_, H, T> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// # use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), _> = HeaderVec::from(['a', 'b', 'c']); + /// let mut drain = hv.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } + + /// Keep unyielded elements in the source `HeaderVec`. + /// + /// # Examples + /// + /// ``` + /// # use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), _> = HeaderVec::from(['a', 'b', 'c']); + /// let mut drain = hv.drain(..); + /// + /// assert_eq!(drain.next().unwrap(), 'a'); + /// + /// // This call keeps 'b' and 'c' in the vec. + /// drain.keep_rest(); + /// + /// // If we wouldn't call `keep_rest()`, + /// // `hv` would be empty. + /// assert_eq!(hv.as_slice(), ['b', 'c']); + /// ``` + pub fn keep_rest(self) { + let mut this = ManuallyDrop::new(self); + + unsafe { + let source_vec = this.vec.as_mut(); + + let start = source_vec.len(); + let tail = this.tail_start; + + let unyielded_len = this.iter.len(); + let unyielded_ptr = this.iter.as_slice().as_ptr(); + + // ZSTs have no identity, so we don't need to move them around. + if std::mem::size_of::() != 0 { + let start_ptr = source_vec.as_mut_ptr().add(start); + + // memmove back unyielded elements + if unyielded_ptr != start_ptr { + let src = unyielded_ptr; + let dst = start_ptr; + + ptr::copy(src, dst, unyielded_len); + } + + // memmove back untouched tail + if tail != (start + unyielded_len) { + let src = source_vec.as_ptr().add(tail); + let dst = start_ptr.add(unyielded_len); + ptr::copy(src, dst, this.tail_len); + } + } + + source_vec.set_len(start + unyielded_len + this.tail_len); + } + } +} + +impl AsRef<[T]> for Drain<'_, H, T> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +unsafe impl Sync for Drain<'_, H, T> {} +unsafe impl Send for Drain<'_, H, T> {} + +impl Iterator for Drain<'_, H, T> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter + .next() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +impl DoubleEndedIterator for Drain<'_, H, T> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter + .next_back() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +impl Drop for Drain<'_, H, T> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, H, T>(&'r mut Drain<'a, H, T>); + + impl Drop for DropGuard<'_, '_, H, T> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.set_len(start + self.0.tail_len); + } + } + } + } + + let iter = mem::take(&mut self.iter); + let drop_len = iter.len(); + + let mut vec = self.vec; + + // unstable: if T::IS_ZST { instead we use size_of + if mem::size_of::() == 0 { + // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + unsafe { + let vec = vec.as_mut(); + let old_len = vec.len(); + vec.set_len(old_len + drop_len + self.tail_len); + vec.truncate(old_len + self.tail_len); + } + + return; + } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // it also gets touched by vec::Splice which may turn it into a dangling pointer + // which would make it and the vec pointer point to different allocations which would + // lead to invalid pointer arithmetic below. + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + + // PLANNED: let drop_offset = drop_ptr.sub_ptr(vec_ptr); is in nightly + let drop_offset = usize::try_from(drop_ptr.offset_from(vec_ptr)).unwrap_unchecked(); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +impl FusedIterator for Drain<'_, H, T> {} + +// PLANNED: unstable features +// impl ExactSizeIterator for Drain<'_, H, T> { +// fn is_empty(&self) -> bool { +// self.iter.is_empty() +// } +// } +// +// #[unstable(feature = "trusted_len", issue = "37572")] +// unsafe impl TrustedLen for Drain<'_, H, T> {} +// diff --git a/src/future_slice.rs b/src/future_slice.rs new file mode 100644 index 0000000..b6311a0 --- /dev/null +++ b/src/future_slice.rs @@ -0,0 +1,61 @@ +//! This module re-implements a unstable slice functions, these should be removed once they +//! are stabilized. These is copy-pasted with slight modifications from std::slice for +//! functions that do not need language magic. + +use std::ops; + +#[track_caller] +#[must_use] +pub(crate) fn range(range: R, bounds: ops::RangeTo) -> ops::Range +where + R: ops::RangeBounds, +{ + let len = bounds.end; + + let start = match range.start_bound() { + ops::Bound::Included(&start) => start, + ops::Bound::Excluded(start) => start + .checked_add(1) + .unwrap_or_else(|| slice_start_index_overflow_fail()), + ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + ops::Bound::Included(end) => end + .checked_add(1) + .unwrap_or_else(|| slice_end_index_overflow_fail()), + ops::Bound::Excluded(&end) => end, + ops::Bound::Unbounded => len, + }; + + if start > end { + slice_index_order_fail(start, end); + } + if end > len { + slice_end_index_len_fail(end, len); + } + + ops::Range { start, end } +} + +#[track_caller] +const fn slice_start_index_overflow_fail() -> ! { + panic!("attempted to index slice from after maximum usize"); +} + +#[track_caller] +const fn slice_end_index_overflow_fail() -> ! { + panic!("attempted to index slice up to maximum usize"); +} + +#[track_caller] +fn slice_index_order_fail(index: usize, end: usize) -> ! { + panic!("slice index start is larger than end, slice index starts at {index} but ends at {end}") +} + +#[track_caller] +fn slice_end_index_len_fail(index: usize, len: usize) -> ! { + panic!( + "slice end index is out of range for slice, range end index {index} out of range for slice of length {len}" + ) +} diff --git a/src/lib.rs b/src/lib.rs index f8f4bed..f7a22ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,10 +13,24 @@ use core::{ }; #[cfg(feature = "std")] -use std::{}; +use std::{ + // core::range::RangeBounds is unstable, we have to rely on std + ops::{Range, RangeBounds}, + slice, +}; mod weak; pub use weak::HeaderVecWeak; + +mod drain; +#[cfg(feature = "std")] +pub use drain::Drain; + +// To implement std/Vec compatibility we would need a few nightly features. +// For the time being we just reimplement them here until they become stabilized. +#[cfg(feature = "std")] +mod future_slice; + #[cfg(feature = "atomic_append")] use core::sync::atomic::{AtomicUsize, Ordering}; @@ -518,6 +532,7 @@ impl HeaderVec { /// ``` /// /// [`clear`]: HeaderVec::clear + /// [`drain`]: HeaderVec::drain pub fn truncate(&mut self, len: usize) { unsafe { let old_len = self.len_exact(); @@ -773,6 +788,76 @@ impl HeaderVec { } } +#[cfg(feature = "std")] +/// The methods that depend on stdlib features. +impl HeaderVec { + /// Removes the specified range from a `HeaderVec` in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the `HeaderVec` to optimize + /// its implementation. + /// + /// # Feature compatibility + /// + /// The `drain()` API and `Drain` iterator are only available when the `std` feature is + /// enabled. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// use header_vec::HeaderVec; + /// let mut v: HeaderVec<(), _> = HeaderVec::from(&[1, 2, 3]); + /// let u: Vec<_> = v.drain(1..).collect(); + /// assert_eq!(v.as_slice(), &[1]); + /// assert_eq!(u.as_slice(), &[2, 3]); + /// + /// // A full range clears the vector, like `clear()` does + /// v.drain(..); + /// assert_eq!(v.as_slice(), &[]); + /// ``` + pub fn drain(&mut self, range: R) -> Drain<'_, H, T> + where + R: RangeBounds, + { + // Memory safety + // + // When the Drain is first created, it shortens the length of + // the source vector to make sure no uninitialized or moved-from elements + // are accessible at all if the Drain's destructor never gets to run. + // + // Drain will ptr::read out the values to remove. + // When finished, remaining tail of the vec is copied back to cover + // the hole, and the vector length is restored to the new length. + // + let len = self.len(); + let Range { start, end } = future_slice::range(range, ..len); + + unsafe { + // set self.vec length's to start, to be safe in case Drain is leaked + self.set_len(start); + let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } +} + impl Drop for HeaderVec { fn drop(&mut self) { unsafe { diff --git a/tests/simple.rs b/tests/simple.rs index 0ae1cc4..b10f231 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -100,3 +100,14 @@ fn test_from_str() { "test".as_bytes() ); } + +#[cfg(feature = "std")] +#[test] +fn test_drain() { + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let drain = hv.drain(1..4); + assert_eq!(drain.as_slice(), [2, 3, 4]); + drop(drain); + assert_eq!(hv.as_slice(), [1, 5, 6]); +} From 546d1af2b800b3c3391937e9ac9fa7c1acb88a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 30 Jan 2025 22:58:13 +0100 Subject: [PATCH 30/52] cosmetic fixes/ trivial lints --- src/lib.rs | 2 +- tests/simple.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f7a22ad..8b4b295 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,7 @@ union AlignedHeader { /// hv.push('z'); /// ``` /// -/// [`HeaderVec`] itself consists solely of a pointer, it's only 8 bytes big. +/// [`HeaderVec`] itself consists solely of a non-null pointer, it's only 8 bytes big. /// All of the data, like our header `OurHeaderType { a: 2 }`, the length of the vector: `2`, /// and the contents of the vector `['x', 'z']` resides on the other side of the pointer. pub struct HeaderVec { diff --git a/tests/simple.rs b/tests/simple.rs index b10f231..afb28d7 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -83,9 +83,9 @@ fn test_push() { fn test_extend_from_slice() { let mut hv = HeaderVec::new(()); - hv.extend_from_slice(&[0, 1, 2]); - hv.extend_from_slice(&[3, 4, 5]); - assert_eq!(hv.as_slice(), &[0, 1, 2, 3, 4, 5]); + hv.extend_from_slice([0, 1, 2]); + hv.extend_from_slice([3, 4, 5]); + assert_eq!(hv.as_slice(), [0, 1, 2, 3, 4, 5]); } #[test] From 61435a6678b29af8018f086e9a73dc79400e33f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 30 Jan 2025 23:35:51 +0100 Subject: [PATCH 31/52] ADD: iter()/iter_mut(), impl IntoIterator These are prerequisite for 'impl Extend' which is prerequisite for splicing, coming soon. --- src/lib.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 8b4b295..65e6219 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -954,3 +954,31 @@ where } } +impl HeaderVec { + pub fn iter(&self) -> slice::Iter<'_, T> { + self.as_slice().iter() + } + + pub fn iter_mut(&mut self) -> slice::IterMut<'_, T> { + self.as_mut_slice().iter_mut() + } +} + +impl<'a, H, T> IntoIterator for &'a HeaderVec { + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, H, T> IntoIterator for &'a mut HeaderVec { + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + From e3d1dc7c23a1028e14c5191e75a922fef436ec2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 30 Jan 2025 23:37:37 +0100 Subject: [PATCH 32/52] move slice dependency fom std to core --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 65e6219..2b22318 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ use core::{ ops::{Deref, DerefMut, Index, IndexMut}, ptr, ptr::NonNull, + slice, slice::SliceIndex, }; @@ -16,7 +17,6 @@ use core::{ use std::{ // core::range::RangeBounds is unstable, we have to rely on std ops::{Range, RangeBounds}, - slice, }; mod weak; From 1d9062b5ddac1f06258734df02c8711c70ae24d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 31 Jan 2025 04:35:30 +0100 Subject: [PATCH 33/52] ADD: impl Extend and Extend<&T> --- src/lib.rs | 18 ++++++++++++++++++ tests/simple.rs | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 2b22318..490e079 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -982,3 +982,21 @@ impl<'a, H, T> IntoIterator for &'a mut HeaderVec { } } +impl Extend for HeaderVec { + #[inline] + #[track_caller] + fn extend>(&mut self, iter: I) { + iter.into_iter().for_each(|item| self.push(item)); + } +} + +/// Extend implementation that copies elements out of references before pushing them onto the Vec. +// Note: from std Vec: not implemented here yet +// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to +// append the entire slice at once. +impl<'a, H, T: Copy + 'a> Extend<&'a T> for HeaderVec { + #[track_caller] + fn extend>(&mut self, iter: I) { + iter.into_iter().for_each(|item| self.push(*item)); + } +} diff --git a/tests/simple.rs b/tests/simple.rs index afb28d7..8127117 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -111,3 +111,19 @@ fn test_drain() { drop(drain); assert_eq!(hv.as_slice(), [1, 5, 6]); } + +#[cfg(feature = "std")] +#[test] +fn test_extend() { + let mut hv = HeaderVec::new(()); + hv.extend([1, 2, 3]); + assert_eq!(hv.as_slice(), [1, 2, 3]); +} + +#[cfg(feature = "std")] +#[test] +fn test_extend_ref() { + let mut hv = HeaderVec::<(), i32>::new(()); + hv.extend([&1, &2, &3]); + assert_eq!(hv.as_slice(), [1, 2, 3]); +} From 76163d6cf8dbe2c2ed6b515f6853895ce6085dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 31 Jan 2025 05:13:13 +0100 Subject: [PATCH 34/52] pass the WeakFixupFn by &mut internally This should make no difference, but the upcoming Slice needs to use it twice so we can't pass by value. --- src/lib.rs | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 490e079..3c17a3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -246,7 +246,7 @@ impl HeaderVec { /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. #[inline] pub fn reserve(&mut self, additional: usize) { - self.reserve_intern(additional, false, None); + self.reserve_intern(additional, false, &mut None); } /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. @@ -254,13 +254,13 @@ impl HeaderVec { /// updating the weak references as additional parameter. #[inline] pub fn reserve_with_weakfix(&mut self, additional: usize, weak_fixup: WeakFixupFn) { - self.reserve_intern(additional, false, Some(weak_fixup)); + self.reserve_intern(additional, false, &mut Some(weak_fixup)); } /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. #[inline] pub fn reserve_exact(&mut self, additional: usize) { - self.reserve_intern(additional, true, None); + self.reserve_intern(additional, true, &mut None); } /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. @@ -268,12 +268,17 @@ impl HeaderVec { /// updating the weak references as additional parameter. #[inline] pub fn reserve_exact_with_weakfix(&mut self, additional: usize, weak_fixup: WeakFixupFn) { - self.reserve_intern(additional, true, Some(weak_fixup)); + self.reserve_intern(additional, true, &mut Some(weak_fixup)); } /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. #[inline(always)] - fn reserve_intern(&mut self, additional: usize, exact: bool, weak_fixup: Option) { + pub(crate) fn reserve_intern( + &mut self, + additional: usize, + exact: bool, + weak_fixup: &mut Option, + ) { if self.spare_capacity() < additional { let len = self.len_exact(); // using saturating_add here ensures that we get a allocation error instead wrapping over and @@ -286,7 +291,7 @@ impl HeaderVec { #[inline] pub fn shrink_to(&mut self, min_capacity: usize) { let requested_capacity = self.len_exact().max(min_capacity); - unsafe { self.resize_cold(requested_capacity, true, None) }; + unsafe { self.resize_cold(requested_capacity, true, &mut None) }; } /// Shrinks the capacity of the `HeaderVec` to the `min_capacity` or `self.len()`, whichever is larger. @@ -295,7 +300,7 @@ impl HeaderVec { #[inline] pub fn shrink_to_with_weakfix(&mut self, min_capacity: usize, weak_fixup: WeakFixupFn) { let requested_capacity = self.len_exact().max(min_capacity); - unsafe { self.resize_cold(requested_capacity, true, Some(weak_fixup)) }; + unsafe { self.resize_cold(requested_capacity, true, &mut Some(weak_fixup)) }; } /// Resizes the vector hold exactly `self.len()` elements. @@ -325,7 +330,7 @@ impl HeaderVec { &mut self, requested_capacity: usize, exact: bool, - weak_fixup: Option, + weak_fixup: &mut Option, ) { // For efficiency we do only a debug_assert here, this is a internal unsafe function // it's contract should be already enforced by the caller which is under our control @@ -392,23 +397,23 @@ impl HeaderVec { self.header_mut().capacity = new_capacity; // Finally run the weak_fixup closure when provided - previous_pointer.map(|ptr| weak_fixup.map(|weak_fixup| weak_fixup(ptr))); + previous_pointer.map(|ptr| weak_fixup.as_mut().map(|weak_fixup| weak_fixup(ptr))); } /// Adds an item to the end of the list. pub fn push(&mut self, item: T) { - self.push_intern(item, None); + self.push_intern(item, &mut None); } /// Adds an item to the end of the list. /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for /// updating the weak references as additional parameter. pub fn push_with_weakfix(&mut self, item: T, weak_fixup: WeakFixupFn) { - self.push_intern(item, Some(weak_fixup)); + self.push_intern(item, &mut Some(weak_fixup)); } #[inline(always)] - fn push_intern(&mut self, item: T, weak_fixup: Option) { + fn push_intern(&mut self, item: T, weak_fixup: &mut Option) { let old_len = self.len_exact(); let new_len = old_len + 1; self.reserve_intern(1, false, weak_fixup); @@ -644,13 +649,13 @@ impl HeaderVec { pub fn from_header_slice(header: H, slice: impl AsRef<[T]>) -> Self { let slice = slice.as_ref(); let mut hv = Self::with_capacity(slice.len(), header); - hv.extend_from_slice_intern(slice, None); + hv.extend_from_slice_intern(slice, &mut None); hv } /// Adds items from a slice to the end of the list. pub fn extend_from_slice(&mut self, slice: impl AsRef<[T]>) { - self.extend_from_slice_intern(slice.as_ref(), None) + self.extend_from_slice_intern(slice.as_ref(), &mut None) } /// Adds items from a slice to the end of the list. @@ -661,11 +666,11 @@ impl HeaderVec { slice: impl AsRef<[T]>, weak_fixup: WeakFixupFn, ) { - self.extend_from_slice_intern(slice.as_ref(), Some(weak_fixup)); + self.extend_from_slice_intern(slice.as_ref(), &mut Some(weak_fixup)); } #[inline(always)] - fn extend_from_slice_intern(&mut self, slice: &[T], weak_fixup: Option) { + fn extend_from_slice_intern(&mut self, slice: &[T], weak_fixup: &mut Option) { self.reserve_intern(slice.len(), false, weak_fixup); // copy data From c42d26f1b37b32605a9c5796bd5d565753c3ea9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 13 Feb 2025 22:40:27 +0100 Subject: [PATCH 35/52] replace Drain::tail_len with tail_end * add tail_len() method upcoming splice needs this, otherwise incrementing the tail_start would require decrementing tail_len as well. --- src/drain.rs | 22 +++++++++++++--------- src/lib.rs | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/drain.rs b/src/drain.rs index 1542b7e..82f69cf 100644 --- a/src/drain.rs +++ b/src/drain.rs @@ -31,8 +31,8 @@ use crate::HeaderVec; pub struct Drain<'a, H, T> { /// Index of tail to preserve pub(super) tail_start: usize, - /// Length of tail - pub(super) tail_len: usize, + /// End index of tail to preserve + pub(super) tail_end: usize, /// Current remaining range to remove pub(super) iter: slice::Iter<'a, T>, pub(super) vec: NonNull>, @@ -115,13 +115,17 @@ impl Drain<'_, H, T> { if tail != (start + unyielded_len) { let src = source_vec.as_ptr().add(tail); let dst = start_ptr.add(unyielded_len); - ptr::copy(src, dst, this.tail_len); + ptr::copy(src, dst, this.tail_len()); } } - source_vec.set_len(start + unyielded_len + this.tail_len); + source_vec.set_len(start + unyielded_len + this.tail_len()); } } + + pub(crate) fn tail_len(&self) -> usize { + self.tail_end - self.tail_start + } } impl AsRef<[T]> for Drain<'_, H, T> { @@ -164,7 +168,7 @@ impl Drop for Drain<'_, H, T> { impl Drop for DropGuard<'_, '_, H, T> { fn drop(&mut self) { - if self.0.tail_len > 0 { + if self.0.tail_len() > 0 { unsafe { let source_vec = self.0.vec.as_mut(); // memmove back untouched tail, update to new length @@ -173,9 +177,9 @@ impl Drop for Drain<'_, H, T> { if tail != start { let src = source_vec.as_ptr().add(tail); let dst = source_vec.as_mut_ptr().add(start); - ptr::copy(src, dst, self.0.tail_len); + ptr::copy(src, dst, self.0.tail_len()); } - source_vec.set_len(start + self.0.tail_len); + source_vec.set_len(start + self.0.tail_len()); } } } @@ -193,8 +197,8 @@ impl Drop for Drain<'_, H, T> { unsafe { let vec = vec.as_mut(); let old_len = vec.len(); - vec.set_len(old_len + drop_len + self.tail_len); - vec.truncate(old_len + self.tail_len); + vec.set_len(old_len + drop_len + self.tail_len()); + vec.truncate(old_len + self.tail_len()); } return; diff --git a/src/lib.rs b/src/lib.rs index 3c17a3d..62a54ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -855,7 +855,7 @@ impl HeaderVec { let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); Drain { tail_start: end, - tail_len: len - end, + tail_end: len, iter: range_slice.iter(), vec: NonNull::from(self), } From 3e9047e7201b7cf7a95bc37ba2ffef786c750e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 14 Feb 2025 02:08:13 +0100 Subject: [PATCH 36/52] assert that T is not a ZST HeaderVec does not support ZST's, this will give a compiletime error when one tries to do so. Before it was a Divide by Zero runtime error in the offset calculation. Remove the ZST code from Drain too (was some remains from std Drain) Its unlikely that supporting ZST's in HeaderVec makes any sense. Forbiding this altogether make this clear. When someone comes up with a real use case this may be reviewed. --- src/drain.rs | 41 ++++++++++++----------------------------- src/lib.rs | 1 + 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/src/drain.rs b/src/drain.rs index 82f69cf..fa6aa5f 100644 --- a/src/drain.rs +++ b/src/drain.rs @@ -99,24 +99,21 @@ impl Drain<'_, H, T> { let unyielded_len = this.iter.len(); let unyielded_ptr = this.iter.as_slice().as_ptr(); - // ZSTs have no identity, so we don't need to move them around. - if std::mem::size_of::() != 0 { - let start_ptr = source_vec.as_mut_ptr().add(start); + let start_ptr = source_vec.as_mut_ptr().add(start); - // memmove back unyielded elements - if unyielded_ptr != start_ptr { - let src = unyielded_ptr; - let dst = start_ptr; + // memmove back unyielded elements + if unyielded_ptr != start_ptr { + let src = unyielded_ptr; + let dst = start_ptr; - ptr::copy(src, dst, unyielded_len); - } + ptr::copy(src, dst, unyielded_len); + } - // memmove back untouched tail - if tail != (start + unyielded_len) { - let src = source_vec.as_ptr().add(tail); - let dst = start_ptr.add(unyielded_len); - ptr::copy(src, dst, this.tail_len()); - } + // memmove back untouched tail + if tail != (start + unyielded_len) { + let src = source_vec.as_ptr().add(tail); + let dst = start_ptr.add(unyielded_len); + ptr::copy(src, dst, this.tail_len()); } source_vec.set_len(start + unyielded_len + this.tail_len()); @@ -190,20 +187,6 @@ impl Drop for Drain<'_, H, T> { let mut vec = self.vec; - // unstable: if T::IS_ZST { instead we use size_of - if mem::size_of::() == 0 { - // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. - // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. - unsafe { - let vec = vec.as_mut(); - let old_len = vec.len(); - vec.set_len(old_len + drop_len + self.tail_len()); - vec.truncate(old_len + self.tail_len()); - } - - return; - } - // ensure elements are moved back into their appropriate places, even when drop_in_place panics let _guard = DropGuard(self); diff --git a/src/lib.rs b/src/lib.rs index 62a54ab..08677e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,6 +85,7 @@ impl HeaderVec { } pub fn with_capacity(capacity: usize, head: H) -> Self { + const { assert!(mem::size_of::() > 0, "HeaderVec does not support ZST's") }; // Allocate the initial memory, which is uninitialized. let layout = Self::layout(capacity); let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut AlignedHeader; From 91fa7326fb117258d922b4e55f586f9753573107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 14 Feb 2025 02:32:33 +0100 Subject: [PATCH 37/52] splice() method and Splice iterator This implements splicing. This allows for insert/replace operations in HeaderVecs. The semantic is the same as the stdlib splice. But we use a slightly different algorithm internally: * The stdlib Splice collects excess replace_with elements in a temporary vector and later fills this in. in some/worst cases this may cause the tail to be moved arouind two times. * Our implementation pushes objects from `replace_with` directly onto the source HeaderVec If tail elements would be overwritten they are stashed on a temporary vector. Only these stashed elements are moved twice, the remainder of the tail is moved only once. Note: moving elements because of reallocation is not accounted here. In both implementations this can happen equally bad. The stdlib can mitigate that by this by some unstable features. We try hard to do it as best as possible. Benchmarks will follow. --- src/lib.rs | 82 +++++++++++++++++++++++ src/splice.rs | 180 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 src/splice.rs diff --git a/src/lib.rs b/src/lib.rs index 08677e2..75e9378 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,10 @@ mod drain; #[cfg(feature = "std")] pub use drain::Drain; +mod splice; +#[cfg(feature = "std")] +pub use splice::Splice; + // To implement std/Vec compatibility we would need a few nightly features. // For the time being we just reimplement them here until they become stabilized. #[cfg(feature = "std")] @@ -862,6 +866,84 @@ impl HeaderVec { } } } + + /// Creates a splicing iterator that replaces the specified range in the vector + /// with the given `replace_with` iterator and yields the removed items. + /// `replace_with` does not need to be the same length as `range`. + /// + /// `range` is removed even if the iterator is not consumed until the end. + /// + /// It is unspecified how many elements are removed from the vector + /// if the `Splice` value is leaked. + /// + /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. + /// + /// This is optimal if: + /// + /// * The tail (elements in the vector after `range`) is empty, + /// * or `replace_with` yields fewer or equal elements than `range`’s length + /// * or the lower bound of its `size_hint()` is exact. + /// + /// Otherwise, a temporary vector is allocated to store the tail elements which are in the way. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Examples + /// + /// ``` + /// use header_vec::HeaderVec; + /// let mut hv: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4]); + /// let new = [7, 8, 9]; + /// let u: Vec<_> = hv.splice(1..3, new).collect(); + /// assert_eq!(hv.as_slice(), [1, 7, 8, 9, 4]); + /// assert_eq!(u, [2, 3]); + /// ``` + #[inline] + pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, H, I::IntoIter> + where + R: RangeBounds, + I: IntoIterator, + { + self.splice_internal(range, replace_with, None) + } + + /// Creates a splicing iterator like [`splice()`]. + /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for + /// updating the weak references as additional parameter. + #[inline] + pub fn splice_with_weakfix<'a, R, I>( + &'a mut self, + range: R, + replace_with: I, + weak_fixup: WeakFixupFn<'a>, + ) -> Splice<'a, H, I::IntoIter> + where + R: RangeBounds, + I: IntoIterator, + { + self.splice_internal(range, replace_with, Some(weak_fixup)) + } + + #[inline(always)] + fn splice_internal<'a, R, I>( + &'a mut self, + range: R, + replace_with: I, + weak_fixup: Option>, + ) -> Splice<'a, H, I::IntoIter> + where + R: RangeBounds, + I: IntoIterator, + { + Splice { + drain: self.drain(range), + replace_with: replace_with.into_iter(), + weak_fixup, + } + } } impl Drop for HeaderVec { diff --git a/src/splice.rs b/src/splice.rs new file mode 100644 index 0000000..802c608 --- /dev/null +++ b/src/splice.rs @@ -0,0 +1,180 @@ +#![cfg(feature = "std")] + +use core::{any::type_name, fmt, ptr}; + +use crate::{Drain, WeakFixupFn}; + +/// A splicing iterator for a `HeaderVec`. +/// +/// This struct is created by [`Vec::splice()`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// # use header_vec::HeaderVec; +/// let mut hv: HeaderVec<(), _> = HeaderVec::from([0, 1, 2]); +/// let new = [7, 8]; +/// let iter = hv.splice(1.., new); +/// ``` +pub struct Splice<'a, H, I: Iterator + 'a> { + pub(super) drain: Drain<'a, H, I::Item>, + pub(super) replace_with: I, + pub(super) weak_fixup: Option>, +} + +impl Iterator for Splice<'_, H, I> { + type Item = I::Item; + + fn next(&mut self) -> Option { + self.drain.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.drain.size_hint() + } +} + +impl Splice<'_, H, I> { + /// Not a standard function, might be useful nevertheless, we use it in tests. + pub fn drained_slice(&self) -> &[I::Item] { + self.drain.as_slice() + } +} + +impl DoubleEndedIterator for Splice<'_, H, I> { + fn next_back(&mut self) -> Option { + self.drain.next_back() + } +} + +impl ExactSizeIterator for Splice<'_, H, I> {} + +impl fmt::Debug for Splice<'_, H, I> +where + I: Iterator + fmt::Debug, + I::Item: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(&format!( + "Splice<{}, {}>", + type_name::(), + type_name::() + )) + .field("drain", &self.drain.as_slice()) + .field("replace_with", &self.replace_with) + .field("weak_fixup", &self.weak_fixup.is_some()) + .finish() + } +} + +impl Drop for Splice<'_, H, I> { + #[track_caller] + fn drop(&mut self) { + self.drain.by_ref().for_each(drop); + // At this point draining is done and the only remaining tasks are splicing + // and moving things into the final place. + // Which means we can replace the slice::Iter with pointers that won't point to deallocated + // memory, so that Drain::drop is still allowed to call iter.len(), otherwise it would break + // the ptr.sub_ptr contract. + self.drain.iter = [].iter(); + + // We will use the replace_with iterator to append elements in place on self.drain.vec. + // When this hits the tail then elements are moved from the tail to tmp_tail. + // When the tail is or becomes empty by that, then the remaining elements can be extended to the vec. + // + // Finally: + // Then have continuous elements in the vec: |head|replace_with|(old_tail|)spare_capacity|. + // The old tail needs to be moved to its final destination. + // Perhaps making space for the elements in the tmp_tail. + let mut tmp_tail = Vec::new(); + + unsafe { + let vec = self.drain.vec.as_mut(); + loop { + if self.drain.tail_len() == 0 { + // If the tail is empty, we can just extend the vector with the remaining elements. + // but we may have stashed some tmp_tail away and should reserve for that. + // PLANNED: should become 'extend_reserve()' + vec.reserve_intern( + self.replace_with.size_hint().0 + tmp_tail.len(), + false, + &mut self.weak_fixup, + ); + vec.extend(self.replace_with.by_ref()); + // in case the size_hint was not exact (or returned 0) we need to reserve for the tmp_tail + // in most cases this will not allocate. later we expect that we have this space reserved. + vec.reserve_intern(tmp_tail.len(), false, &mut self.weak_fixup); + break; + } else if let Some(next) = self.replace_with.next() { + if vec.len_exact() >= self.drain.tail_start && self.drain.tail_len() > 0 { + // move one element from the tail to the tmp_tail + // We reserve for as much elements are hinted by replace_with or the remaining tail, + // whatever is smaller. + tmp_tail + .reserve(self.replace_with.size_hint().0.min(self.drain.tail_len())); + tmp_tail.push(ptr::read(vec.as_ptr().add(self.drain.tail_start))); + self.drain.tail_start += 1; + } + + // since we overwrite the old tail here this will never reallocate. + // PLANNED: vec.push_within_capacity().unwrap_unchecked() + vec.push(next); + } else { + // replace_with is depleted + break; + } + } + + let tail_len = self.drain.tail_len(); + if tail_len > 0 { + // In case we need to shift the tail farther back we need to reserve space for that. + // Reserve needs to preserve the tail we have, thus we temporarily set the length to the + // tail_end and then restore it after the reserve. + let old_len = vec.len_exact(); + vec.set_len(self.drain.tail_end); + vec.reserve_intern(tmp_tail.len(), false, &mut self.weak_fixup); + vec.set_len(old_len); + + // now we can move the tail around + ptr::copy( + vec.as_ptr().add(self.drain.tail_start), + vec.as_mut_ptr().add(vec.len_exact() + tmp_tail.len()), + tail_len, + ); + + // all elements are moved from the tail, ensure that Drain drop does nothing. + // PLANNED: eventually we may not need use Drain here + self.drain.tail_start = self.drain.tail_end; + } + + let tmp_tail_len = tmp_tail.len(); + if !tmp_tail.is_empty() { + // When we stashed tail elements to tmp_tail, then fill the gap + tmp_tail.set_len(0); + ptr::copy_nonoverlapping( + tmp_tail.as_ptr(), + vec.as_mut_ptr().add(vec.len_exact()), + tmp_tail_len, + ); + } + + // finally fix the vec length + let new_len = vec.len_exact() + tmp_tail_len + tail_len; + vec.set_len(new_len); + } + + // IDEA: implement and benchmark batched copying. This leaves a gap in front of the tails which + // needs to be filled before resizing. + // Batch size: + // Moving one element per iteration to the tmp_tail is not efficient to make space for + // a element from the replace_with. Thus we determine a number of elements that we + // transfer in a batch to the tmp_tail. We compute the batch size to be roughly 4kb + // (Common page size on many systems) (or I::Item, whatever is larger) or the size of + // the tail when it is smaller. The later ensure that we do a single reserve with the + // minimum space needed when the tail is smaller than a batch would be . + // let batch_size = (4096 / std::mem::size_of::()) + // .max(1) + // .min(self.drain.tail_len); + } +} From c5e3081d0ab2080a0758db8a5e071e915c981f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 14 Feb 2025 02:50:46 +0100 Subject: [PATCH 38/52] move std tests to tests/std.rs --- tests/simple.rs | 27 ------- tests/std.rs | 187 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 27 deletions(-) create mode 100644 tests/std.rs diff --git a/tests/simple.rs b/tests/simple.rs index 8127117..4f39f1e 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -100,30 +100,3 @@ fn test_from_str() { "test".as_bytes() ); } - -#[cfg(feature = "std")] -#[test] -fn test_drain() { - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let drain = hv.drain(1..4); - assert_eq!(drain.as_slice(), [2, 3, 4]); - drop(drain); - assert_eq!(hv.as_slice(), [1, 5, 6]); -} - -#[cfg(feature = "std")] -#[test] -fn test_extend() { - let mut hv = HeaderVec::new(()); - hv.extend([1, 2, 3]); - assert_eq!(hv.as_slice(), [1, 2, 3]); -} - -#[cfg(feature = "std")] -#[test] -fn test_extend_ref() { - let mut hv = HeaderVec::<(), i32>::new(()); - hv.extend([&1, &2, &3]); - assert_eq!(hv.as_slice(), [1, 2, 3]); -} diff --git a/tests/std.rs b/tests/std.rs new file mode 100644 index 0000000..617411a --- /dev/null +++ b/tests/std.rs @@ -0,0 +1,187 @@ +//! Tests for the `std` feature of the `header_vec` crate. +#![cfg(feature = "std")] + +use header_vec::*; + +#[test] +fn test_extend() { + let mut hv = HeaderVec::new(()); + hv.extend([1, 2, 3]); + assert_eq!(hv.as_slice(), [1, 2, 3]); +} + +#[test] +fn test_extend_ref() { + let mut hv = HeaderVec::<(), i32>::new(()); + hv.extend([&1, &2, &3]); + assert_eq!(hv.as_slice(), [1, 2, 3]); +} + +#[test] +fn test_drain() { + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let drain = hv.drain(1..4); + assert_eq!(drain.as_slice(), [2, 3, 4]); + drop(drain); + assert_eq!(hv.as_slice(), [1, 5, 6]); +} + +#[test] +fn test_splice_nop() { + // drain at begin + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(0..0, []); + + assert_eq!(splice.drained_slice(), []); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6]); + + // drain inbetween + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(3..3, []); + + assert_eq!(splice.drained_slice(), []); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6]); + + // drain at end + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(6..6, []); + + assert_eq!(splice.drained_slice(), []); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6]); +} + +#[test] +fn test_splice_insert() { + // drain at begin + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(..0, [-2, -1, 0]); + + assert_eq!(splice.drained_slice(), []); + drop(splice); + assert_eq!(hv.as_slice(), [-2, -1, 0, 1, 2, 3, 4, 5, 6]); + + // drain inbetween + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(3..3, [31, 32, 33]); + + assert_eq!(splice.drained_slice(), []); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 31, 32, 33, 4, 5, 6]); + + // drain at end + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(6.., [7, 8, 9]); + + assert_eq!(splice.drained_slice(), []); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8, 9]); +} + +#[test] +fn test_splice_remove() { + // drain at begin + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(..2, []); + + assert_eq!(splice.drained_slice(), [1, 2]); + drop(splice); + assert_eq!(hv.as_slice(), [3, 4, 5, 6]); + + // drain inbetween + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(3..5, []); + + assert_eq!(splice.drained_slice(), [4, 5]); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 6]); + + // drain at end + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(4.., []); + + assert_eq!(splice.drained_slice(), [5, 6]); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 4]); + + // drain all + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(.., []); + + assert_eq!(splice.drained_slice(), [1, 2, 3, 4, 5, 6]); + drop(splice); + assert_eq!(hv.as_slice(), []); +} + +#[test] +fn test_splice_replace() { + // same length + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(3..5, [44, 55]); + + assert_eq!(splice.drained_slice(), [4, 5]); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 44, 55, 6]); + + // shorter + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(3..5, [44]); + + assert_eq!(splice.drained_slice(), [4, 5]); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 44, 6]); + + // longer + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(4..5, [44, 55]); + + assert_eq!(splice.drained_slice(), [5]); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 4, 44, 55, 6]); + + // longer than tail + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(3..5, [44, 55, 56, 57, 58, 59]); + + assert_eq!(splice.drained_slice(), [4, 5]); + drop(splice); + assert_eq!(hv.as_slice(), [1, 2, 3, 44, 55, 56, 57, 58, 59, 6]); + + // all + let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); + + let splice = hv.splice(.., [11, 22, 33]); + + assert_eq!(splice.drained_slice(), [1, 2, 3, 4, 5, 6]); + drop(splice); + assert_eq!(hv.as_slice(), [11, 22, 33]); +} + +// #[test] +// fn test_splice_zst() { +// // same length +// let mut hv = HeaderVec::from_header_slice((), [(),(),(),(),(),()]); +// +// // let splice = hv.splice(3..5, [(),()]); +// // +// // assert_eq!(splice.drained_slice(), [(),()]); +// // drop(splice); +// // assert_eq!(hv.as_slice(), [(),(),(),(),(),()]); +// } From 723dcdf00235cc6d373c3e72f14d51f63e9b4418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Sat, 15 Feb 2025 02:08:06 +0100 Subject: [PATCH 39/52] add benchmarks for drain and splice This shows that splice has to be optimized --- Cargo.toml | 2 + benches/compare_std_vec.rs | 92 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index aaa236c..a6fce19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,5 @@ default = ["std", "atomic_append"] std = [] atomic_append = [] +[dev-dependencies] +xmacro = "0.2.0" diff --git a/benches/compare_std_vec.rs b/benches/compare_std_vec.rs index 0d84314..7b67934 100644 --- a/benches/compare_std_vec.rs +++ b/benches/compare_std_vec.rs @@ -126,3 +126,95 @@ fn test_regular_vec_create(b: &mut Bencher) { // acc // }); // } + +#[cfg(feature = "std")] +mod stdbench { + use super::*; + use std::ops::{Range, RangeBounds}; + use xmacro::xmacro; + + xmacro! { + $[ + benchfunc: type: + hv_drain_bench (HeaderVec::<(), _>) + vec_drain_bench (Vec) + ] + + fn $benchfunc(b: &mut Bencher, init: &[T], range: R) + where + T: Clone + Default, + R: RangeBounds + Clone, + { + b.iter(|| { + let mut v = $type::from(init); + v.drain(range.clone()); + v + }); + } + } + + xmacro! { + $[ + bench: init: range: + middle [123; 1000] (100..500) + begin [123; 1000] (..500) + end [123; 1000] (100..) + ] + + #[bench] + fn $+test_hv_drain_$bench(b: &mut Bencher) { + hv_drain_bench(b, &$init, $range); + } + + #[bench] + fn $+test_vec_drain_$bench(b: &mut Bencher) { + vec_drain_bench(b, &$init, $range); + } + } + + xmacro! { + $[ + benchfunc: type: + hv_splice_bench (HeaderVec::<(), _>) + vec_splice_bench (Vec) + ] + + fn $benchfunc(b: &mut Bencher, init: &[T], range: R, replace_with: I) + where + T: Clone + Default, + R: RangeBounds + Clone, + I: IntoIterator + Clone, + { + b.iter(|| { + let mut v = $type::from(init); + v.splice(range.clone(), replace_with.clone()); + v + }); + } + } + + xmacro! { + $[ + bench: init: range: replace_with: + nop [123; 1000] (0..0) [] + insert [123; 1000] (100..100) [123;500] + remove [123; 1000] (100..600) [] + middle_shorter [123; 1000] (400..500) [234;50] + middle_longer [123; 1000] (400..500) [345;200] + middle_same [123; 1000] (400..500) [456;100] + end_shorter [123; 1000] (900..) [234;50] + end_longer [123; 1000] (900..) [345;200] + end_same [123; 1000] (900..) [456;100] + ] + + #[bench] + fn $+test_hv_splice_$bench(b: &mut Bencher) { + hv_splice_bench(b, &$init, $range, $replace_with) + } + + #[bench] + fn $+test_vec_splice_$bench(b: &mut Bencher) { + vec_splice_bench(b, &$init, $range, $replace_with) + } + } +} From 25788c1a8257e127faf85400a6b147ec26015157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 18 Feb 2025 04:45:10 +0100 Subject: [PATCH 40/52] FIX: bug in offset() calculation Off-By-One error when rouding to the next offset. I totally missed that a bug slipped into the offset calculation because I only tested with u8 and i32 data. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ab426b9..6bdb519 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -381,7 +381,7 @@ impl HeaderVec { const fn offset() -> usize { // The first location, in units of size_of::(), that is after the header // It's the end of the header, rounded up to the nearest size_of::() - mem::size_of::>() / mem::size_of::() + (mem::size_of::>()-1) / mem::size_of::() + 1 } /// Compute the number of elements (in units of T) to allocate for a given capacity. From 156cb393f2638b745207114cf48c765ec98b31c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 21:59:31 +0100 Subject: [PATCH 41/52] simple/cosmetic changes only --- src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 02584bf..0854d68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -493,7 +493,8 @@ impl HeaderVec { pub unsafe fn set_len(&mut self, new_len: usize) { debug_assert!( new_len <= self.capacity(), - "new_len is greater than capacity" + "new_len [{new_len}] is greater than capacity [{}]", + self.capacity() ); self.header_mut().len = new_len.into(); } @@ -592,7 +593,7 @@ impl HeaderVec { const fn offset() -> usize { // The first location, in units of size_of::(), that is after the header // It's the end of the header, rounded up to the nearest size_of::() - (mem::size_of::>()-1) / mem::size_of::() + 1 + (mem::size_of::>() - 1) / mem::size_of::() + 1 } /// Compute the number of elements (in units of T) to allocate for a given capacity. From 2e845142feb9fea34c7ab342c265a706503a601f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 22:01:31 +0100 Subject: [PATCH 42/52] improve headervec drop() by dropping the elements as slice --- src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0854d68..df01520 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -950,10 +950,8 @@ impl HeaderVec { impl Drop for HeaderVec { fn drop(&mut self) { unsafe { + ptr::drop_in_place(self.as_mut_slice()); ptr::drop_in_place(&mut self.header_mut().head); - for ix in 0..self.len_exact() { - ptr::drop_in_place(self.as_mut_ptr().add(ix)); - } alloc::alloc::dealloc(self.ptr() as *mut u8, Self::layout(self.capacity())); } } From d168f01499e2630c57b79044a10175e5720cc45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 22:05:21 +0100 Subject: [PATCH 43/52] rewrite the splice drop algo to the same std Vec uses My initial idea stashing the tail aside didnt worked out as well as i hoped for. Also adds a good deal of tests generated by xmacros. Some may be removed later, but I tried do cover some corner cases (begin/middle/end, larger/same/shorter replacements) Since we using a a lot unsafe code eventually all code paths should be checked (see cargo mutants). The existing tests passing under miri. --- src/drain.rs | 16 ++--- src/lib.rs | 2 +- src/splice.rs | 161 +++++++++++++++++++----------------------- tests/std.rs | 192 +++++++++----------------------------------------- 4 files changed, 114 insertions(+), 257 deletions(-) diff --git a/src/drain.rs b/src/drain.rs index fa6aa5f..7b8547d 100644 --- a/src/drain.rs +++ b/src/drain.rs @@ -32,7 +32,7 @@ pub struct Drain<'a, H, T> { /// Index of tail to preserve pub(super) tail_start: usize, /// End index of tail to preserve - pub(super) tail_end: usize, + pub(super) tail_len: usize, /// Current remaining range to remove pub(super) iter: slice::Iter<'a, T>, pub(super) vec: NonNull>, @@ -113,16 +113,12 @@ impl Drain<'_, H, T> { if tail != (start + unyielded_len) { let src = source_vec.as_ptr().add(tail); let dst = start_ptr.add(unyielded_len); - ptr::copy(src, dst, this.tail_len()); + ptr::copy(src, dst, this.tail_len); } - source_vec.set_len(start + unyielded_len + this.tail_len()); + source_vec.set_len(start + unyielded_len + this.tail_len); } } - - pub(crate) fn tail_len(&self) -> usize { - self.tail_end - self.tail_start - } } impl AsRef<[T]> for Drain<'_, H, T> { @@ -165,7 +161,7 @@ impl Drop for Drain<'_, H, T> { impl Drop for DropGuard<'_, '_, H, T> { fn drop(&mut self) { - if self.0.tail_len() > 0 { + if self.0.tail_len > 0 { unsafe { let source_vec = self.0.vec.as_mut(); // memmove back untouched tail, update to new length @@ -174,9 +170,9 @@ impl Drop for Drain<'_, H, T> { if tail != start { let src = source_vec.as_ptr().add(tail); let dst = source_vec.as_mut_ptr().add(start); - ptr::copy(src, dst, self.0.tail_len()); + ptr::copy(src, dst, self.0.tail_len); } - source_vec.set_len(start + self.0.tail_len()); + source_vec.set_len(start + self.0.tail_len); } } } diff --git a/src/lib.rs b/src/lib.rs index df01520..7a51fc3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -861,7 +861,7 @@ impl HeaderVec { let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); Drain { tail_start: end, - tail_end: len, + tail_len: len - end, iter: range_slice.iter(), vec: NonNull::from(self), } diff --git a/src/splice.rs b/src/splice.rs index 802c608..a1f1054 100644 --- a/src/splice.rs +++ b/src/splice.rs @@ -1,6 +1,6 @@ #![cfg(feature = "std")] -use core::{any::type_name, fmt, ptr}; +use core::{any::type_name, fmt, ptr, slice}; use crate::{Drain, WeakFixupFn}; @@ -77,104 +77,87 @@ impl Drop for Splice<'_, H, I> { // Which means we can replace the slice::Iter with pointers that won't point to deallocated // memory, so that Drain::drop is still allowed to call iter.len(), otherwise it would break // the ptr.sub_ptr contract. - self.drain.iter = [].iter(); - - // We will use the replace_with iterator to append elements in place on self.drain.vec. - // When this hits the tail then elements are moved from the tail to tmp_tail. - // When the tail is or becomes empty by that, then the remaining elements can be extended to the vec. - // - // Finally: - // Then have continuous elements in the vec: |head|replace_with|(old_tail|)spare_capacity|. - // The old tail needs to be moved to its final destination. - // Perhaps making space for the elements in the tmp_tail. - let mut tmp_tail = Vec::new(); unsafe { let vec = self.drain.vec.as_mut(); - loop { - if self.drain.tail_len() == 0 { - // If the tail is empty, we can just extend the vector with the remaining elements. - // but we may have stashed some tmp_tail away and should reserve for that. - // PLANNED: should become 'extend_reserve()' - vec.reserve_intern( - self.replace_with.size_hint().0 + tmp_tail.len(), - false, - &mut self.weak_fixup, - ); - vec.extend(self.replace_with.by_ref()); - // in case the size_hint was not exact (or returned 0) we need to reserve for the tmp_tail - // in most cases this will not allocate. later we expect that we have this space reserved. - vec.reserve_intern(tmp_tail.len(), false, &mut self.weak_fixup); - break; - } else if let Some(next) = self.replace_with.next() { - if vec.len_exact() >= self.drain.tail_start && self.drain.tail_len() > 0 { - // move one element from the tail to the tmp_tail - // We reserve for as much elements are hinted by replace_with or the remaining tail, - // whatever is smaller. - tmp_tail - .reserve(self.replace_with.size_hint().0.min(self.drain.tail_len())); - tmp_tail.push(ptr::read(vec.as_ptr().add(self.drain.tail_start))); - self.drain.tail_start += 1; - } - - // since we overwrite the old tail here this will never reallocate. - // PLANNED: vec.push_within_capacity().unwrap_unchecked() - vec.push(next); - } else { - // replace_with is depleted - break; - } + + if self.drain.tail_len == 0 { + vec.extend(self.replace_with.by_ref()); + return; } - let tail_len = self.drain.tail_len(); - if tail_len > 0 { - // In case we need to shift the tail farther back we need to reserve space for that. - // Reserve needs to preserve the tail we have, thus we temporarily set the length to the - // tail_end and then restore it after the reserve. - let old_len = vec.len_exact(); - vec.set_len(self.drain.tail_end); - vec.reserve_intern(tmp_tail.len(), false, &mut self.weak_fixup); - vec.set_len(old_len); - - // now we can move the tail around - ptr::copy( - vec.as_ptr().add(self.drain.tail_start), - vec.as_mut_ptr().add(vec.len_exact() + tmp_tail.len()), - tail_len, - ); - - // all elements are moved from the tail, ensure that Drain drop does nothing. - // PLANNED: eventually we may not need use Drain here - self.drain.tail_start = self.drain.tail_end; + // First fill the range left by drain(). + if !self.drain.fill(&mut self.replace_with) { + return; } - let tmp_tail_len = tmp_tail.len(); - if !tmp_tail.is_empty() { - // When we stashed tail elements to tmp_tail, then fill the gap - tmp_tail.set_len(0); - ptr::copy_nonoverlapping( - tmp_tail.as_ptr(), - vec.as_mut_ptr().add(vec.len_exact()), - tmp_tail_len, - ); + // There may be more elements. Use the lower bound as an estimate. + // FIXME: Is the upper bound a better guess? Or something else? + let (lower_bound, _upper_bound) = self.replace_with.size_hint(); + if lower_bound > 0 { + self.drain.move_tail(lower_bound, &mut self.weak_fixup); + if !self.drain.fill(&mut self.replace_with) { + return; + } } - // finally fix the vec length - let new_len = vec.len_exact() + tmp_tail_len + tail_len; - vec.set_len(new_len); + // Collect any remaining elements. + // This is a zero-length vector which does not allocate if `lower_bound` was exact. + let mut collected = self + .replace_with + .by_ref() + .collect::>() + .into_iter(); + // Now we have an exact count. + if collected.len() > 0 { + self.drain.move_tail(collected.len(), &mut self.weak_fixup); + let filled = self.drain.fill(&mut collected); + debug_assert!(filled); + debug_assert_eq!(collected.len(), 0); + } + } + } +} + +/// Private helper methods for `Splice::drop` +impl Drain<'_, H, T> { + /// The range from `self.vec.len` to `self.tail_start` contains elements + /// that have been moved out. + /// Fill that range as much as possible with new elements from the `replace_with` iterator. + /// Returns `true` if we filled the entire range. (`replace_with.next()` didn’t return `None`.) + unsafe fn fill>(&mut self, replace_with: &mut I) -> bool { + let vec = unsafe { self.vec.as_mut() }; + let range_start = vec.len_exact(); + let range_end = self.tail_start; + let range_slice = unsafe { + slice::from_raw_parts_mut(vec.as_mut_ptr().add(range_start), range_end - range_start) + }; + + for place in range_slice { + if let Some(new_item) = replace_with.next() { + unsafe { ptr::write(place, new_item) }; + let len = vec.len_exact(); + vec.set_len(len + 1); + } else { + return false; + } } + true + } - // IDEA: implement and benchmark batched copying. This leaves a gap in front of the tails which - // needs to be filled before resizing. - // Batch size: - // Moving one element per iteration to the tmp_tail is not efficient to make space for - // a element from the replace_with. Thus we determine a number of elements that we - // transfer in a batch to the tmp_tail. We compute the batch size to be roughly 4kb - // (Common page size on many systems) (or I::Item, whatever is larger) or the size of - // the tail when it is smaller. The later ensure that we do a single reserve with the - // minimum space needed when the tail is smaller than a batch would be . - // let batch_size = (4096 / std::mem::size_of::()) - // .max(1) - // .min(self.drain.tail_len); + /// Makes room for inserting more elements before the tail. + #[track_caller] + unsafe fn move_tail(&mut self, additional: usize, weak_fixup: &mut Option>) { + let vec = unsafe { self.vec.as_mut() }; + let len = self.tail_start + self.tail_len; + vec.reserve_intern(len + additional, false, weak_fixup); + + let new_tail_start = self.tail_start + additional; + unsafe { + let src = vec.as_ptr().add(self.tail_start); + let dst = vec.as_mut_ptr().add(new_tail_start); + ptr::copy(src, dst, self.tail_len); + } + self.tail_start = new_tail_start; } } diff --git a/tests/std.rs b/tests/std.rs index 617411a..721902d 100644 --- a/tests/std.rs +++ b/tests/std.rs @@ -2,6 +2,7 @@ #![cfg(feature = "std")] use header_vec::*; +use xmacro::xmacro; #[test] fn test_extend() { @@ -27,161 +28,38 @@ fn test_drain() { assert_eq!(hv.as_slice(), [1, 5, 6]); } -#[test] -fn test_splice_nop() { - // drain at begin - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(0..0, []); - - assert_eq!(splice.drained_slice(), []); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6]); - - // drain inbetween - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(3..3, []); - - assert_eq!(splice.drained_slice(), []); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6]); - - // drain at end - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(6..6, []); - - assert_eq!(splice.drained_slice(), []); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6]); -} - -#[test] -fn test_splice_insert() { - // drain at begin - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(..0, [-2, -1, 0]); - - assert_eq!(splice.drained_slice(), []); - drop(splice); - assert_eq!(hv.as_slice(), [-2, -1, 0, 1, 2, 3, 4, 5, 6]); - - // drain inbetween - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(3..3, [31, 32, 33]); - - assert_eq!(splice.drained_slice(), []); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 31, 32, 33, 4, 5, 6]); - - // drain at end - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(6.., [7, 8, 9]); - - assert_eq!(splice.drained_slice(), []); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8, 9]); +xmacro! { + $[ + // tests with simple i32 lists + name: init: range: replace: drained: result: + nop_begin [1, 2, 3, 4, 5, 6] (0..0) [] [] [1, 2, 3, 4, 5, 6] + nop_middle [1, 2, 3, 4, 5, 6] (3..3) [] [] [1, 2, 3, 4, 5, 6] + nop_end [1, 2, 3, 4, 5, 6] (6..6) [] [] [1, 2, 3, 4, 5, 6] + insert_begin [1, 2, 3, 4, 5, 6] (0..0) [-1, 0] [] [-1, 0, 1, 2, 3, 4, 5, 6] + insert_middle [1, 2, 3, 4, 5, 6] (3..3) [33, 34] [] [1, 2, 3, 33, 34, 4, 5, 6] + insert_end [1, 2, 3, 4, 5, 6] (6..6) [7, 8] [] [1, 2, 3, 4, 5, 6, 7, 8] + remove_begin [1, 2, 3, 4, 5, 6] (0..2) [] [1, 2] [3, 4, 5, 6] + remove_middle [1, 2, 3, 4, 5, 6] (3..5) [] [4, 5] [1, 2, 3, 6] + remove_end [1, 2, 3, 4, 5, 6] (4..) [] [5, 6] [1, 2, 3, 4] + replace_begin_shorter [1, 2, 3, 4, 5, 6] (0..2) [11] [1, 2] [11,3, 4, 5, 6] + replace_middle_shorter [1, 2, 3, 4, 5, 6] (3..5) [44] [4, 5] [1, 2, 3, 44, 6] + replace_end_shorter [1, 2, 3, 4, 5, 6] (4..) [55] [5, 6] [1, 2, 3, 4, 55] + replace_begin_same [1, 2, 3, 4, 5, 6] (0..2) [11, 22] [1, 2] [11, 22, 3, 4, 5, 6] + replace_middle_same [1, 2, 3, 4, 5, 6] (3..5) [44, 55] [4, 5] [1, 2, 3, 44,55, 6] + replace_end_same [1, 2, 3, 4, 5, 6] (4..) [55, 66] [5, 6] [1, 2, 3, 4, 55, 66] + replace_begin_longer [1, 2, 3, 4, 5, 6] (0..2) [11, 22, 33] [1, 2] [11, 22, 33, 3, 4, 5, 6] + replace_middle_longer [1, 2, 3, 4, 5, 6] (3..5) [44, 55, 66] [4, 5] [1, 2, 3, 44, 55, 66, 6] + replace_end_longer [1, 2, 3, 4, 5, 6] (4..) [66, 77, 88] [5, 6] [1, 2, 3, 4, 66, 77, 88] + big_nop [[1; 64]; 64] (0..0) [[0; 64]; 0] [[0; 64]; 0] [[1; 64]; 64] + ] + + #[test] + fn $+test_splice_$name() { + let mut hv = HeaderVec::from_header_slice((), $init); + let splice = hv.splice($range, $replace); + + assert_eq!(splice.drained_slice(), $drained); + drop(splice); + assert_eq!(hv.as_slice(), $result); + } } - -#[test] -fn test_splice_remove() { - // drain at begin - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(..2, []); - - assert_eq!(splice.drained_slice(), [1, 2]); - drop(splice); - assert_eq!(hv.as_slice(), [3, 4, 5, 6]); - - // drain inbetween - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(3..5, []); - - assert_eq!(splice.drained_slice(), [4, 5]); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 6]); - - // drain at end - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(4.., []); - - assert_eq!(splice.drained_slice(), [5, 6]); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 4]); - - // drain all - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(.., []); - - assert_eq!(splice.drained_slice(), [1, 2, 3, 4, 5, 6]); - drop(splice); - assert_eq!(hv.as_slice(), []); -} - -#[test] -fn test_splice_replace() { - // same length - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(3..5, [44, 55]); - - assert_eq!(splice.drained_slice(), [4, 5]); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 44, 55, 6]); - - // shorter - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(3..5, [44]); - - assert_eq!(splice.drained_slice(), [4, 5]); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 44, 6]); - - // longer - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(4..5, [44, 55]); - - assert_eq!(splice.drained_slice(), [5]); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 4, 44, 55, 6]); - - // longer than tail - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(3..5, [44, 55, 56, 57, 58, 59]); - - assert_eq!(splice.drained_slice(), [4, 5]); - drop(splice); - assert_eq!(hv.as_slice(), [1, 2, 3, 44, 55, 56, 57, 58, 59, 6]); - - // all - let mut hv = HeaderVec::from_header_slice((), [1, 2, 3, 4, 5, 6]); - - let splice = hv.splice(.., [11, 22, 33]); - - assert_eq!(splice.drained_slice(), [1, 2, 3, 4, 5, 6]); - drop(splice); - assert_eq!(hv.as_slice(), [11, 22, 33]); -} - -// #[test] -// fn test_splice_zst() { -// // same length -// let mut hv = HeaderVec::from_header_slice((), [(),(),(),(),(),()]); -// -// // let splice = hv.splice(3..5, [(),()]); -// // -// // assert_eq!(splice.drained_slice(), [(),()]); -// // drop(splice); -// // assert_eq!(hv.as_slice(), [(),(),(),(),(),()]); -// } From 5dd7b59e08707953fe532a8dfdc1e07ddff396c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 22:11:19 +0100 Subject: [PATCH 44/52] improve benchmarks, generated by xmacros This adds a lot benchmarks that compares HeaderVec std Vec. Results show that we are in the same ballpark than std Vec. Also enables debug symbols for bench builds as this is quasi-required for some profilers and flamegraphs. --- Cargo.toml | 4 ++ benches/compare_std_vec.rs | 81 ++++++++++++++++++++++++++++---------- 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a6fce19..4a05e05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,7 @@ atomic_append = [] [dev-dependencies] xmacro = "0.2.0" + +# include debug info for flamgraph and other profiling tools +[profile.bench] +debug = true diff --git a/benches/compare_std_vec.rs b/benches/compare_std_vec.rs index 7b67934..b4b11a5 100644 --- a/benches/compare_std_vec.rs +++ b/benches/compare_std_vec.rs @@ -2,6 +2,7 @@ extern crate std; extern crate test; +use xmacro::xmacro; use header_vec::*; use test::Bencher; @@ -127,11 +128,48 @@ fn test_regular_vec_create(b: &mut Bencher) { // }); // } +xmacro! { + $[ + benchfunc: type: + hv_create_bench (HeaderVec::<(), _>) + vec_create_bench (Vec) + ] + + fn $benchfunc(b: &mut Bencher, init: &[T]) + where + T: Clone + Default, + { + b.iter(|| { + let v = $type::from(init); + v + }); + } +} + +xmacro! { + // benching construction times. + $[ + bench: init: + small [123; 1] + middle [123; 1000] + large [[123;32]; 100000] + ] + + #[bench] + fn $+bench_hv_create_$bench(b: &mut Bencher) { + hv_create_bench(b, &$init); + } + + #[bench] + fn $+bench_vec_create_$bench(b: &mut Bencher) { + vec_create_bench(b, &$init); + } +} + #[cfg(feature = "std")] mod stdbench { use super::*; - use std::ops::{Range, RangeBounds}; - use xmacro::xmacro; + use std::ops::RangeBounds; xmacro! { $[ @@ -156,18 +194,18 @@ mod stdbench { xmacro! { $[ bench: init: range: - middle [123; 1000] (100..500) - begin [123; 1000] (..500) - end [123; 1000] (100..) + begin [123; 10000] (..5000) + middle [123; 10000] (1000..5000) + end [123; 10000] (1000..) ] #[bench] - fn $+test_hv_drain_$bench(b: &mut Bencher) { + fn $+bench_hv_drain_$bench(b: &mut Bencher) { hv_drain_bench(b, &$init, $range); } #[bench] - fn $+test_vec_drain_$bench(b: &mut Bencher) { + fn $+bench_vec_drain_$bench(b: &mut Bencher) { vec_drain_bench(b, &$init, $range); } } @@ -181,7 +219,7 @@ mod stdbench { fn $benchfunc(b: &mut Bencher, init: &[T], range: R, replace_with: I) where - T: Clone + Default, + T: Clone, R: RangeBounds + Clone, I: IntoIterator + Clone, { @@ -195,25 +233,28 @@ mod stdbench { xmacro! { $[ - bench: init: range: replace_with: - nop [123; 1000] (0..0) [] - insert [123; 1000] (100..100) [123;500] - remove [123; 1000] (100..600) [] - middle_shorter [123; 1000] (400..500) [234;50] - middle_longer [123; 1000] (400..500) [345;200] - middle_same [123; 1000] (400..500) [456;100] - end_shorter [123; 1000] (900..) [234;50] - end_longer [123; 1000] (900..) [345;200] - end_same [123; 1000] (900..) [456;100] + bench: init: range: replace_with: + nop [123; 10000] (0..0) [] + insert [123; 10000] (1000..1000) [123; 5000] + insert_big [[123;64]; 10000] (1000..1000) [[123; 64]; 5000] + remove [123; 10000] (1000..6000) [] + middle_shorter [123; 10000] (4000..5000) [234; 500] + middle_longer [123; 10000] (4000..5000) [345; 2000] + middle_same [123; 10000] (4000..5000) [456; 1000] + end_shorter [123; 10000] (9000..) [234; 500] + end_longer [123; 10000] (9000..) [345; 2000] + end_same [123; 10000] (9000..) [456; 1000] + append_big [[123;64]; 10000] (10000..) [[456; 64]; 5000] + append_front_big [[123;64]; 100000] (0..0) [[456; 64]; 1] ] #[bench] - fn $+test_hv_splice_$bench(b: &mut Bencher) { + fn $+bench_hv_splice_$bench(b: &mut Bencher) { hv_splice_bench(b, &$init, $range, $replace_with) } #[bench] - fn $+test_vec_splice_$bench(b: &mut Bencher) { + fn $+bench_vec_splice_$bench(b: &mut Bencher) { vec_splice_bench(b, &$init, $range, $replace_with) } } From 9d839ff130f3c7a95b4ef7d96597a4cfdd980439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 22:58:29 +0100 Subject: [PATCH 45/52] Document the atomic_append API / add Safety Section --- src/lib.rs | 72 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6bdb519..85b0271 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -462,8 +462,52 @@ impl HeaderVec { #[cfg(feature = "atomic_append")] /// The atomic append API is only enabled when the `atomic_append` feature flag is set (which -/// is the default). +/// is the default). The [`push_atomic()`] or [`extend_from_slice_atomic()`] methods then +/// become available and some internals using atomic operations. +/// +/// This API implements interior-mutable appending to a shared `HeaderVec`. To other threads +/// the appended elements are either not seen or all seen at once. Without additional +/// synchronization these appends are racy but memory safe. The intention behind this API is to +/// provide facilities for building other container abstractions the benefit from the shared +/// non blocking nature while being unaffected from the racy semantics or provide synchronization +/// on their own (Eg: reference counted data, interners, streaming parsers, etc). Since the +/// `HeaderVec` is a shared object and we have only a `&self`, it can not be reallocated and moved, +/// therefore appending can only be done within the reserved capacity. +/// +/// # Safety +/// +/// Only one single thread must try to [`push_atomic()`] or [`extend_from_slice_atomic()`] the +/// `HeaderVec` at at time using the atomic append API's. The actual implementations of this +/// restriction is left to the caller. This can be done by mutexes or guard objects. Or +/// simply by staying single threaded or ensuring somehow else that there is only a single +/// thread using the atomic_appending API. impl HeaderVec { + /// Atomically adds an item to the end of the list without reallocation. + /// + /// # Errors + /// + /// If the vector is full, the item is returned. + /// + /// # Safety + /// + /// There must be only one thread calling this method at any time. Synchronization has to + /// be provided by the user. + pub unsafe fn push_atomic(&self, item: T) -> Result<(), T> { + // relaxed is good enough here because this should be the only thread calling this method. + let len = self.len_atomic_relaxed(); + if len < self.capacity() { + unsafe { + core::ptr::write(self.end_ptr_atomic_mut(), item); + }; + let len_again = self.len_atomic_add_release(1); + // in debug builds we check for races, the chance to catch these are still pretty minimal + debug_assert_eq!(len_again, len, "len was updated by another thread"); + Ok(()) + } else { + Err(item) + } + } + /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is /// properly synchronized after it got atomically updated. #[inline(always)] @@ -505,32 +549,6 @@ impl HeaderVec { fn end_ptr_atomic_mut(&self) -> *mut T { unsafe { self.start_ptr().add(self.len_atomic_acquire()) as *mut T } } - - /// Atomically adds an item to the end of the list without reallocation. - /// - /// # Errors - /// - /// If the vector is full, the item is returned. - /// - /// # Safety - /// - /// There must be only one thread calling this method at any time. Synchronization has to - /// be provided by the user. - pub unsafe fn push_atomic(&self, item: T) -> Result<(), T> { - // relaxed is good enough here because this should be the only thread calling this method. - let len = self.len_atomic_relaxed(); - if len < self.capacity() { - unsafe { - core::ptr::write(self.end_ptr_atomic_mut(), item); - }; - let len_again = self.len_atomic_add_release(1); - // in debug builds we check for races, the chance to catch these are still pretty minimal - debug_assert_eq!(len_again, len, "len was updated by another thread"); - Ok(()) - } else { - Err(item) - } - } } #[cfg(feature = "atomic_append")] From 98fae2230f39c633fc662d6aec282216a37b2228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 22:59:03 +0100 Subject: [PATCH 46/52] remove is_empty_atomic_acquire as_slice_atomic_acquire These public but undocumented and unused, i added them in hindsight but probably they are not required. Otherwise they could be re-added later. --- src/lib.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 85b0271..4ab0967 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -533,16 +533,6 @@ impl HeaderVec { self.header().len.fetch_add(n, Ordering::Release) } - #[inline(always)] - pub fn is_empty_atomic_acquire(&self) -> bool { - self.len_atomic_acquire() == 0 - } - - #[inline(always)] - pub fn as_slice_atomic_acquire(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } - } - /// Gets the pointer to the end of the slice. This returns a mutable pointer to /// uninitialized memory behind the last element. #[inline(always)] From eb3efcadbf3569f2b9ef3be059b9411ae4596c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Thu, 20 Feb 2025 23:04:23 +0100 Subject: [PATCH 47/52] formatting --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 4ab0967..8b7544c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -381,7 +381,7 @@ impl HeaderVec { const fn offset() -> usize { // The first location, in units of size_of::(), that is after the header // It's the end of the header, rounded up to the nearest size_of::() - (mem::size_of::>()-1) / mem::size_of::() + 1 + (mem::size_of::>() - 1) / mem::size_of::() + 1 } /// Compute the number of elements (in units of T) to allocate for a given capacity. From d7a0d1a0110640c88623f95d454ed4c58686f6af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 21 Feb 2025 02:08:42 +0100 Subject: [PATCH 48/52] try to reserve space in the Extend impls --- src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fc2598e..e916e79 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1082,17 +1082,18 @@ impl Extend for HeaderVec { #[inline] #[track_caller] fn extend>(&mut self, iter: I) { - iter.into_iter().for_each(|item| self.push(item)); + let iter = iter.into_iter(); + self.reserve(iter.size_hint().0); + iter.for_each(|item| self.push(item)); } } /// Extend implementation that copies elements out of references before pushing them onto the Vec. -// Note: from std Vec: not implemented here yet -// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to -// append the entire slice at once. impl<'a, H, T: Copy + 'a> Extend<&'a T> for HeaderVec { #[track_caller] fn extend>(&mut self, iter: I) { - iter.into_iter().for_each(|item| self.push(*item)); + let iter = iter.into_iter(); + self.reserve(iter.size_hint().0); + iter.for_each(|item| self.push(*item)); } } From aa3ba671c4e0c82f503bcb4ac399c1031b5c7e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 21 Feb 2025 02:43:22 +0100 Subject: [PATCH 49/52] ADD: from_header_elements() constructor consuming its input from_header_slice() clones the elements from a slice, in many cases this wont be optimal. from_header_elements() fixes this by consuming elements for a HeaderVec from a IntoIterator --- src/lib.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index e916e79..dfdea8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,6 +115,25 @@ impl HeaderVec { this } + /// Creates a new `HeaderVec` with the given header from owned elements. + /// This functions consumes elements from a `IntoIterator` and creates + /// a `HeaderVec` from these. See [`from_header_slice()`] which creates a `HeaderVec` + /// by cloning elements from a slice. + /// + /// # Example + /// + /// ``` + /// # use header_vec::HeaderVec; + /// let hv = HeaderVec::from_header_elements(42, [1, 2, 3]); + /// assert_eq!(hv.as_slice(), [1, 2, 3]); + /// ``` + pub fn from_header_elements(header: H, elements: impl IntoIterator) -> Self { + let iter = elements.into_iter(); + let mut hv = HeaderVec::with_capacity(iter.size_hint().0, header); + hv.extend(iter); + hv + } + /// Get the length of the vector from a mutable reference. When one has a `&mut /// HeaderVec`, this is the method is always exact and can be slightly faster than the non /// mutable `len()`. @@ -653,6 +672,8 @@ impl HeaderVec { impl HeaderVec { /// Creates a new `HeaderVec` with the given header from some data. + /// The data cloned from a `AsRef<[T]>`, see [`from_header_elements()`] for + /// constructing a `HeaderVec` from owned elements. pub fn from_header_slice(header: H, slice: impl AsRef<[T]>) -> Self { let slice = slice.as_ref(); let mut hv = Self::with_capacity(slice.len(), header); From 779c992ebb75f4a1e4efb2c2e7512458d7e118a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 21 Feb 2025 04:07:54 +0100 Subject: [PATCH 50/52] add tests/mutants.rs --- tests/mutants.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/mutants.rs diff --git a/tests/mutants.rs b/tests/mutants.rs new file mode 100644 index 0000000..2c94a42 --- /dev/null +++ b/tests/mutants.rs @@ -0,0 +1,19 @@ +use header_vec::HeaderVec; +#[test] +fn test_truncate_remaining_len() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + let len = vec.len(); + vec.truncate(2); + assert_eq!(vec.len(), 2); + assert_eq!(vec.as_slice(), &[1, 2]); + assert!(vec.capacity() >= len); +} + +#[test] +fn test_drain_size_hint() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + let drain = vec.drain(1..4); + let (lower, upper) = drain.size_hint(); + assert_eq!(lower, 3); + assert_eq!(upper, Some(3)); +} From 442036672e024a3e81afd44baf28ce352083c627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 21 Feb 2025 05:48:52 +0100 Subject: [PATCH 51/52] clippy cosmetics --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dfdea8a..851db9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -818,10 +818,10 @@ impl HeaderVec { } } // correct the len - let len_again = self.len_atomic_add_release(slice.len()); + let _len_again = self.len_atomic_add_release(slice.len()); // in debug builds we check for races, the chance to catch these are still pretty minimal #[cfg(debug_assertions)] - debug_assert_eq!(len_again, len, "len was updated by another thread"); + debug_assert_eq!(_len_again, len, "len was updated by another thread"); Ok(()) } else { Err(slice) From e6bcf0654f75c6077d5a0dbb7ab27dff0dde34ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Fri, 21 Feb 2025 05:53:29 +0100 Subject: [PATCH 52/52] Add mutant testing This adds tests/mutants.rs which was created by githup copilot assistance. Note: This file is partially is generated with github copilot assistance. The *only* objective here is to generate coverage over all code paths to make 'cargo mutants' pass. Then the main test suite should pass 'cargo +nightly miri test'. Unless otherwise noted the tests here are not extensively reviewed for semantic correctness. Eventually human reviewed tests here should be moved to other unit tests. Hard to test side effects and functions that are trivially correct and are marked with #[mutants::skip]. --- Cargo.toml | 3 + src/drain.rs | 1 + src/lib.rs | 13 ++ src/splice.rs | 2 + tests/mutants.rs | 374 ++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 385 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4a05e05..017fdee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,6 @@ xmacro = "0.2.0" # include debug info for flamgraph and other profiling tools [profile.bench] debug = true + +[dependencies] +mutants = "0.0.3" diff --git a/src/drain.rs b/src/drain.rs index 7b8547d..8a720e6 100644 --- a/src/drain.rs +++ b/src/drain.rs @@ -39,6 +39,7 @@ pub struct Drain<'a, H, T> { } impl fmt::Debug for Drain<'_, H, T> { + #[mutants::skip] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct(&format!( "Drain<{}, {}>", diff --git a/src/lib.rs b/src/lib.rs index 851db9c..7293d9a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -138,11 +138,13 @@ impl HeaderVec { /// HeaderVec`, this is the method is always exact and can be slightly faster than the non /// mutable `len()`. #[cfg(feature = "atomic_append")] + #[mutants::skip] #[inline(always)] pub fn len_exact(&mut self) -> usize { *self.header_mut().len.get_mut() } #[cfg(not(feature = "atomic_append"))] + #[mutants::skip] #[inline(always)] pub fn len_exact(&mut self) -> usize { self.header_mut().len @@ -152,11 +154,13 @@ impl HeaderVec { /// produce racy results in case another thread atomically appended to /// `&self`. Nevertheless it is always safe to use. #[cfg(feature = "atomic_append")] + #[mutants::skip] #[inline(always)] pub fn len(&self) -> usize { self.len_atomic_relaxed() } #[cfg(not(feature = "atomic_append"))] + #[mutants::skip] #[inline(always)] pub fn len(&self) -> usize { self.header().len @@ -168,11 +172,13 @@ impl HeaderVec { /// atomically appends data to this `HeaderVec` while we still work with the result of /// this method. #[cfg(not(feature = "atomic_append"))] + #[mutants::skip] #[inline(always)] pub fn len_strict(&self) -> usize { self.header().len } #[cfg(feature = "atomic_append")] + #[mutants::skip] #[inline(always)] pub fn len_strict(&self) -> usize { self.len_atomic_acquire() @@ -283,6 +289,7 @@ impl HeaderVec { } /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. + #[mutants::skip] #[inline] pub fn reserve_exact(&mut self, additional: usize) { self.reserve_intern(additional, true, &mut None); @@ -291,6 +298,7 @@ impl HeaderVec { /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for /// updating the weak references as additional parameter. + #[mutants::skip] #[inline] pub fn reserve_exact_with_weakfix(&mut self, additional: usize, weak_fixup: WeakFixupFn) { self.reserve_intern(additional, true, &mut Some(weak_fixup)); @@ -329,6 +337,7 @@ impl HeaderVec { } /// Resizes the vector hold exactly `self.len()` elements. + #[mutants::skip] #[inline(always)] pub fn shrink_to_fit(&mut self) { self.shrink_to(0); @@ -337,6 +346,7 @@ impl HeaderVec { /// Resizes the vector hold exactly `self.len()` elements. /// This method must be used when `HeaderVecWeak` are used. It takes a closure that is responsible for /// updating the weak references as additional parameter. + #[mutants::skip] #[inline(always)] pub fn shrink_to_fit_with_weakfix(&mut self, weak_fixup: WeakFixupFn) { self.shrink_to_with_weakfix(0, weak_fixup); @@ -564,6 +574,7 @@ impl HeaderVec { /// /// [`clear`]: HeaderVec::clear /// [`drain`]: HeaderVec::drain + #[mutants::skip] pub fn truncate(&mut self, len: usize) { unsafe { let old_len = self.len_exact(); @@ -609,6 +620,7 @@ impl HeaderVec { } /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. + #[mutants::skip] #[inline(always)] const fn offset() -> usize { // The first location, in units of size_of::(), that is after the header @@ -1054,6 +1066,7 @@ where H: Debug, T: Debug, { + #[mutants::skip] fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("HeaderVec") .field("header", &self.header().head) diff --git a/src/splice.rs b/src/splice.rs index a1f1054..e3b0290 100644 --- a/src/splice.rs +++ b/src/splice.rs @@ -55,6 +55,7 @@ where I: Iterator + fmt::Debug, I::Item: fmt::Debug, { + #[mutants::skip] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct(&format!( "Splice<{}, {}>", @@ -70,6 +71,7 @@ where impl Drop for Splice<'_, H, I> { #[track_caller] + #[mutants::skip] fn drop(&mut self) { self.drain.by_ref().for_each(drop); // At this point draining is done and the only remaining tasks are splicing diff --git a/tests/mutants.rs b/tests/mutants.rs index 2c94a42..e39d8cc 100644 --- a/tests/mutants.rs +++ b/tests/mutants.rs @@ -1,19 +1,377 @@ +//! This file is partially is generated with github copilot assistance. +//! The only objective here is to generate coverage over all code paths to +//! make 'cargo mutants' pass. Then the test suite should pass +//! 'cargo +nightly miri test'. Unless otherwise noted the tests here are +//! not extensively reviewed for semantic correctness. Eventually human +//! reviewed tests here should be moved to other unit tests. + use header_vec::HeaderVec; + #[test] -fn test_truncate_remaining_len() { +fn test_drain_size_hint() { let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); - let len = vec.len(); - vec.truncate(2); + let drain = vec.drain(1..4); + let (lower, upper) = drain.size_hint(); + assert_eq!(lower, 3); + assert_eq!(upper, Some(3)); +} + +#[test] +fn test_is_empty_exact() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + assert!(vec.is_empty_exact()); + + vec.push(1); + assert!(!vec.is_empty_exact()); + + vec.truncate(0); + assert!(vec.is_empty_exact()); +} + +#[test] +fn test_push_with_weakfix() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + let mut called = false; + vec.push_with_weakfix(1, &mut |_| called = true); + assert_eq!(vec.as_slice(), &[1]); + + // Test that push_with_weakfix actually pushes the value + assert_eq!(vec.len(), 1); + assert_eq!(vec[0], 1); + + // Test that multiple values can be pushed + vec.push_with_weakfix(2, &mut |_| {}); assert_eq!(vec.len(), 2); - assert_eq!(vec.as_slice(), &[1, 2]); - assert!(vec.capacity() >= len); + assert_eq!(vec[1], 2); } #[test] -fn test_drain_size_hint() { +fn test_splice_size_hint() { let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); - let drain = vec.drain(1..4); - let (lower, upper) = drain.size_hint(); + let splice = vec.splice(1..4, [8, 9]); + let (lower, upper) = splice.size_hint(); assert_eq!(lower, 3); assert_eq!(upper, Some(3)); } + +#[test] +fn test_reserve() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + let initial_cap = vec.capacity(); + vec.reserve(100); + assert!(vec.capacity() >= initial_cap + 100); +} + +#[test] +fn test_shrink_to() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + vec.reserve(100); + let big_cap = vec.capacity(); + vec.shrink_to(10); + assert!(vec.capacity() < big_cap); + assert!(vec.capacity() >= 10); +} + +#[test] +fn test_splice_drop() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + { + let _splice = vec.splice(1..4, [8, 9]); + // Let splice drop here + } + assert_eq!(vec.as_slice(), &[1, 8, 9, 5]); +} + +#[test] +fn test_drain_debug() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let drain = vec.drain(1..); + assert!(format!("{:?}", drain).contains("Drain")); +} + +#[test] +fn test_is() { + let vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let ptr = vec.ptr(); + assert!(vec.is(ptr)); + assert!(!vec.is(std::ptr::null())); +} + +#[test] +fn test_shrink_to_fit_with_weakfix() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + vec.reserve(100); + let big_cap = vec.capacity(); + let mut called = false; + vec.shrink_to_fit_with_weakfix(&mut |_| called = true); + assert!(vec.capacity() < big_cap); + assert_eq!(vec.capacity(), vec.len()); +} + +#[test] +fn test_into_iter() { + let vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let mut iter = (&vec).into_iter(); + assert_eq!(iter.next(), Some(&1)); + assert_eq!(iter.next(), Some(&2)); + assert_eq!(iter.next(), Some(&3)); + assert_eq!(iter.next(), None); +} + +#[test] +fn test_len_strict() { + let vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + assert_eq!(vec.len_strict(), 3); + assert_eq!(vec.len_strict(), vec.len()); +} + +#[test] +fn test_drain_as_ref() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let drain = vec.drain(..); + assert_eq!(drain.as_ref(), &[1, 2, 3]); +} + +#[test] +fn test_drain_keep_rest() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4]); + { + let mut drain = vec.drain(..); + assert_eq!(drain.next(), Some(1)); + drain.keep_rest(); + } + assert_eq!(vec.as_slice(), &[2, 3, 4]); +} + +#[test] +fn test_partial_eq() { + let vec1: HeaderVec<&str, i32> = HeaderVec::from_header_slice("header1", [1, 2]); + let vec2: HeaderVec<&str, i32> = HeaderVec::from_header_slice("header1", [1, 2]); + let vec3: HeaderVec<&str, i32> = HeaderVec::from_header_slice("header2", [1, 2]); + let vec4: HeaderVec<&str, i32> = HeaderVec::from_header_slice("header1", [1, 3]); + + assert_eq!(vec1, vec2); + assert_ne!(vec1, vec3); // Different header + assert_ne!(vec1, vec4); // Different elements +} + +#[test] +fn test_splice_next_back() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4]); + let mut splice = vec.splice(1..3, [5, 6]); + assert_eq!(splice.next_back(), Some(3)); + assert_eq!(splice.next_back(), Some(2)); + assert_eq!(splice.next_back(), None); +} + +#[test] +fn test_drain_keep_rest_tail_len() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + { + let mut drain = vec.drain(1..3); + drain.next(); + drain.keep_rest(); + } + assert_eq!(vec.as_slice(), &[1, 3, 4, 5]); +} + +#[test] +fn test_header_vec_basics() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + + // Test empty state + assert!(vec.is_empty()); + assert_eq!(vec.len(), 0); + assert_eq!(vec.len_strict(), 0); + assert_eq!(vec.as_slice().len(), 0); + + // Test non-empty state + vec.push(1); + assert!(!vec.is_empty()); + assert_eq!(vec.len(), 1); + assert_eq!(vec.len_strict(), 1); + assert_eq!(vec.as_mut_slice(), &mut [1]); + + // Test reserve_exact_with_weakfix + let initial_cap = vec.capacity(); + let mut called = false; + vec.reserve_exact_with_weakfix(10, &mut |_| called = true); + assert!(vec.capacity() >= initial_cap + 10); + + // Test offset calculation implicitly through indexing + vec.push(2); + vec.push(3); + assert_eq!(vec[0], 1); + assert_eq!(vec[1], 2); + assert_eq!(vec[2], 3); +} + +#[test] +fn test_splice_drop_behavior() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + { + let mut splice = vec.splice(1..3, vec![4, 5, 6]); + assert_eq!(splice.next(), Some(2)); + // Let splice drop here with remaining elements + } + assert_eq!(vec.as_slice(), &[1, 4, 5, 6]); +} + +#[test] +fn test_truncate_minus() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + vec.truncate(2); + assert_eq!(vec.len(), 2); + assert_eq!(vec.as_slice(), &[1, 2]); +} + +#[test] +fn test_is_empty_strict() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + assert!(vec.is_empty_strict()); + vec.push(1); + assert!(!vec.is_empty_strict()); +} + +#[test] +fn test_spare_capacity_mut() { + let mut vec: HeaderVec<(), i32> = HeaderVec::with_capacity(10, ()); + vec.push(1); + let spare = vec.spare_capacity_mut(); + assert!(!spare.is_empty()); + assert_eq!(spare.len(), 9); +} + +#[test] +fn test_weak_debug() { + let vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let weak = unsafe { vec.weak() }; + assert!(!format!("{:?}", weak).is_empty()); +} + +#[test] +fn test_offset_alignment() { + let mut vec: HeaderVec<(), i32> = HeaderVec::with_capacity(1, ()); + vec.push(42); + assert_eq!(vec[0], 42); // Tests correct memory layout/offset +} + +#[test] +fn test_reserve_with_weakfix() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + let mut called = false; + vec.reserve_with_weakfix(100, &mut |_| called = true); + assert!(vec.capacity() >= 100); +} + +#[test] +fn test_drop_behavior() { + struct DropCheck(std::rc::Rc>); + impl Drop for DropCheck { + fn drop(&mut self) { + *self.0.borrow_mut() = true; + } + } + + let dropped = std::rc::Rc::new(std::cell::RefCell::new(false)); + { + let mut vec: HeaderVec<(), _> = HeaderVec::new(()); + vec.push(DropCheck(dropped.clone())); + } + assert!(*dropped.borrow()); +} + +#[test] +fn test_offset_arithmetic() { + let mut vec: HeaderVec<(), u64> = HeaderVec::with_capacity(2, ()); + vec.push(123); + vec.push(456); + assert_eq!(vec[0], 123); + assert_eq!(vec[1], 456); +} + +#[test] +fn test_splice_debug() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let splice = vec.splice(1..3, vec![4, 5]); + assert!(format!("{:?}", splice).contains("Splice")); +} + +#[test] +fn test_extend_from_slice_with_weakfix() { + let mut vec: HeaderVec<(), i32> = HeaderVec::new(()); + let mut called = false; + vec.extend_from_slice_with_weakfix([1, 2, 3], &mut |_| called = true); + assert_eq!(vec.as_slice(), &[1, 2, 3]); +} + +#[test] +fn test_truncate_plus() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + vec.truncate(2); + assert_eq!(vec.len(), 2); + assert_eq!(vec.as_slice(), &[1, 2]); + + // Test truncating to larger size (should have no effect) + vec.truncate(10); + assert_eq!(vec.len(), 2); + assert_eq!(vec.as_slice(), &[1, 2]); +} + +#[test] +fn test_into_iter_mut() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + let mut iter = (&mut vec).into_iter(); + + // Verify we can mutate through the iterator + if let Some(first) = iter.next() { + *first = 100; + } + + // Verify we get mutable references to all elements in correct order + let mut collected: Vec<&mut i32> = iter.collect(); + *collected[0] = 200; + *collected[1] = 300; + + // Verify mutations happened + assert_eq!(vec.as_slice(), &[100, 200, 300]); +} + +#[test] +fn test_len_exact() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3]); + assert_eq!(vec.len_exact(), 3); + + vec.push(4); + assert_eq!(vec.len_exact(), 4); + + vec.truncate(2); + assert_eq!(vec.len_exact(), 2); + + // Additional checks that depend on len_exact + assert_eq!(vec.capacity(), vec.len_exact() + vec.spare_capacity()); + assert_eq!(vec.len_exact(), vec.as_slice().len()); + assert_eq!(vec.is_empty_exact(), vec.len_exact() == 0); +} + +#[test] +fn test_drain_keep_rest_advanced() { + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5]); + { + let mut drain = vec.drain(1..3); + // Take one item to test non-zero start position + assert_eq!(drain.next(), Some(2)); + drain.keep_rest(); + } + // Verify that items are correctly placed when keeping rest with non-zero start position + assert_eq!(vec.as_slice(), &[1, 3, 4, 5]); + + // Test with larger gaps to verify addition vs multiplication difference + let mut vec: HeaderVec<(), i32> = HeaderVec::from([1, 2, 3, 4, 5, 6, 7, 8]); + { + let mut drain = vec.drain(2..6); + assert_eq!(drain.next(), Some(3)); + drain.keep_rest(); + } + assert_eq!(vec.as_slice(), &[1, 2, 4, 5, 6, 7, 8]); +}