From d44385da2de58ffdb0e98cc3f326eed09afe1111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 17 Sep 2024 11:38:51 +0200 Subject: [PATCH 1/8] WIP: make len atomic, add and use methods using atomics This is a very early draft. method names and semantic may change. --- src/lib.rs | 74 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3aabad4..0918d7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,12 +10,13 @@ use core::{ ops::{Deref, DerefMut, Index, IndexMut}, ptr, slice::SliceIndex, + sync::atomic::{AtomicUsize, Ordering}, }; struct HeaderVecHeader { head: H, capacity: usize, - len: usize, + len: AtomicUsize, } /// A vector with a header of your choosing, behind a thin pointer @@ -78,19 +79,62 @@ impl HeaderVec { unsafe { core::ptr::write(&mut header.head, head) }; // These primitive types don't have drop implementations. header.capacity = capacity; - header.len = 0; + header.len = AtomicUsize::new(0); this } + /// Get the length of the vector from a mutable reference. + #[inline(always)] + pub fn len_from_mut(&mut self) -> usize { + *self.header_mut().len.get_mut() + } + + /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is + /// properly synchronized after it got atomically updated. + #[inline(always)] + pub fn len_atomic_acquire(&self) -> usize { + self.header().len.load(Ordering::Acquire) + } + + /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't + /// need exact synchronization semantic. + #[inline(always)] + pub fn len_atomic_relaxed(&self) -> usize { + self.header().len.load(Ordering::Relaxed) + } + + /// Alias for [`HeaderVec::len_atomic_relaxed`]. This gives always a valid view of the + /// length when used in isolation. #[inline(always)] pub fn len(&self) -> usize { - self.header().len + self.len_atomic_relaxed() + } + + /// Add `n` to the length of the vector atomically with `Ordering::Release`. + /// + /// # Safety + /// + /// Before incrementing the length of the vector, you must ensure that new elements are + /// properly initialized. + #[inline(always)] + pub unsafe fn len_atomic_add_release(&self, n: usize) -> usize { + self.header().len.fetch_add(n, Ordering::Release) + } + + #[inline(always)] + pub fn is_empty_from_mut(&mut self) -> bool { + self.len_from_mut() == 0 } #[inline(always)] pub fn is_empty(&self) -> bool { - self.len() == 0 + self.len_atomic_relaxed() == 0 + } + + #[inline(always)] + pub fn is_empty_atomic_acquire(&self) -> bool { + self.len_atomic_acquire() == 0 } #[inline(always)] @@ -100,12 +144,17 @@ impl HeaderVec { #[inline(always)] pub fn as_slice(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len()) } + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_relaxed()) } + } + + #[inline(always)] + pub fn as_slice_atomic_acquire(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } } #[inline(always)] pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len()) } + unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_from_mut()) } } /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`]. @@ -196,7 +245,7 @@ impl HeaderVec { /// Returns `true` if the memory was moved to a new location. /// In this case, you are responsible for updating the weak nodes. pub fn push(&mut self, item: T) -> Option<*const ()> { - let old_len = self.len(); + let old_len = self.len_from_mut(); let new_len = old_len + 1; let old_capacity = self.capacity(); // If it isn't big enough. @@ -208,7 +257,7 @@ impl HeaderVec { unsafe { core::ptr::write(self.start_ptr_mut().add(old_len), item); } - self.header_mut().len = new_len; + self.header_mut().len = new_len.into(); previous_pointer } @@ -221,7 +270,7 @@ impl HeaderVec { // This keeps track of the length (and next position) of the contiguous retained elements // at the beginning of the vector. let mut head = 0; - let original_len = self.len(); + let original_len = self.len_from_mut(); // Get the offset of the beginning of the slice. let start_ptr = self.start_ptr_mut(); // Go through each index. @@ -243,7 +292,7 @@ impl HeaderVec { } } // The head now represents the new length of the vector. - self.header_mut().len = head; + self.header_mut().len = head.into(); } /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. @@ -305,7 +354,7 @@ impl Drop for HeaderVec { fn drop(&mut self) { unsafe { ptr::drop_in_place(&mut self.header_mut().head); - for ix in 0..self.len() { + for ix in 0..self.len_from_mut() { ptr::drop_in_place(self.start_ptr_mut().add(ix)); } alloc::alloc::dealloc(self.ptr as *mut u8, Self::layout(self.capacity())); @@ -367,7 +416,8 @@ where T: Clone, { fn clone(&self) -> Self { - let mut new_vec = Self::with_capacity(self.len(), self.header().head.clone()); + let mut new_vec = + Self::with_capacity(self.len_atomic_acquire(), self.header().head.clone()); for e in self.as_slice() { new_vec.push(e.clone()); } From c9d53888b1b0923e03c621a3b2833f2330091b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 17 Sep 2024 16:39:41 +0200 Subject: [PATCH 2/8] rename methods in _exact/_strict, add 'atomic_append' feature, add 'spare_capacity()' method --- Cargo.toml | 4 ++ src/lib.rs | 143 ++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 107 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9d4a58c..4ba66aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,7 @@ keywords = ["header", "heap", "vec", "vector", "graph"] categories = ["no-std"] license = "MIT" readme = "README.md" + +[features] +default = ["atomic_append"] +atomic_append = [] diff --git a/src/lib.rs b/src/lib.rs index 79f5b05..0ca244c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,13 +10,18 @@ use core::{ ops::{Deref, DerefMut, Index, IndexMut}, ptr, slice::SliceIndex, - sync::atomic::{AtomicUsize, Ordering}, }; +#[cfg(feature = "atomic_append")] +use core::sync::atomic::{AtomicUsize, Ordering}; + struct HeaderVecHeader { head: H, capacity: usize, + #[cfg(feature = "atomic_append")] len: AtomicUsize, + #[cfg(not(feature = "atomic_append"))] + len: usize, } /// A vector with a header of your choosing behind a thin pointer @@ -75,62 +80,73 @@ impl HeaderVec { unsafe { core::ptr::write(&mut header.head, head) }; // These primitive types don't have drop implementations. header.capacity = capacity; - header.len = AtomicUsize::new(0); + header.len = 0usize.into(); this } - /// Get the length of the vector from a mutable reference. + /// Get the length of the vector from a mutable reference. When one has a `&mut + /// HeaderVec`, this is the method is always exact and can be slightly faster than the non + /// mutable `len()`. + #[cfg(feature = "atomic_append")] #[inline(always)] - pub fn len_from_mut(&mut self) -> usize { + pub fn len_exact(&mut self) -> usize { *self.header_mut().len.get_mut() } - - /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is - /// properly synchronized after it got atomically updated. + #[cfg(not(feature = "atomic_append"))] #[inline(always)] - pub fn len_atomic_acquire(&self) -> usize { - self.header().len.load(Ordering::Acquire) + pub fn len_exact(&mut self) -> usize { + self.header_mut().len } - /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't - /// need exact synchronization semantic. + /// This gives the length of the `HeaderVec`. This is the non synchronized variant may + /// produce racy results in case another thread atomically appended to + /// `&self`. Nevertheless it is always safe to use. + #[cfg(feature = "atomic_append")] #[inline(always)] - pub fn len_atomic_relaxed(&self) -> usize { - self.header().len.load(Ordering::Relaxed) + pub fn len(&self) -> usize { + self.len_atomic_relaxed() } - - /// Alias for [`HeaderVec::len_atomic_relaxed`]. This gives always a valid view of the - /// length when used in isolation. + #[cfg(not(feature = "atomic_append"))] #[inline(always)] pub fn len(&self) -> usize { - self.len_atomic_relaxed() + self.header().len } - /// Add `n` to the length of the vector atomically with `Ordering::Release`. - /// - /// # Safety - /// - /// Before incrementing the length of the vector, you must ensure that new elements are - /// properly initialized. + /// This gives the length of the `HeaderVec`. With `atomic_append` enabled this gives a + /// exact result *after* another thread atomically appended to this `HeaderVec`. It still + /// requires synchronization because the length may become invalidated when another thread + /// atomically appends data to this `HeaderVec` while we still work with the result of + /// this method. + #[cfg(not(feature = "atomic_append"))] #[inline(always)] - pub unsafe fn len_atomic_add_release(&self, n: usize) -> usize { - self.header().len.fetch_add(n, Ordering::Release) + pub fn len_strict(&self) -> usize { + self.header().len + } + #[cfg(feature = "atomic_append")] + #[inline(always)] + pub fn len_strict(&self) -> usize { + self.len_atomic_acquire() } + /// Check whenever a `HeaderVec` is empty. This uses a `&mut self` reference and is + /// always exact and may be slightly faster than the non mutable variant. #[inline(always)] - pub fn is_empty_from_mut(&mut self) -> bool { - self.len_from_mut() == 0 + pub fn is_empty_exact(&mut self) -> bool { + self.len_exact() == 0 } + /// Check whenever a `HeaderVec` is empty. This uses a `&self` reference and may be racy + /// when another thread atomically appended to this `HeaderVec`. #[inline(always)] pub fn is_empty(&self) -> bool { - self.len_atomic_relaxed() == 0 + self.len() == 0 } + /// Check whenever a `HeaderVec` is empty. see [`len_strict()`] about the exactness guarantees. #[inline(always)] - pub fn is_empty_atomic_acquire(&self) -> bool { - self.len_atomic_acquire() == 0 + pub fn is_empty_strict(&self) -> bool { + self.len_strict() == 0 } #[inline(always)] @@ -138,19 +154,20 @@ impl HeaderVec { self.header().capacity } + /// This is the amount of elements that can be added to the `HeaderVec` without reallocation. #[inline(always)] - pub fn as_slice(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_relaxed()) } + pub fn spare_capacity(&self) -> usize { + self.header().capacity - self.len_strict() } #[inline(always)] - pub fn as_slice_atomic_acquire(&self) -> &[T] { - unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } + pub fn as_slice(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_strict()) } } #[inline(always)] pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_from_mut()) } + unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut(), self.len_exact()) } } /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`]. @@ -241,7 +258,7 @@ impl HeaderVec { /// Returns `true` if the memory was moved to a new location. /// In this case, you are responsible for updating the weak nodes. pub fn push(&mut self, item: T) -> Option<*const ()> { - let old_len = self.len_from_mut(); + let old_len = self.len_exact(); let new_len = old_len + 1; let old_capacity = self.capacity(); // If it isn't big enough. @@ -266,7 +283,7 @@ impl HeaderVec { // This keeps track of the length (and next position) of the contiguous retained elements // at the beginning of the vector. let mut head = 0; - let original_len = self.len_from_mut(); + let original_len = self.len_exact(); // Get the offset of the beginning of the slice. let start_ptr = self.start_ptr_mut(); // Go through each index. @@ -346,11 +363,58 @@ impl HeaderVec { } } +#[cfg(feature = "atomic_append")] +/// The atomic append API is only enabled when the `atomic_append` feature flag is set (which +/// is the default). +impl HeaderVec { + /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is + /// properly synchronized after it got atomically updated. + #[inline(always)] + fn len_atomic_acquire(&self) -> usize { + self.header().len.load(Ordering::Acquire) + } + + /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't + /// need exact synchronization semantic. + #[inline(always)] + fn len_atomic_relaxed(&self) -> usize { + self.header().len.load(Ordering::Relaxed) + } + + /// Add `n` to the length of the vector atomically with `Ordering::Release`. + /// + /// # Safety + /// + /// Before incrementing the length of the vector, you must ensure that new elements are + /// properly initialized. + #[inline(always)] + unsafe fn len_atomic_add_release(&self, n: usize) -> usize { + self.header().len.fetch_add(n, Ordering::Release) + } + + #[inline(always)] + pub fn is_empty_atomic_acquire(&self) -> bool { + self.len_atomic_acquire() == 0 + } + + #[inline(always)] + pub fn as_slice_atomic_acquire(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.start_ptr(), self.len_atomic_acquire()) } + } + + /// Gets the pointer to the end of the slice. This returns a mutable pointer to + /// uninitialized memory behind the last element. + #[inline(always)] + fn end_ptr_atomic_mut(&self) -> *mut T { + unsafe { self.ptr.add(Self::offset()).add(self.len_atomic_acquire()) } + } +} + impl Drop for HeaderVec { fn drop(&mut self) { unsafe { ptr::drop_in_place(&mut self.header_mut().head); - for ix in 0..self.len_from_mut() { + for ix in 0..self.len_exact() { ptr::drop_in_place(self.start_ptr_mut().add(ix)); } alloc::alloc::dealloc(self.ptr as *mut u8, Self::layout(self.capacity())); @@ -412,8 +476,7 @@ where T: Clone, { fn clone(&self) -> Self { - let mut new_vec = - Self::with_capacity(self.len_atomic_acquire(), self.header().head.clone()); + let mut new_vec = Self::with_capacity(self.len_strict(), self.header().head.clone()); for e in self.as_slice() { new_vec.push(e.clone()); } From ecc95544ecd398d54bdb83b0a37cf23f7893435d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Tue, 17 Sep 2024 17:20:04 +0200 Subject: [PATCH 3/8] add push_atomic() --- src/lib.rs | 26 ++++++++++++++++++++++++++ tests/atomic_append.rs | 16 ++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tests/atomic_append.rs diff --git a/src/lib.rs b/src/lib.rs index 0ca244c..bbb4fb2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -408,6 +408,32 @@ impl HeaderVec { fn end_ptr_atomic_mut(&self) -> *mut T { unsafe { self.ptr.add(Self::offset()).add(self.len_atomic_acquire()) } } + + /// Atomically adds an item to the end of the list without reallocation. + /// + /// # Errors + /// + /// If the vector is full, the item is returned. + /// + /// # Safety + /// + /// There must be only one thread calling this method at any time. Synchronization has to + /// be provided by the user. + pub unsafe fn push_atomic(&self, item: T) -> Result<(), T> { + // relaxed is good enough here because this should be the only thread calling this method. + let len = self.len_atomic_relaxed(); + if len < self.capacity() { + unsafe { + core::ptr::write(self.end_ptr_atomic_mut(), item); + }; + let len_again = self.len_atomic_add_release(1); + // in debug builds we check for races, the chance to catch these are still pretty minimal + debug_assert_eq!(len_again, len, "len was updated by another thread"); + Ok(()) + } else { + Err(item) + } + } } impl Drop for HeaderVec { diff --git a/tests/atomic_append.rs b/tests/atomic_append.rs new file mode 100644 index 0000000..a1112db --- /dev/null +++ b/tests/atomic_append.rs @@ -0,0 +1,16 @@ +#![cfg(feature = "atomic_append")] +extern crate std; + +use header_vec::*; + +#[test] +fn test_atomic_append() { + let mut hv = HeaderVec::with_capacity(10, ()); + + hv.push(1); + unsafe { hv.push_atomic(2).unwrap() }; + hv.push(3); + + assert_eq!(hv.len(), 3); + assert_eq!(hv.as_slice(), [1, 2, 3]); +} From 993cad03125f1d817c3db192b85065fd7f1760b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 03:50:22 +0200 Subject: [PATCH 4/8] WIP: reserve/shrink API's --- src/lib.rs | 91 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bbb4fb2..fd3e430 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -222,12 +222,83 @@ impl HeaderVec { self.ptr = weak.ptr; } + /// Reserves capacity for at least `additional` more elements to be inserted in the given `HeaderVec`. + #[inline(always)] + pub fn reserve(&mut self, additional: usize) -> Option<*const ()> { + if self.spare_capacity() < additional { + let len = self.len_exact(); + unsafe { self.resize_cold(len + additional, false) } + } else { + None + } + } + + /// Reserves capacity for exactly `additional` more elements to be inserted in the given `HeaderVec`. + #[inline] + pub fn reserve_exact(&mut self, additional: usize) -> Option<*const ()> { + if self.spare_capacity() < additional { + let len = self.len_exact(); + unsafe { self.resize_cold(len + additional, true) } + } else { + None + } + } + + /// Shrinks the capacity of the `HeaderVec` to the `min_capacity` or `self.len()`, whichever is larger. + #[inline] + pub fn shrink_to(&mut self, min_capacity: usize) -> Option<*const ()> { + let requested_capacity = self.len_exact().max(min_capacity); + unsafe { self.resize_cold(requested_capacity, true) } + } + + /// Resizes the vector hold exactly `self.len()` elements. + #[inline(always)] + pub fn shrink_to_fit(&mut self) -> Option<*const ()> { + let len = self.len_exact(); + self.shrink_to(len) + } + + /// Resize the vector to have at least room for `additional` more elements. + /// does exact resizing if `exact` is true. + /// + /// Returns `Some(*const ())` if the memory was moved to a new location. + /// + /// # Safety + /// + /// `requested_capacity` must be greater or equal than `self.len()` #[cold] - fn resize_insert(&mut self) -> Option<*const ()> { + unsafe fn resize_cold(&mut self, requested_capacity: usize, exact: bool) -> Option<*const ()> { + // For efficiency we do only a debug_assert here + debug_assert!( + self.len_exact() <= requested_capacity, + "requested capacity is less than current length" + ); let old_capacity = self.capacity(); - let new_capacity = old_capacity * 2; - // Set the new capacity. - self.header_mut().capacity = new_capacity; + debug_assert_ne!(old_capacity, 0, "capacity of 0 not yet supported"); + debug_assert_ne!(requested_capacity, 0, "capacity of 0 not yet supported"); + + let new_capacity = if requested_capacity > old_capacity { + if exact { + // exact growing + requested_capacity + } else if requested_capacity <= old_capacity * 2 { + // doubling the capacity is sufficient + old_capacity * 2 + } else { + // requested more than twice as much space, reserve the next multiple of + // old_capacity that is greater than the requested capacity. This gives headroom + // for new inserts while not doubling the memory requirement with bulk requests + (requested_capacity / old_capacity + 1).saturating_mul(old_capacity) + } + } else if exact { + // exact shrinking + requested_capacity + } else { + unimplemented!() + // or: (has no public API yet) + // // shrink to the next power of two or self.capacity, whichever is smaller + // requested_capacity.next_power_of_two().min(self.capacity()) + }; // Reallocate the pointer. let ptr = unsafe { alloc::alloc::realloc( @@ -249,24 +320,20 @@ impl HeaderVec { }; // Assign the new pointer. self.ptr = ptr; + // And set the new capacity. + self.header_mut().capacity = new_capacity; previous_pointer } /// Adds an item to the end of the list. /// - /// Returns `true` if the memory was moved to a new location. + /// Returns `Some(*const ())` if the memory was moved to a new location. /// In this case, you are responsible for updating the weak nodes. pub fn push(&mut self, item: T) -> Option<*const ()> { let old_len = self.len_exact(); let new_len = old_len + 1; - let old_capacity = self.capacity(); - // If it isn't big enough. - let previous_pointer = if new_len > old_capacity { - self.resize_insert() - } else { - None - }; + let previous_pointer = self.reserve(1); unsafe { core::ptr::write(self.start_ptr_mut().add(old_len), item); } From 4bcab1f6ed2c60cd7afd938d32f7c657e8330eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 14:36:20 +0200 Subject: [PATCH 5/8] FIX: the reseve functions need saturating add as well --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fd3e430..0a25ad7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -227,7 +227,7 @@ impl HeaderVec { pub fn reserve(&mut self, additional: usize) -> Option<*const ()> { if self.spare_capacity() < additional { let len = self.len_exact(); - unsafe { self.resize_cold(len + additional, false) } + unsafe { self.resize_cold(len.saturating_add(additional), false) } } else { None } @@ -238,7 +238,7 @@ impl HeaderVec { pub fn reserve_exact(&mut self, additional: usize) -> Option<*const ()> { if self.spare_capacity() < additional { let len = self.len_exact(); - unsafe { self.resize_cold(len + additional, true) } + unsafe { self.resize_cold(len.saturating_add(additional), true) } } else { None } From decaeed7c7d8bd4660e66082460d500321bdc223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 15:51:14 +0200 Subject: [PATCH 6/8] FIX: Miri, unaligned access * introduces a helper union to fix simplify alignment calculations no need for cmp and phantom data anymore * add simple testcase that triggered the miri issue * change end_ptr_atomic_mut(), using the rewritten start_ptr() --- src/lib.rs | 39 ++++++++++++++++++--------------------- tests/simple.rs | 9 +++++++++ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0a25ad7..5bcfb68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,9 +3,7 @@ extern crate alloc; use core::{ - cmp, fmt::Debug, - marker::PhantomData, mem::{self, ManuallyDrop}, ops::{Deref, DerefMut, Index, IndexMut}, ptr, @@ -24,6 +22,12 @@ struct HeaderVecHeader { len: usize, } +// This union will be properly aligned and sized to store headers followed by T's. +union AlignedHeader { + _header: ManuallyDrop>, + _data: ManuallyDrop<[T; 0]>, +} + /// A vector with a header of your choosing behind a thin pointer /// /// # Example @@ -47,8 +51,7 @@ struct HeaderVecHeader { /// All of the data, like our header `OurHeaderType { a: 2 }`, the length of the vector: `2`, /// and the contents of the vector `['x', 'z']` resides on the other side of the pointer. pub struct HeaderVec { - ptr: *mut T, - _phantom: PhantomData, + ptr: *mut AlignedHeader, } impl HeaderVec { @@ -58,9 +61,9 @@ impl HeaderVec { pub fn with_capacity(capacity: usize, head: H) -> Self { assert!(capacity > 0, "HeaderVec capacity cannot be 0"); - // Allocate the initial memory, which is unititialized. + // Allocate the initial memory, which is uninitialized. let layout = Self::layout(capacity); - let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut T; + let ptr = unsafe { alloc::alloc::alloc(layout) } as *mut AlignedHeader; // Handle out-of-memory. if ptr.is_null() { @@ -68,10 +71,7 @@ impl HeaderVec { } // Create self. - let mut this = Self { - ptr, - _phantom: PhantomData, - }; + let mut this = Self { ptr }; // Set the header. let header = this.header_mut(); @@ -204,10 +204,7 @@ impl HeaderVec { #[inline(always)] pub unsafe fn weak(&self) -> HeaderVecWeak { HeaderVecWeak { - header_vec: ManuallyDrop::new(Self { - ptr: self.ptr, - _phantom: PhantomData, - }), + header_vec: ManuallyDrop::new(Self { ptr: self.ptr }), } } @@ -305,7 +302,7 @@ impl HeaderVec { self.ptr as *mut u8, Self::layout(old_capacity), Self::elems_to_mem_bytes(new_capacity), - ) as *mut T + ) as *mut AlignedHeader }; // Handle out-of-memory. if ptr.is_null() { @@ -377,10 +374,10 @@ impl HeaderVec { /// Gives the offset in units of T (as if the pointer started at an array of T) that the slice actually starts at. #[inline(always)] - fn offset() -> usize { + const fn offset() -> usize { // The first location, in units of size_of::(), that is after the header // It's the end of the header, rounded up to the nearest size_of::() - (mem::size_of::>() + mem::size_of::() - 1) / mem::size_of::() + mem::size_of::>() / mem::size_of::() } /// Compute the number of elements (in units of T) to allocate for a given capacity. @@ -400,7 +397,7 @@ impl HeaderVec { fn layout(capacity: usize) -> alloc::alloc::Layout { alloc::alloc::Layout::from_size_align( Self::elems_to_mem_bytes(capacity), - cmp::max(mem::align_of::(), mem::align_of::()), + mem::align_of::>() ) .expect("unable to produce memory layout with Hrc key type (is it a zero sized type? they are not permitted)") } @@ -408,13 +405,13 @@ impl HeaderVec { /// Gets the pointer to the start of the slice. #[inline(always)] fn start_ptr(&self) -> *const T { - unsafe { self.ptr.add(Self::offset()) } + unsafe { (self.ptr as *const T).add(Self::offset()) } } /// Gets the pointer to the start of the slice. #[inline(always)] fn start_ptr_mut(&mut self) -> *mut T { - unsafe { self.ptr.add(Self::offset()) } + unsafe { (self.ptr as *mut T).add(Self::offset()) } } #[inline(always)] @@ -473,7 +470,7 @@ impl HeaderVec { /// uninitialized memory behind the last element. #[inline(always)] fn end_ptr_atomic_mut(&self) -> *mut T { - unsafe { self.ptr.add(Self::offset()).add(self.len_atomic_acquire()) } + unsafe { self.start_ptr().add(self.len_atomic_acquire()) as *mut T } } /// Atomically adds an item to the end of the list without reallocation. diff --git a/tests/simple.rs b/tests/simple.rs index 2b42d85..2be342f 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -44,3 +44,12 @@ fn test_head_array() { v_orig.as_slice().iter().copied().collect::() ); } + +// This shown a miri error +#[test] +fn test_push() { + let mut hv = HeaderVec::with_capacity(10, ()); + + hv.push(123); + assert_eq!(hv[0], 123); +} From 58d68a14d5b6a67ece1c278ca262422a1a5e4459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 18:33:22 +0200 Subject: [PATCH 7/8] add: extend_from_slice() --- src/lib.rs | 29 +++++++++++++++++++++++++++++ tests/simple.rs | 9 +++++++++ 2 files changed, 38 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 5bcfb68..8fa9440 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -414,6 +414,13 @@ impl HeaderVec { unsafe { (self.ptr as *mut T).add(Self::offset()) } } + /// Gets the pointer to the end of the slice. This returns a mutable pointer to + /// uninitialized memory behind the last element. + #[inline(always)] + fn end_ptr_mut(&mut self) -> *mut T { + unsafe { self.start_ptr_mut().add(self.len_exact()) } + } + #[inline(always)] fn header(&self) -> &HeaderVecHeader { // The beginning of the memory is always the header. @@ -427,6 +434,28 @@ impl HeaderVec { } } +impl HeaderVec { + /// Adds items from a slice to the end of the list. + /// + /// Returns `Some(*const ())` if the memory was moved to a new location. + /// In this case, you are responsible for updating the weak nodes. + pub fn extend_from_slice(&mut self, slice: &[T]) -> Option<*const ()> { + let previous_pointer = self.reserve(slice.len()); + + // copy data + let end_ptr = self.end_ptr_mut(); + for (index, item) in slice.iter().enumerate() { + unsafe { + core::ptr::write(end_ptr.add(index), item.clone()); + } + } + // correct the len + self.header_mut().len = (self.len_exact() + slice.len()).into(); + + previous_pointer + } +} + #[cfg(feature = "atomic_append")] /// The atomic append API is only enabled when the `atomic_append` feature flag is set (which /// is the default). diff --git a/tests/simple.rs b/tests/simple.rs index 2be342f..82aa8f4 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -53,3 +53,12 @@ fn test_push() { hv.push(123); assert_eq!(hv[0], 123); } + +#[test] +fn test_extend_from_slice() { + let mut hv = HeaderVec::new(()); + + hv.extend_from_slice(&[0, 1, 2]); + hv.extend_from_slice(&[3, 4, 5]); + assert_eq!(hv.as_slice(), &[0, 1, 2, 3, 4, 5]); +} From 00be0d8ddb208360d84804d87aff66034cfbfd93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Th=C3=A4ter?= Date: Mon, 23 Sep 2024 19:27:33 +0200 Subject: [PATCH 8/8] add: extend_from_slice_atomic() --- src/lib.rs | 35 +++++++++++++++++++++++++++++++++++ tests/atomic_append.rs | 11 +++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 8fa9440..0e6bfe6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -529,6 +529,41 @@ impl HeaderVec { } } +#[cfg(feature = "atomic_append")] +impl HeaderVec { + /// Atomically add items from a slice to the end of the list. without reallocation + /// + /// # Errors + /// + /// If the vector is full, the item is returned. + /// + /// # Safety + /// + /// There must be only one thread calling this method at any time. Synchronization has to + /// be provided by the user. + pub unsafe fn extend_from_slice_atomic<'a>(&self, slice: &'a [T]) -> Result<(), &'a [T]> { + #[cfg(debug_assertions)] // only for the race check later + let len = self.len_atomic_relaxed(); + if self.spare_capacity() >= slice.len() { + // copy data + let end_ptr = self.end_ptr_atomic_mut(); + for (index, item) in slice.iter().enumerate() { + unsafe { + core::ptr::write(end_ptr.add(index), item.clone()); + } + } + // correct the len + let len_again = self.len_atomic_add_release(slice.len()); + // in debug builds we check for races, the chance to catch these are still pretty minimal + #[cfg(debug_assertions)] + debug_assert_eq!(len_again, len, "len was updated by another thread"); + Ok(()) + } else { + Err(slice) + } + } +} + impl Drop for HeaderVec { fn drop(&mut self) { unsafe { diff --git a/tests/atomic_append.rs b/tests/atomic_append.rs index a1112db..42f5535 100644 --- a/tests/atomic_append.rs +++ b/tests/atomic_append.rs @@ -14,3 +14,14 @@ fn test_atomic_append() { assert_eq!(hv.len(), 3); assert_eq!(hv.as_slice(), [1, 2, 3]); } + +#[test] +fn test_extend_from_slice() { + let hv = HeaderVec::with_capacity(6, ()); + + unsafe { + hv.extend_from_slice_atomic(&[0, 1, 2]).unwrap(); + hv.extend_from_slice_atomic(&[3, 4, 5]).unwrap(); + } + assert_eq!(hv.as_slice(), &[0, 1, 2, 3, 4, 5]); +}