Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(rust): Use size_of/align_of from prelude #19311

Merged
merged 5 commits into from
Oct 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/array/primitive/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,8 @@ impl<T: NativeType> PrimitiveArray<T> {

// SAFETY: this is fine, we checked size and alignment, and NativeType
// is always Pod.
assert_eq!(std::mem::size_of::<T>(), std::mem::size_of::<U>());
assert_eq!(std::mem::align_of::<T>(), std::mem::align_of::<U>());
assert_eq!(size_of::<T>(), size_of::<U>());
assert_eq!(align_of::<T>(), align_of::<U>());
let new_values = unsafe { std::mem::transmute::<Buffer<T>, Buffer<U>>(values) };
PrimitiveArray::new(U::PRIMITIVE.into(), new_values, validity)
}
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/aligned.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ impl<'a, T: BitChunk> AlignedBitmapSlice<'a, T> {
/// The length (in bits) of the portion of the bitmap found in bulk.
#[inline(always)]
pub fn bulk_bitlen(&self) -> usize {
8 * std::mem::size_of::<T>() * self.bulk.len()
8 * size_of::<T>() * self.bulk.len()
}

/// The length (in bits) of the portion of the bitmap found in suffix.
Expand All @@ -77,7 +77,7 @@ impl<'a, T: BitChunk> AlignedBitmapSlice<'a, T> {
offset %= 8;

// Fast-path: fits entirely in one chunk.
let chunk_len = std::mem::size_of::<T>();
let chunk_len = size_of::<T>();
let chunk_len_bits = 8 * chunk_len;
if offset + len <= chunk_len_bits {
let mut prefix = load_chunk_le::<T>(bytes) >> offset;
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/bitmap_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pub(crate) fn push_bitchunk<T: BitChunk>(buffer: &mut Vec<u8>, value: T) {

/// Creates a [`Vec<u8>`] from a [`TrustedLen`] of [`BitChunk`].
pub fn chunk_iter_to_vec<T: BitChunk, I: TrustedLen<Item = T>>(iter: I) -> Vec<u8> {
let cap = iter.size_hint().0 * std::mem::size_of::<T>();
let cap = iter.size_hint().0 * size_of::<T>();
let mut buffer = Vec::with_capacity(cap);
for v in iter {
push_bitchunk(&mut buffer, v)
Expand All @@ -24,7 +24,7 @@ fn chunk_iter_to_vec_and_remainder<T: BitChunk, I: TrustedLen<Item = T>>(
iter: I,
remainder: T,
) -> Vec<u8> {
let cap = (iter.size_hint().0 + 1) * std::mem::size_of::<T>();
let cap = (iter.size_hint().0 + 1) * size_of::<T>();
let mut buffer = Vec::with_capacity(cap);
for v in iter {
push_bitchunk(&mut buffer, v)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ impl<'a, T: BitChunk> BitChunksExact<'a, T> {
#[inline]
pub fn new(bitmap: &'a [u8], length: usize) -> Self {
assert!(length <= bitmap.len() * 8);
let size_of = std::mem::size_of::<T>();
let size_of = size_of::<T>();

let bitmap = &bitmap[..length.saturating_add(7) / 8];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ where
// expected = [n5, n6, n7, c0, c1, c2, c3, c4]

// 1. unset most significants of `next` up to `offset`
let inverse_offset = std::mem::size_of::<T>() * 8 - offset;
let inverse_offset = size_of::<T>() * 8 - offset;
next <<= inverse_offset;
// next = [n5, n6, n7, 0 , 0 , 0 , 0 , 0 ]

Expand Down
8 changes: 4 additions & 4 deletions crates/polars-arrow/src/bitmap/utils/chunk_iterator/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ fn copy_with_merge<T: BitChunk>(dst: &mut T::Bytes, bytes: &[u8], bit_offset: us
bytes
.windows(2)
.chain(std::iter::once([bytes[bytes.len() - 1], 0].as_ref()))
.take(std::mem::size_of::<T>())
.take(size_of::<T>())
.enumerate()
.for_each(|(i, w)| {
let val = merge_reversed(w[0], w[1], bit_offset);
Expand All @@ -59,7 +59,7 @@ impl<'a, T: BitChunk> BitChunks<'a, T> {

let slice = &slice[offset / 8..];
let bit_offset = offset % 8;
let size_of = std::mem::size_of::<T>();
let size_of = size_of::<T>();

let bytes_len = len / 8;
let bytes_upper_len = (len + bit_offset + 7) / 8;
Expand Down Expand Up @@ -120,7 +120,7 @@ impl<'a, T: BitChunk> BitChunks<'a, T> {
// all remaining bytes
self.remainder_bytes
.iter()
.take(std::mem::size_of::<T>())
.take(size_of::<T>())
.enumerate()
.for_each(|(i, val)| remainder[i] = *val);

Expand All @@ -137,7 +137,7 @@ impl<'a, T: BitChunk> BitChunks<'a, T> {

/// Returns the remainder bits in [`BitChunks::remainder`].
pub fn remainder_len(&self) -> usize {
self.len - (std::mem::size_of::<T>() * ((self.len / 8) / std::mem::size_of::<T>()) * 8)
self.len - (size_of::<T>() * ((self.len / 8) / size_of::<T>()) * 8)
}
}

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/utils/chunks_exact_mut.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use super::BitChunk;
///
/// # Safety
/// The slices returned by this iterator are guaranteed to have length equal to
/// `std::mem::size_of::<T>()`.
/// `size_of::<T>()`.
#[derive(Debug)]
pub struct BitChunksExactMut<'a, T: BitChunk> {
chunks: std::slice::ChunksExactMut<'a, u8>,
Expand All @@ -18,7 +18,7 @@ impl<'a, T: BitChunk> BitChunksExactMut<'a, T> {
#[inline]
pub fn new(bitmap: &'a mut [u8], length: usize) -> Self {
assert!(length <= bitmap.len() * 8);
let size_of = std::mem::size_of::<T>();
let size_of = size_of::<T>();

let bitmap = &mut bitmap[..length.saturating_add(7) / 8];

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/utils/iterator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ impl<'a> BitmapIter<'a> {
let num_words = n / 64;

if num_words > 0 {
assert!(self.bytes.len() >= num_words * std::mem::size_of::<u64>());
assert!(self.bytes.len() >= num_words * size_of::<u64>());

bitmap.extend_from_slice(self.bytes, 0, num_words * u64::BITS as usize);

Expand All @@ -189,7 +189,7 @@ impl<'a> BitmapIter<'a> {
return;
}

assert!(self.bytes.len() >= std::mem::size_of::<u64>());
assert!(self.bytes.len() >= size_of::<u64>());

self.word_len = usize::min(self.rest_len, 64);
self.rest_len -= self.word_len;
Expand Down
8 changes: 4 additions & 4 deletions crates/polars-arrow/src/buffer/immutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,10 @@ impl<T: crate::types::NativeType> From<arrow_buffer::Buffer> for Buffer<T> {
impl<T: crate::types::NativeType> From<Buffer<T>> for arrow_buffer::Buffer {
fn from(value: Buffer<T>) -> Self {
let offset = value.offset();
value.storage.into_arrow_buffer().slice_with_length(
offset * std::mem::size_of::<T>(),
value.length * std::mem::size_of::<T>(),
)
value
.storage
.into_arrow_buffer()
.slice_with_length(offset * size_of::<T>(), value.length * size_of::<T>())
}
}

Expand Down
16 changes: 8 additions & 8 deletions crates/polars-arrow/src/compute/aggregate/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ macro_rules! dyn_binary {
let values_end = offsets[offsets.len() - 1] as usize;

values_end - values_start
+ offsets.len() * std::mem::size_of::<$o>()
+ offsets.len() * size_of::<$o>()
+ validity_size(array.validity())
}};
}
Expand Down Expand Up @@ -50,15 +50,15 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
},
Primitive(PrimitiveType::DaysMs) => {
let array = array.as_any().downcast_ref::<DaysMsArray>().unwrap();
array.values().len() * std::mem::size_of::<i32>() * 2 + validity_size(array.validity())
array.values().len() * size_of::<i32>() * 2 + validity_size(array.validity())
},
Primitive(primitive) => with_match_primitive_type_full!(primitive, |$T| {
let array = array
.as_any()
.downcast_ref::<PrimitiveArray<$T>>()
.unwrap();

array.values().len() * std::mem::size_of::<$T>() + validity_size(array.validity())
array.values().len() * size_of::<$T>() + validity_size(array.validity())
}),
Binary => dyn_binary!(array, BinaryArray<i32>, i32),
FixedSizeBinary => {
Expand All @@ -74,7 +74,7 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
List => {
let array = array.as_any().downcast_ref::<ListArray<i32>>().unwrap();
estimated_bytes_size(array.values().as_ref())
+ array.offsets().len_proxy() * std::mem::size_of::<i32>()
+ array.offsets().len_proxy() * size_of::<i32>()
+ validity_size(array.validity())
},
FixedSizeList => {
Expand All @@ -84,7 +84,7 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
LargeList => {
let array = array.as_any().downcast_ref::<ListArray<i64>>().unwrap();
estimated_bytes_size(array.values().as_ref())
+ array.offsets().len_proxy() * std::mem::size_of::<i64>()
+ array.offsets().len_proxy() * size_of::<i64>()
+ validity_size(array.validity())
},
Struct => {
Expand All @@ -99,11 +99,11 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
},
Union => {
let array = array.as_any().downcast_ref::<UnionArray>().unwrap();
let types = array.types().len() * std::mem::size_of::<i8>();
let types = array.types().len() * size_of::<i8>();
let offsets = array
.offsets()
.as_ref()
.map(|x| x.len() * std::mem::size_of::<i32>())
.map(|x| x.len() * size_of::<i32>())
.unwrap_or_default();
let fields = array
.fields()
Expand All @@ -124,7 +124,7 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
BinaryView => binview_size::<[u8]>(array.as_any().downcast_ref().unwrap()),
Map => {
let array = array.as_any().downcast_ref::<MapArray>().unwrap();
let offsets = array.offsets().len_proxy() * std::mem::size_of::<i32>();
let offsets = array.offsets().len_proxy() * size_of::<i32>();
offsets + estimated_bytes_size(array.field().as_ref()) + validity_size(array.validity())
},
}
Expand Down
8 changes: 2 additions & 6 deletions crates/polars-arrow/src/ffi/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,11 +225,7 @@ unsafe fn get_buffer_ptr<T: NativeType>(
);
}

if array
.buffers
.align_offset(std::mem::align_of::<*mut *const u8>())
!= 0
{
if array.buffers.align_offset(align_of::<*mut *const u8>()) != 0 {
polars_bail!( ComputeError:
"an ArrowArray of type {dtype:?}
must have buffer {index} aligned to type {}",
Expand Down Expand Up @@ -294,7 +290,7 @@ unsafe fn create_buffer<T: NativeType>(

// We have to check alignment.
// This is the zero-copy path.
if ptr.align_offset(std::mem::align_of::<T>()) == 0 {
if ptr.align_offset(align_of::<T>()) == 0 {
let storage = SharedStorage::from_internal_arrow_array(ptr, len, owner);
Ok(Buffer::from_storage(storage).sliced(offset, len - offset))
}
Expand Down
14 changes: 6 additions & 8 deletions crates/polars-arrow/src/io/avro/read/deserialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -195,19 +195,17 @@ fn deserialize_value<'a>(
array.push(Some(value))
},
PrimitiveType::Float32 => {
let value =
f32::from_le_bytes(block[..std::mem::size_of::<f32>()].try_into().unwrap());
block = &block[std::mem::size_of::<f32>()..];
let value = f32::from_le_bytes(block[..size_of::<f32>()].try_into().unwrap());
block = &block[size_of::<f32>()..];
let array = array
.as_mut_any()
.downcast_mut::<MutablePrimitiveArray<f32>>()
.unwrap();
array.push(Some(value))
},
PrimitiveType::Float64 => {
let value =
f64::from_le_bytes(block[..std::mem::size_of::<f64>()].try_into().unwrap());
block = &block[std::mem::size_of::<f64>()..];
let value = f64::from_le_bytes(block[..size_of::<f64>()].try_into().unwrap());
block = &block[size_of::<f64>()..];
let array = array
.as_mut_any()
.downcast_mut::<MutablePrimitiveArray<f64>>()
Expand Down Expand Up @@ -404,10 +402,10 @@ fn skip_item<'a>(
let _ = util::zigzag_i64(&mut block)?;
},
PrimitiveType::Float32 => {
block = &block[std::mem::size_of::<f32>()..];
block = &block[size_of::<f32>()..];
},
PrimitiveType::Float64 => {
block = &block[std::mem::size_of::<f64>()..];
block = &block[size_of::<f64>()..];
},
PrimitiveType::MonthDayNano => {
block = &block[12..];
Expand Down
6 changes: 3 additions & 3 deletions crates/polars-arrow/src/io/ipc/read/read_basic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ fn read_swapped<T: NativeType, R: Read + Seek>(
is_little_endian: bool,
) -> PolarsResult<()> {
// slow case where we must reverse bits
let mut slice = vec![0u8; length * std::mem::size_of::<T>()];
let mut slice = vec![0u8; length * size_of::<T>()];
reader.read_exact(&mut slice)?;

let chunks = slice.chunks_exact(std::mem::size_of::<T>());
let chunks = slice.chunks_exact(size_of::<T>());
if !is_little_endian {
// machine is little endian, file is big endian
buffer
Expand Down Expand Up @@ -67,7 +67,7 @@ fn read_uncompressed_buffer<T: NativeType, R: Read + Seek>(
length: usize,
is_little_endian: bool,
) -> PolarsResult<Vec<T>> {
let required_number_of_bytes = length.saturating_mul(std::mem::size_of::<T>());
let required_number_of_bytes = length.saturating_mul(size_of::<T>());
if required_number_of_bytes > buffer_length {
polars_bail!(
oos = OutOfSpecKind::InvalidBuffer {
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/io/ipc/write/serialize/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ fn _write_buffer_from_iter<T: NativeType, I: TrustedLen<Item = T>>(
is_little_endian: bool,
) {
let len = buffer.size_hint().0;
arrow_data.reserve(len * std::mem::size_of::<T>());
arrow_data.reserve(len * size_of::<T>());
if is_little_endian {
buffer
.map(|x| T::to_le_bytes(&x))
Expand All @@ -303,7 +303,7 @@ fn _write_compressed_buffer_from_iter<T: NativeType, I: TrustedLen<Item = T>>(
compression: Compression,
) {
let len = buffer.size_hint().0;
let mut swapped = Vec::with_capacity(len * std::mem::size_of::<T>());
let mut swapped = Vec::with_capacity(len * size_of::<T>());
if is_little_endian {
buffer
.map(|x| T::to_le_bytes(&x))
Expand Down
3 changes: 1 addition & 2 deletions crates/polars-arrow/src/legacy/kernels/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ impl<'a> MaskedSlicesIterator<'a> {
pub(crate) fn new(mask: &'a BooleanArray) -> Self {
let chunks = mask.values().chunks::<u64>();

let chunk_bits = 8 * std::mem::size_of::<u64>();
let chunk_len = mask.len() / chunk_bits;
let chunk_len = mask.len() / 64;
let remainder_len = chunks.remainder_len();
let remainder_mask = chunks.remainder();

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/mmap/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fn check_bytes_len_and_is_aligned<T: NativeType>(
bytes: &[u8],
expected_len: usize,
) -> PolarsResult<bool> {
if bytes.len() < std::mem::size_of::<T>() * expected_len {
if bytes.len() < size_of::<T>() * expected_len {
polars_bail!(ComputeError: "buffer's length is too small in mmap")
};

Expand Down Expand Up @@ -281,7 +281,7 @@ fn mmap_primitive<P: NativeType, T: AsRef<[u8]>>(
let bytes = get_bytes(data_ref, block_offset, buffers)?;
let is_aligned = check_bytes_len_and_is_aligned::<P>(bytes, num_rows)?;

let out = if is_aligned || std::mem::size_of::<T>() <= 8 {
let out = if is_aligned || size_of::<T>() <= 8 {
assert!(
is_aligned,
"primitive type with size <= 8 bytes should have been aligned"
Expand Down
6 changes: 3 additions & 3 deletions crates/polars-arrow/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ impl<T> SharedStorage<T> {
impl<T: crate::types::NativeType> SharedStorage<T> {
pub fn from_arrow_buffer(buffer: arrow_buffer::Buffer) -> Self {
let ptr = buffer.as_ptr();
let align_offset = ptr.align_offset(std::mem::align_of::<T>());
let align_offset = ptr.align_offset(align_of::<T>());
assert_eq!(align_offset, 0, "arrow_buffer::Buffer misaligned");
let length = buffer.len() / std::mem::size_of::<T>();
let length = buffer.len() / size_of::<T>();

let inner = SharedStorageInner {
ref_count: AtomicU64::new(1),
Expand All @@ -119,7 +119,7 @@ impl<T: crate::types::NativeType> SharedStorage<T> {

pub fn into_arrow_buffer(self) -> arrow_buffer::Buffer {
let ptr = NonNull::new(self.as_ptr() as *mut u8).unwrap();
let len = self.len() * std::mem::size_of::<T>();
let len = self.len() * size_of::<T>();
let arc = std::sync::Arc::new(self);
unsafe { arrow_buffer::Buffer::from_custom_allocation(ptr, len, arc) }
}
Expand Down
2 changes: 1 addition & 1 deletion crates/polars-arrow/src/types/bit_chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ impl<T: BitChunk> BitChunkIter<T> {
/// Creates a new [`BitChunkIter`] with `len` bits.
#[inline]
pub fn new(value: T, len: usize) -> Self {
assert!(len <= std::mem::size_of::<T>() * 8);
assert!(len <= size_of::<T>() * 8);
Self {
value,
remaining: len,
Expand Down
2 changes: 1 addition & 1 deletion crates/polars-arrow/src/types/native.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ macro_rules! native_type {
impl NativeType for $type {
const PRIMITIVE: PrimitiveType = $primitive_type;

type Bytes = [u8; std::mem::size_of::<Self>()];
type Bytes = [u8; size_of::<Self>()];
#[inline]
fn to_le_bytes(&self) -> Self::Bytes {
Self::to_le_bytes(*self)
Expand Down
Loading