fix: Properly implement struct

crates/polars-arrow/src/array/static_array.rs

@@ -1,12 +1,13 @@
 use bytemuck::Zeroable;
+use polars_utils::no_call_const;
 
 use crate::array::binview::BinaryViewValueIter;
 use crate::array::growable::{Growable, GrowableFixedSizeList};
 use crate::array::static_array_collect::ArrayFromIterDtype;
 use crate::array::{
     Array, ArrayValuesIter, BinaryArray, BinaryValueIter, BinaryViewArray, BooleanArray,
     FixedSizeListArray, ListArray, ListValuesIter, MutableBinaryViewArray, PrimitiveArray,
-    Utf8Array, Utf8ValuesIter, Utf8ViewArray,
+    StructArray, Utf8Array, Utf8ValuesIter, Utf8ViewArray,
 };
 use crate::bitmap::utils::{BitmapIter, ZipValidity};
 use crate::bitmap::Bitmap;
@@ -64,15 +65,22 @@ pub trait StaticArray:
 
     /// # Safety
     /// It is the callers responsibility that the `idx < self.len()`.
-    unsafe fn value_unchecked(&self, idx: usize) -> Self::ValueT<'_>;
+    #[allow(unused_variables)]
+    unsafe fn value_unchecked(&self, idx: usize) -> Self::ValueT<'_> {
+        no_call_const!()
+    }
 
     #[inline(always)]
     fn as_slice(&self) -> Option<&[Self::ValueT<'_>]> {
         None
     }
 
-    fn iter(&self) -> ZipValidity<Self::ValueT<'_>, Self::ValueIterT<'_>, BitmapIter>;
-    fn values_iter(&self) -> Self::ValueIterT<'_>;
+    fn iter(&self) -> ZipValidity<Self::ValueT<'_>, Self::ValueIterT<'_>, BitmapIter> {
+        no_call_const!()
+    }
+    fn values_iter(&self) -> Self::ValueIterT<'_> {
+        no_call_const!()
+    }
     fn with_validity_typed(self, validity: Option<Bitmap>) -> Self;
 
     fn from_vec(v: Vec<Self::ValueT<'_>>, dtype: ArrowDataType) -> Self {
@@ -392,3 +400,17 @@ impl StaticArray for FixedSizeListArray {
         arr.into()
     }
 }
+
+impl StaticArray for StructArray {
+    type ValueT<'a> = ();
+    type ZeroableValueT<'a> = ();
+    type ValueIterT<'a> = std::iter::Repeat<()>;
+
+    fn with_validity_typed(self, validity: Option<Bitmap>) -> Self {
+        self.with_validity(validity)
+    }
+
+    fn full_null(length: usize, dtype: ArrowDataType) -> Self {
+        Self::new_null(dtype, length)
+    }
+}

crates/polars-arrow/src/array/static_array_collect.rs

@@ -1,11 +1,13 @@
 use std::borrow::Cow;
 use std::sync::Arc;
 
+use polars_utils::no_call_const;
+
 use crate::array::static_array::{ParameterFreeDtypeStaticArray, StaticArray};
 use crate::array::{
     Array, BinaryArray, BinaryViewArray, BooleanArray, FixedSizeListArray, ListArray,
     MutableBinaryArray, MutableBinaryValuesArray, MutableBinaryViewArray, PrimitiveArray,
-    Utf8Array, Utf8ViewArray,
+    StructArray, Utf8Array, Utf8ViewArray,
 };
 use crate::bitmap::Bitmap;
 use crate::datatypes::ArrowDataType;
@@ -1016,3 +1018,57 @@ impl ArrayFromIterDtype<Option<Box<dyn Array>>> for FixedSizeListArray {
         Ok(Self::arr_from_iter_with_dtype(dtype, iter_values))
     }
 }
+
+impl ArrayFromIter<Option<()>> for StructArray {
+    fn arr_from_iter<I: IntoIterator<Item = Option<()>>>(_iter: I) -> Self {
+        no_call_const!()
+    }
+
+    fn try_arr_from_iter<E, I: IntoIterator<Item = Result<Option<()>, E>>>(
+        _iter: I,
+    ) -> Result<Self, E> {
+        no_call_const!()
+    }
+}
+
+impl ArrayFromIter<()> for StructArray {
+    fn arr_from_iter<I: IntoIterator<Item = ()>>(_iter: I) -> Self {
+        no_call_const!()
+    }
+
+    fn try_arr_from_iter<E, I: IntoIterator<Item = Result<(), E>>>(_iter: I) -> Result<Self, E> {
+        no_call_const!()
+    }
+}
+
+impl ArrayFromIterDtype<()> for StructArray {
+    fn arr_from_iter_with_dtype<I: IntoIterator<Item = ()>>(
+        _dtype: ArrowDataType,
+        _iter: I,
+    ) -> Self {
+        no_call_const!()
+    }
+
+    fn try_arr_from_iter_with_dtype<E, I: IntoIterator<Item = Result<(), E>>>(
+        _dtype: ArrowDataType,
+        _iter: I,
+    ) -> Result<Self, E> {
+        no_call_const!()
+    }
+}
+
+impl ArrayFromIterDtype<Option<()>> for StructArray {
+    fn arr_from_iter_with_dtype<I: IntoIterator<Item = Option<()>>>(
+        _dtype: ArrowDataType,
+        _iter: I,
+    ) -> Self {
+        no_call_const!()
+    }
+
+    fn try_arr_from_iter_with_dtype<E, I: IntoIterator<Item = Result<Option<()>, E>>>(
+        _dtype: ArrowDataType,
+        _iter: I,
+    ) -> Result<Self, E> {
+        no_call_const!()
+    }
+}

crates/polars-arrow/src/array/struct_/mod.rs

@@ -11,6 +11,8 @@ mod mutable;
 pub use mutable::*;
 use polars_error::{polars_bail, PolarsResult};
 
+use crate::compute::utils::combine_validities_and;
+
 /// A [`StructArray`] is a nested [`Array`] with an optional validity representing
 /// multiple [`Array`] with the same number of rows.
 /// # Example
@@ -192,6 +194,27 @@ impl StructArray {
             .for_each(|x| x.slice_unchecked(offset, length));
     }
 
+    /// Set the outer nulls into the inner arrays, and clear the outer validity.
+    pub fn propagate_nulls(&self) -> StructArray {
+        let has_nulls = self.null_count() > 0;
+        let mut out = self.clone();
+        if !has_nulls {
+            return out;
+        };
+
+        for value_arr in &mut out.values {
+            let new = if has_nulls {
+                let new_validity = combine_validities_and(self.validity(), value_arr.validity());
+                value_arr.with_validity(new_validity)
+            } else {
+                value_arr.clone()
+            };
+
+            *value_arr = new;
+        }
+        out.with_validity(None)
+    }
+
     impl_sliced!();
 
     impl_mut_validity!();

crates/polars-core/src/chunked_array/cast.rs

@@ -125,7 +125,7 @@ fn cast_single_to_struct(
         new_fields.push(Series::full_null(&fld.name, length, &fld.dtype));
     }
 
-    Ok(StructChunked::new_unchecked(name, &new_fields).into_series())
+    StructChunked2::from_series(name, &new_fields).map(|ca| ca.into_series())
 }
 
 impl<T> ChunkedArray<T>

crates/polars-core/src/chunked_array/comparison/mod.rs

@@ -3,7 +3,7 @@ mod scalar;
 #[cfg(feature = "dtype-categorical")]
 mod categorical;
 
-use std::ops::Not;
+use std::ops::{BitAnd, Not};
 
 use arrow::array::BooleanArray;
 use arrow::bitmap::MutableBitmap;
@@ -14,6 +14,7 @@ use polars_compute::comparisons::{TotalEqKernel, TotalOrdKernel};
 use crate::prelude::*;
 use crate::series::implementations::null::NullChunked;
 use crate::series::IsSorted;
+use crate::utils::align_chunks_binary;
 
 impl<T> ChunkCompare<&ChunkedArray<T>> for ChunkedArray<T>
 where
@@ -643,77 +644,82 @@ impl ChunkCompare<&ListChunked> for ListChunked {
 }
 
 #[cfg(feature = "dtype-struct")]
-impl ChunkCompare<&StructChunked> for StructChunked {
-    type Item = BooleanChunked;
-    fn equal(&self, rhs: &StructChunked) -> BooleanChunked {
-        use std::ops::BitAnd;
-        if self.len() != rhs.len() || self.fields().len() != rhs.fields().len() {
-            BooleanChunked::full("", false, self.len())
-        } else {
-            self.fields()
-                .iter()
-                .zip(rhs.fields().iter())
-                .map(|(l, r)| l.equal(r).unwrap())
-                .reduce(|lhs, rhs| lhs.bitand(rhs))
-                .unwrap()
-        }
-    }
-
-    fn equal_missing(&self, rhs: &StructChunked) -> BooleanChunked {
-        use std::ops::BitAnd;
-        if self.len() != rhs.len() || self.fields().len() != rhs.fields().len() {
-            BooleanChunked::full("", false, self.len())
-        } else {
-            self.fields()
-                .iter()
-                .zip(rhs.fields().iter())
-                .map(|(l, r)| l.equal_missing(r).unwrap())
-                .reduce(|lhs, rhs| lhs.bitand(rhs))
-                .unwrap()
-        }
-    }
-
-    fn not_equal(&self, rhs: &StructChunked) -> BooleanChunked {
-        if self.len() != rhs.len() || self.fields().len() != rhs.fields().len() {
-            BooleanChunked::full("", true, self.len())
-        } else {
-            self.fields()
-                .iter()
-                .zip(rhs.fields().iter())
-                .map(|(l, r)| l.not_equal(r).unwrap())
-                .reduce(|lhs, rhs| lhs | rhs)
-                .unwrap()
-        }
-    }
-
-    fn not_equal_missing(&self, rhs: &StructChunked) -> BooleanChunked {
-        if self.len() != rhs.len() || self.fields().len() != rhs.fields().len() {
-            BooleanChunked::full("", true, self.len())
-        } else {
-            self.fields()
-                .iter()
-                .zip(rhs.fields().iter())
-                .map(|(l, r)| l.not_equal_missing(r).unwrap())
-                .reduce(|lhs, rhs| lhs | rhs)
-                .unwrap()
+fn struct_helper<F, R>(
+    a: &StructChunked2,
+    b: &StructChunked2,
+    op: F,
+    reduce: R,
+    value: bool,
+) -> BooleanChunked
+where
+    F: Fn(&Series, &Series) -> BooleanChunked,
+    R: Fn(BooleanChunked, BooleanChunked) -> BooleanChunked,
+{
+    if a.len() != b.len() || a.struct_fields().len() != b.struct_fields().len() {
+        BooleanChunked::full("", value, a.len())
+    } else {
+        let (a, b) = align_chunks_binary(a, b);
+        let mut out = a
+            .fields_as_series()
+            .iter()
+            .zip(b.fields_as_series().iter())
+            .map(|(l, r)| op(l, r))
+            .reduce(reduce)
+            .unwrap();
+        if a.null_count() > 0 || b.null_count() > 0 {
+            let mut a = a.into_owned();
+            a.zip_outer_validity(&b);
+            unsafe {
+                for (arr, a) in out.downcast_iter_mut().zip(a.downcast_iter()) {
+                    arr.set_validity(a.validity().cloned())
+                }
+            }
         }
+        out
     }
+}
 
-    // following are not implemented because gt, lt comparison of series don't make sense
-    fn gt(&self, _rhs: &StructChunked) -> BooleanChunked {
-        unimplemented!()
-    }
-
-    fn gt_eq(&self, _rhs: &StructChunked) -> BooleanChunked {
-        unimplemented!()
-    }
-
-    fn lt(&self, _rhs: &StructChunked) -> BooleanChunked {
-        unimplemented!()
-    }
-
-    fn lt_eq(&self, _rhs: &StructChunked) -> BooleanChunked {
-        unimplemented!()
+#[cfg(feature = "dtype-struct")]
+impl ChunkCompare<&StructChunked2> for StructChunked2 {
+    type Item = BooleanChunked;
+    fn equal(&self, rhs: &StructChunked2) -> BooleanChunked {
+        struct_helper(
+            self,
+            rhs,
+            |l, r| l.equal(r).unwrap(),
+            |a, b| a.bitand(b),
+            false,
+        )
+    }
+
+    fn equal_missing(&self, rhs: &StructChunked2) -> BooleanChunked {
+        struct_helper(
+            self,
+            rhs,
+            |l, r| l.equal_missing(r).unwrap(),
+            |a, b| a.bitand(b),
+            false,
+        )
+    }
+
+    fn not_equal(&self, rhs: &StructChunked2) -> BooleanChunked {
+        struct_helper(
+            self,
+            rhs,
+            |l, r| l.not_equal(r).unwrap(),
+            |a, b| a.not_equal(&b).unique().unwrap(),
+            true,
+        )
+    }
+
+    fn not_equal_missing(&self, rhs: &StructChunked2) -> BooleanChunked {
+        struct_helper(
+            self,
+            rhs,
+            |l, r| l.not_equal_missing(r).unwrap(),
+            |a, b| a.not_equal_missing(&b).unique().unwrap(),
+            true,
+        )
     }
 }
 

crates/polars-core/src/chunked_array/from.rs

@@ -192,6 +192,8 @@ where
             dtype @ DataType::List(_) => from_chunks_list_dtype(&mut chunks, dtype),
             #[cfg(feature = "dtype-array")]
             dtype @ DataType::Array(_, _) => from_chunks_list_dtype(&mut chunks, dtype),
+            #[cfg(feature = "dtype-struct")]
+            dtype @ DataType::Struct(_) => from_chunks_list_dtype(&mut chunks, dtype),
             dt => dt,
         };
         Self::from_chunks_and_dtype(name, chunks, dtype)