Skip to main content

polars_core/utils/
mod.rs

1mod any_value;
2pub mod cut;
3use polars_arrow::compute::concatenate::concatenate_validities;
4use polars_arrow::compute::utils::combine_validities_and;
5pub mod flatten;
6pub(crate) mod series;
7mod supertype;
8use std::borrow::Cow;
9use std::ops::{Deref, DerefMut};
10mod schema;
11
12pub use any_value::*;
13use flatten::*;
14use num_traits::{One, Zero};
15pub use polars_arrow;
16use polars_arrow::bitmap::Bitmap;
17pub use polars_arrow::legacy::utils::*;
18pub use polars_arrow::trusted_len::TrustMyLength;
19use polars_utils::slice_enum::Slice;
20pub use rayon;
21use rayon::prelude::*;
22pub use schema::*;
23pub use series::*;
24pub use supertype::*;
25
26use crate::prelude::*;
27use crate::runtime::RAYON;
28
29#[repr(transparent)]
30pub struct Wrap<T>(pub T);
31
32impl<T> Deref for Wrap<T> {
33    type Target = T;
34    fn deref(&self) -> &Self::Target {
35        &self.0
36    }
37}
38
39#[inline(always)]
40pub fn _set_partition_size() -> usize {
41    RAYON.current_num_threads()
42}
43
44/// Iterate `items` in parallel with the number of rayon tasks bounded by the thread count.
45///
46/// Use this when the length of `items` grows with the data. A worker adds a stack frame per
47/// stolen job, so an unbounded number of tasks can overflow a worker stack.
48pub fn par_iter_bounded<T: Sync>(items: &[T]) -> rayon::iter::MinLen<rayon::slice::Iter<'_, T>> {
49    const TASKS_PER_THREAD: usize = 8;
50    let min_len = items
51        .len()
52        .div_ceil(_set_partition_size() * TASKS_PER_THREAD);
53    items.par_iter().with_min_len(min_len.max(1))
54}
55
56/// Just a wrapper structure which is useful for certain impl specializations.
57///
58/// This is for instance used to implement
59/// `impl<T> FromIterator<T::Native> for NoNull<ChunkedArray<T>>`
60/// as `Option<T::Native>` was already implemented:
61/// `impl<T> FromIterator<Option<T::Native>> for ChunkedArray<T>`
62pub struct NoNull<T> {
63    inner: T,
64}
65
66impl<T> NoNull<T> {
67    pub fn new(inner: T) -> Self {
68        NoNull { inner }
69    }
70
71    pub fn into_inner(self) -> T {
72        self.inner
73    }
74}
75
76impl<T> Deref for NoNull<T> {
77    type Target = T;
78
79    fn deref(&self) -> &Self::Target {
80        &self.inner
81    }
82}
83
84impl<T> DerefMut for NoNull<T> {
85    fn deref_mut(&mut self) -> &mut Self::Target {
86        &mut self.inner
87    }
88}
89
90pub(crate) fn get_iter_capacity<T, I: Iterator<Item = T>>(iter: &I) -> usize {
91    match iter.size_hint() {
92        (_lower, Some(upper)) => upper,
93        (0, None) => 1024,
94        (lower, None) => lower,
95    }
96}
97
98// prefer this one over split_ca, as this can push the null_count into the thread pool
99// returns an `(offset, length)` tuple
100#[doc(hidden)]
101pub fn _split_offsets(len: usize, n: usize) -> Vec<(usize, usize)> {
102    if n == 1 {
103        vec![(0, len)]
104    } else {
105        let chunk_size = len / n;
106
107        (0..n)
108            .map(|partition| {
109                let offset = partition * chunk_size;
110                let len = if partition == (n - 1) {
111                    len - offset
112                } else {
113                    chunk_size
114                };
115                (partition * chunk_size, len)
116            })
117            .collect_trusted()
118    }
119}
120
121#[allow(clippy::len_without_is_empty)]
122pub trait Container: Clone {
123    fn slice(&self, slice: impl Into<Slice>) -> Self;
124
125    fn split_at(&self, offset: i64) -> (Self, Self);
126
127    fn len(&self) -> usize;
128
129    fn iter_chunks(&self) -> impl Iterator<Item = Self>;
130
131    fn should_rechunk(&self) -> bool;
132
133    fn n_chunks(&self) -> usize;
134
135    fn chunk_lengths(&self) -> impl Iterator<Item = usize>;
136}
137
138impl Container for DataFrame {
139    fn slice(&self, slice: impl Into<Slice>) -> Self {
140        DataFrame::slice(self, slice)
141    }
142
143    fn split_at(&self, offset: i64) -> (Self, Self) {
144        DataFrame::split_at(self, offset)
145    }
146
147    fn len(&self) -> usize {
148        self.height()
149    }
150
151    fn iter_chunks(&self) -> impl Iterator<Item = Self> {
152        flatten_df_iter(self)
153    }
154
155    fn should_rechunk(&self) -> bool {
156        self.should_rechunk()
157    }
158
159    fn n_chunks(&self) -> usize {
160        DataFrame::first_col_n_chunks(self)
161    }
162
163    fn chunk_lengths(&self) -> impl Iterator<Item = usize> {
164        self.first_col_chunk_lengths()
165    }
166}
167
168impl<T: PolarsDataType> Container for ChunkedArray<T> {
169    fn slice(&self, slice: impl Into<Slice>) -> Self {
170        ChunkedArray::slice(self, slice)
171    }
172
173    fn split_at(&self, offset: i64) -> (Self, Self) {
174        ChunkedArray::split_at(self, offset)
175    }
176
177    fn len(&self) -> usize {
178        ChunkedArray::len(self)
179    }
180
181    fn iter_chunks(&self) -> impl Iterator<Item = Self> {
182        self.downcast_iter()
183            .map(|arr| Self::with_chunk(self.name().clone(), arr.clone()))
184    }
185
186    fn should_rechunk(&self) -> bool {
187        false
188    }
189
190    fn n_chunks(&self) -> usize {
191        self.chunks().len()
192    }
193
194    fn chunk_lengths(&self) -> impl Iterator<Item = usize> {
195        ChunkedArray::chunk_lengths(self)
196    }
197}
198
199impl Container for Series {
200    fn slice(&self, slice: impl Into<Slice>) -> Self {
201        Series::slice(self, slice)
202    }
203
204    fn split_at(&self, offset: i64) -> (Self, Self) {
205        self.0.split_at(offset)
206    }
207
208    fn len(&self) -> usize {
209        self.0.len()
210    }
211
212    fn iter_chunks(&self) -> impl Iterator<Item = Self> {
213        (0..self.0.n_chunks()).map(|i| self.select_chunk(i))
214    }
215
216    fn should_rechunk(&self) -> bool {
217        false
218    }
219
220    fn n_chunks(&self) -> usize {
221        self.chunks().len()
222    }
223
224    fn chunk_lengths(&self) -> impl Iterator<Item = usize> {
225        self.0.chunk_lengths()
226    }
227}
228
229fn split_impl<C: Container>(container: &C, target: usize, chunk_size: usize) -> Vec<C> {
230    if target == 1 {
231        return vec![container.clone()];
232    }
233    let mut out = Vec::with_capacity(target);
234    let chunk_size = chunk_size as i64;
235
236    // First split
237    let (chunk, mut remainder) = container.split_at(chunk_size);
238    out.push(chunk);
239
240    // Take the rest of the splits of exactly chunk size, but skip the last remainder as we won't split that.
241    for _ in 1..target - 1 {
242        let (a, b) = remainder.split_at(chunk_size);
243        out.push(a);
244        remainder = b
245    }
246    // This can be slightly larger than `chunk_size`, but is smaller than `2 * chunk_size`.
247    out.push(remainder);
248    out
249}
250
251/// Splits, but doesn't flatten chunks. E.g. a container can still have multiple chunks.
252pub fn split<C: Container>(container: &C, target: usize) -> Vec<C> {
253    let total_len = container.len();
254    if total_len == 0 {
255        return vec![container.clone()];
256    }
257
258    let chunk_size = std::cmp::max(total_len / target, 1);
259
260    if container.n_chunks() == target
261        && container
262            .chunk_lengths()
263            .all(|len| len.abs_diff(chunk_size) < 100)
264        // We cannot get chunks if they are misaligned
265        && !container.should_rechunk()
266    {
267        return container.iter_chunks().collect();
268    }
269    split_impl(container, target, chunk_size)
270}
271
272/// Split a [`Container`] in `target` elements. The target doesn't have to be respected if not
273/// Deviation of the target might be done to create more equal size chunks.
274pub fn split_and_flatten<C: Container>(container: &C, target: usize) -> Vec<C> {
275    let total_len = container.len();
276    if total_len == 0 {
277        return vec![container.clone()];
278    }
279
280    let chunk_size = std::cmp::max(total_len / target, 1);
281
282    if container.n_chunks() == target
283        && container
284            .chunk_lengths()
285            .all(|len| len.abs_diff(chunk_size) < 100)
286        // We cannot get chunks if they are misaligned
287        && !container.should_rechunk()
288    {
289        return container.iter_chunks().collect();
290    }
291
292    if container.n_chunks() == 1 {
293        split_impl(container, target, chunk_size)
294    } else {
295        let mut out = Vec::with_capacity(target);
296        let chunks = container.iter_chunks();
297
298        'new_chunk: for mut chunk in chunks {
299            loop {
300                let h = chunk.len();
301                if h < chunk_size {
302                    // TODO if the chunk is much smaller than chunk size, we should try to merge it with the next one.
303                    out.push(chunk);
304                    continue 'new_chunk;
305                }
306
307                // If a split leads to the next chunk being smaller than 30% take the whole chunk
308                if ((h - chunk_size) as f64 / chunk_size as f64) < 0.3 {
309                    out.push(chunk);
310                    continue 'new_chunk;
311                }
312
313                let (a, b) = chunk.split_at(chunk_size as i64);
314                out.push(a);
315                chunk = b;
316            }
317        }
318        out
319    }
320}
321
322/// Split a [`DataFrame`] in `target` elements. The target doesn't have to be respected if not
323/// strict. Deviation of the target might be done to create more equal size chunks.
324///
325/// # Panics
326/// if chunks are not aligned
327pub fn split_df_as_ref(df: &DataFrame, target: usize, strict: bool) -> Vec<DataFrame> {
328    if strict {
329        split(df, target)
330    } else {
331        split_and_flatten(df, target)
332    }
333}
334
335#[doc(hidden)]
336/// Split a [`DataFrame`] into `n` parts. We take a `&mut` to be able to repartition/align chunks.
337/// `strict` in that it respects `n` even if the chunks are suboptimal.
338pub fn split_df(df: &mut DataFrame, target: usize, strict: bool) -> Vec<DataFrame> {
339    if target == 0 || df.height() == 0 {
340        return vec![df.clone()];
341    }
342    // make sure that chunks are aligned.
343    df.align_chunks_par();
344    split_df_as_ref(df, target, strict)
345}
346
347/// Apply a macro on the Series
348#[macro_export]
349macro_rules! match_dtype_to_physical_apply_macro {
350    ($obj:expr, $macro:ident, $macro_string:ident, $macro_bool:ident $(, $opt_args:expr)*) => {{
351        match $obj {
352            DataType::String => $macro_string!($($opt_args)*),
353            DataType::Boolean => $macro_bool!($($opt_args)*),
354            #[cfg(feature = "dtype-u8")]
355            DataType::UInt8 => $macro!(u8 $(, $opt_args)*),
356            #[cfg(feature = "dtype-u16")]
357            DataType::UInt16 => $macro!(u16 $(, $opt_args)*),
358            DataType::UInt32 => $macro!(u32 $(, $opt_args)*),
359            DataType::UInt64 => $macro!(u64 $(, $opt_args)*),
360            #[cfg(feature = "dtype-i8")]
361            DataType::Int8 => $macro!(i8 $(, $opt_args)*),
362            #[cfg(feature = "dtype-i16")]
363            DataType::Int16 => $macro!(i16 $(, $opt_args)*),
364            DataType::Int32 => $macro!(i32 $(, $opt_args)*),
365            DataType::Int64 => $macro!(i64 $(, $opt_args)*),
366            #[cfg(feature = "dtype-i128")]
367            DataType::Int128 => $macro!(i128 $(, $opt_args)*),
368            #[cfg(feature = "dtype-f16")]
369            DataType::Float16 => $macro!(pf16 $(, $opt_args)*),
370            DataType::Float32 => $macro!(f32 $(, $opt_args)*),
371            DataType::Float64 => $macro!(f64 $(, $opt_args)*),
372            dt => panic!("not implemented for dtype {:?}", dt),
373        }
374    }};
375}
376
377/// Apply a macro on the Series
378#[macro_export]
379macro_rules! match_dtype_to_logical_apply_macro {
380    ($obj:expr, $macro:ident, $macro_string:ident, $macro_binary:ident, $macro_bool:ident $(, $opt_args:expr)*) => {{
381        match $obj {
382            DataType::String => $macro_string!($($opt_args)*),
383            DataType::Binary => $macro_binary!($($opt_args)*),
384            DataType::Boolean => $macro_bool!($($opt_args)*),
385            #[cfg(feature = "dtype-u8")]
386            DataType::UInt8 => $macro!(UInt8Type $(, $opt_args)*),
387            #[cfg(feature = "dtype-u16")]
388            DataType::UInt16 => $macro!(UInt16Type $(, $opt_args)*),
389            DataType::UInt32 => $macro!(UInt32Type $(, $opt_args)*),
390            DataType::UInt64 => $macro!(UInt64Type $(, $opt_args)*),
391            #[cfg(feature = "dtype-u128")]
392            DataType::UInt128 => $macro!(UInt128Type $(, $opt_args)*),
393            #[cfg(feature = "dtype-i8")]
394            DataType::Int8 => $macro!(Int8Type $(, $opt_args)*),
395            #[cfg(feature = "dtype-i16")]
396            DataType::Int16 => $macro!(Int16Type $(, $opt_args)*),
397            DataType::Int32 => $macro!(Int32Type $(, $opt_args)*),
398            DataType::Int64 => $macro!(Int64Type $(, $opt_args)*),
399            #[cfg(feature = "dtype-i128")]
400            DataType::Int128 => $macro!(Int128Type $(, $opt_args)*),
401            #[cfg(feature = "dtype-f16")]
402            DataType::Float16 => $macro!(Float16Type $(, $opt_args)*),
403            DataType::Float32 => $macro!(Float32Type $(, $opt_args)*),
404            DataType::Float64 => $macro!(Float64Type $(, $opt_args)*),
405            dt => panic!("not implemented for dtype {:?}", dt),
406        }
407    }};
408}
409
410/// Apply a macro on the Downcasted ChunkedArrays
411#[macro_export]
412macro_rules! match_arrow_dtype_apply_macro_ca {
413    ($self:expr, $macro:ident, $macro_string:ident, $macro_bool:ident $(, $opt_args:expr)*) => {{
414        match $self.dtype() {
415            DataType::String => $macro_string!($self.str().unwrap() $(, $opt_args)*),
416            DataType::Boolean => $macro_bool!($self.bool().unwrap() $(, $opt_args)*),
417            #[cfg(feature = "dtype-u8")]
418            DataType::UInt8 => $macro!($self.u8().unwrap() $(, $opt_args)*),
419            #[cfg(feature = "dtype-u16")]
420            DataType::UInt16 => $macro!($self.u16().unwrap() $(, $opt_args)*),
421            DataType::UInt32 => $macro!($self.u32().unwrap() $(, $opt_args)*),
422            DataType::UInt64 => $macro!($self.u64().unwrap() $(, $opt_args)*),
423            #[cfg(feature = "dtype-u128")]
424            DataType::UInt128 => $macro!($self.u128().unwrap() $(, $opt_args)*),
425            #[cfg(feature = "dtype-i8")]
426            DataType::Int8 => $macro!($self.i8().unwrap() $(, $opt_args)*),
427            #[cfg(feature = "dtype-i16")]
428            DataType::Int16 => $macro!($self.i16().unwrap() $(, $opt_args)*),
429            DataType::Int32 => $macro!($self.i32().unwrap() $(, $opt_args)*),
430            DataType::Int64 => $macro!($self.i64().unwrap() $(, $opt_args)*),
431            #[cfg(feature = "dtype-i128")]
432            DataType::Int128 => $macro!($self.i128().unwrap() $(, $opt_args)*),
433            #[cfg(feature = "dtype-f16")]
434            DataType::Float16 => $macro!($self.f16().unwrap() $(, $opt_args)*),
435            DataType::Float32 => $macro!($self.f32().unwrap() $(, $opt_args)*),
436            DataType::Float64 => $macro!($self.f64().unwrap() $(, $opt_args)*),
437            dt => panic!("not implemented for dtype {:?}", dt),
438        }
439    }};
440}
441
442#[macro_export]
443macro_rules! with_match_physical_numeric_type {(
444    $dtype:expr, | $_:tt $T:ident | $($body:tt)*
445) => ({
446    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
447    #[cfg(feature = "dtype-f16")]
448    use polars_utils::float16::pf16;
449    use $crate::datatypes::DataType::*;
450    match $dtype {
451        #[cfg(feature = "dtype-i8")]
452        Int8 => __with_ty__! { i8 },
453        #[cfg(feature = "dtype-i16")]
454        Int16 => __with_ty__! { i16 },
455        Int32 => __with_ty__! { i32 },
456        Int64 => __with_ty__! { i64 },
457        #[cfg(feature = "dtype-i128")]
458        Int128 => __with_ty__! { i128 },
459        #[cfg(feature = "dtype-u8")]
460        UInt8 => __with_ty__! { u8 },
461        #[cfg(feature = "dtype-u16")]
462        UInt16 => __with_ty__! { u16 },
463        UInt32 => __with_ty__! { u32 },
464        UInt64 => __with_ty__! { u64 },
465        #[cfg(feature = "dtype-u128")]
466        UInt128 => __with_ty__! { u128 },
467        #[cfg(feature = "dtype-f16")]
468        Float16 => __with_ty__! { pf16 },
469        Float32 => __with_ty__! { f32 },
470        Float64 => __with_ty__! { f64 },
471        dt => panic!("not implemented for dtype {:?}", dt),
472    }
473})}
474
475#[macro_export]
476macro_rules! with_match_physical_integer_type {(
477    $dtype:expr, | $_:tt $T:ident | $($body:tt)*
478) => ({
479    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
480    #[cfg(feature = "dtype-f16")]
481    use polars_utils::float16::pf16;
482    use $crate::datatypes::DataType::*;
483    match $dtype {
484        #[cfg(feature = "dtype-i8")]
485        Int8 => __with_ty__! { i8 },
486        #[cfg(feature = "dtype-i16")]
487        Int16 => __with_ty__! { i16 },
488        Int32 => __with_ty__! { i32 },
489        Int64 => __with_ty__! { i64 },
490        #[cfg(feature = "dtype-i128")]
491        Int128 => __with_ty__! { i128 },
492        #[cfg(feature = "dtype-u8")]
493        UInt8 => __with_ty__! { u8 },
494        #[cfg(feature = "dtype-u16")]
495        UInt16 => __with_ty__! { u16 },
496        UInt32 => __with_ty__! { u32 },
497        UInt64 => __with_ty__! { u64 },
498        #[cfg(feature = "dtype-u128")]
499        UInt128 => __with_ty__! { u128 },
500        dt => panic!("not implemented for dtype {:?}", dt),
501    }
502})}
503
504#[macro_export]
505macro_rules! with_match_physical_float_type {(
506    $dtype:expr, | $_:tt $T:ident | $($body:tt)*
507) => ({
508    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
509    use polars_utils::float16::pf16;
510    use $crate::datatypes::DataType::*;
511    match $dtype {
512        #[cfg(feature = "dtype-f16")]
513        Float16 => __with_ty__! { pf16 },
514        Float32 => __with_ty__! { f32 },
515        Float64 => __with_ty__! { f64 },
516        dt => panic!("not implemented for dtype {:?}", dt),
517    }
518})}
519
520#[macro_export]
521macro_rules! with_match_physical_float_polars_type {(
522    $key_type:expr, | $_:tt $T:ident | $($body:tt)*
523) => ({
524    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
525    use $crate::datatypes::DataType::*;
526    match $key_type {
527        #[cfg(feature = "dtype-f16")]
528        Float16 => __with_ty__! { Float16Type },
529        Float32 => __with_ty__! { Float32Type },
530        Float64 => __with_ty__! { Float64Type },
531        dt => panic!("not implemented for dtype {:?}", dt),
532    }
533})}
534
535#[macro_export]
536macro_rules! with_match_physical_numeric_polars_type {(
537    $key_type:expr, | $_:tt $T:ident | $($body:tt)*
538) => ({
539    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
540    use $crate::datatypes::DataType::*;
541    match $key_type {
542            #[cfg(feature = "dtype-i8")]
543        Int8 => __with_ty__! { Int8Type },
544            #[cfg(feature = "dtype-i16")]
545        Int16 => __with_ty__! { Int16Type },
546        Int32 => __with_ty__! { Int32Type },
547        Int64 => __with_ty__! { Int64Type },
548            #[cfg(feature = "dtype-i128")]
549        Int128 => __with_ty__! { Int128Type },
550            #[cfg(feature = "dtype-u8")]
551        UInt8 => __with_ty__! { UInt8Type },
552            #[cfg(feature = "dtype-u16")]
553        UInt16 => __with_ty__! { UInt16Type },
554        UInt32 => __with_ty__! { UInt32Type },
555        UInt64 => __with_ty__! { UInt64Type },
556            #[cfg(feature = "dtype-u128")]
557        UInt128 => __with_ty__! { UInt128Type },
558            #[cfg(feature = "dtype-f16")]
559        Float16 => __with_ty__! { Float16Type },
560        Float32 => __with_ty__! { Float32Type },
561        Float64 => __with_ty__! { Float64Type },
562        dt => panic!("not implemented for dtype {:?}", dt),
563    }
564})}
565
566#[macro_export]
567macro_rules! with_match_physical_integer_polars_type {(
568    $key_type:expr, | $_:tt $T:ident | $($body:tt)*
569) => ({
570    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
571    use $crate::datatypes::DataType::*;
572    use $crate::datatypes::*;
573    match $key_type {
574        #[cfg(feature = "dtype-i8")]
575        Int8 => __with_ty__! { Int8Type },
576        #[cfg(feature = "dtype-i16")]
577        Int16 => __with_ty__! { Int16Type },
578        Int32 => __with_ty__! { Int32Type },
579        Int64 => __with_ty__! { Int64Type },
580        #[cfg(feature = "dtype-i128")]
581        Int128 => __with_ty__! { Int128Type },
582        #[cfg(feature = "dtype-u8")]
583        UInt8 => __with_ty__! { UInt8Type },
584        #[cfg(feature = "dtype-u16")]
585        UInt16 => __with_ty__! { UInt16Type },
586        UInt32 => __with_ty__! { UInt32Type },
587        UInt64 => __with_ty__! { UInt64Type },
588        #[cfg(feature = "dtype-u128")]
589        UInt128 => __with_ty__! { UInt128Type },
590        dt => panic!("not implemented for dtype {:?}", dt),
591    }
592})}
593
594#[macro_export]
595macro_rules! with_match_categorical_physical_type {(
596    $dtype:expr, | $_:tt $T:ident | $($body:tt)*
597) => ({
598    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
599    match $dtype {
600        CategoricalPhysical::U8 => __with_ty__! { Categorical8Type },
601        CategoricalPhysical::U16 => __with_ty__! { Categorical16Type },
602        CategoricalPhysical::U32 => __with_ty__! { Categorical32Type },
603    }
604})}
605
606/// Apply a macro on the Downcasted ChunkedArrays of DataTypes that are logical numerics.
607/// So no logical.
608#[macro_export]
609macro_rules! downcast_as_macro_arg_physical {
610    ($self:expr, $macro:ident $(, $opt_args:expr)*) => {{
611        match $self.dtype() {
612            #[cfg(feature = "dtype-u8")]
613            DataType::UInt8 => $macro!($self.u8().unwrap() $(, $opt_args)*),
614            #[cfg(feature = "dtype-u16")]
615            DataType::UInt16 => $macro!($self.u16().unwrap() $(, $opt_args)*),
616            DataType::UInt32 => $macro!($self.u32().unwrap() $(, $opt_args)*),
617            DataType::UInt64 => $macro!($self.u64().unwrap() $(, $opt_args)*),
618            #[cfg(feature = "dtype-u128")]
619            DataType::UInt128 => $macro!($self.u128().unwrap() $(, $opt_args)*),
620            #[cfg(feature = "dtype-i8")]
621            DataType::Int8 => $macro!($self.i8().unwrap() $(, $opt_args)*),
622            #[cfg(feature = "dtype-i16")]
623            DataType::Int16 => $macro!($self.i16().unwrap() $(, $opt_args)*),
624            DataType::Int32 => $macro!($self.i32().unwrap() $(, $opt_args)*),
625            DataType::Int64 => $macro!($self.i64().unwrap() $(, $opt_args)*),
626            #[cfg(feature = "dtype-i128")]
627            DataType::Int128 => $macro!($self.i128().unwrap() $(, $opt_args)*),
628            #[cfg(feature = "dtype-f16")]
629            DataType::Float16 => $macro!($self.f16().unwrap() $(, $opt_args)*),
630            DataType::Float32 => $macro!($self.f32().unwrap() $(, $opt_args)*),
631            DataType::Float64 => $macro!($self.f64().unwrap() $(, $opt_args)*),
632            dt => panic!("not implemented for {:?}", dt),
633        }
634    }};
635}
636
637/// Apply a macro on the Downcasted ChunkedArrays of DataTypes that are logical numerics.
638/// So no logical.
639#[macro_export]
640macro_rules! downcast_as_macro_arg_physical_mut {
641    ($self:expr, $macro:ident $(, $opt_args:expr)*) => {{
642        // clone so that we do not borrow
643        match $self.dtype().clone() {
644            #[cfg(feature = "dtype-u8")]
645            DataType::UInt8 => {
646                let ca: &mut UInt8Chunked = $self.as_mut();
647                $macro!(UInt8Type, ca $(, $opt_args)*)
648            },
649            #[cfg(feature = "dtype-u16")]
650            DataType::UInt16 => {
651                let ca: &mut UInt16Chunked = $self.as_mut();
652                $macro!(UInt16Type, ca $(, $opt_args)*)
653            },
654            DataType::UInt32 => {
655                let ca: &mut UInt32Chunked = $self.as_mut();
656                $macro!(UInt32Type, ca $(, $opt_args)*)
657            },
658            DataType::UInt64 => {
659                let ca: &mut UInt64Chunked = $self.as_mut();
660                $macro!(UInt64Type, ca $(, $opt_args)*)
661            },
662            #[cfg(feature = "dtype-u128")]
663            DataType::UInt128 => {
664                let ca: &mut UInt128Chunked = $self.as_mut();
665                $macro!(UInt128Type, ca $(, $opt_args)*)
666            },
667            #[cfg(feature = "dtype-i8")]
668            DataType::Int8 => {
669                let ca: &mut Int8Chunked = $self.as_mut();
670                $macro!(Int8Type, ca $(, $opt_args)*)
671            },
672            #[cfg(feature = "dtype-i16")]
673            DataType::Int16 => {
674                let ca: &mut Int16Chunked = $self.as_mut();
675                $macro!(Int16Type, ca $(, $opt_args)*)
676            },
677            DataType::Int32 => {
678                let ca: &mut Int32Chunked = $self.as_mut();
679                $macro!(Int32Type, ca $(, $opt_args)*)
680            },
681            DataType::Int64 => {
682                let ca: &mut Int64Chunked = $self.as_mut();
683                $macro!(Int64Type, ca $(, $opt_args)*)
684            },
685            #[cfg(feature = "dtype-i128")]
686            DataType::Int128 => {
687                let ca: &mut Int128Chunked = $self.as_mut();
688                $macro!(Int128Type, ca $(, $opt_args)*)
689            },
690            #[cfg(feature = "dtype-f16")]
691            DataType::Float16 => {
692                let ca: &mut Float16Chunked = $self.as_mut();
693                $macro!(Float16Type, ca $(, $opt_args)*)
694            },
695            DataType::Float32 => {
696                let ca: &mut Float32Chunked = $self.as_mut();
697                $macro!(Float32Type, ca $(, $opt_args)*)
698            },
699            DataType::Float64 => {
700                let ca: &mut Float64Chunked = $self.as_mut();
701                $macro!(Float64Type, ca $(, $opt_args)*)
702            },
703            dt => panic!("not implemented for {:?}", dt),
704        }
705    }};
706}
707
708#[macro_export]
709macro_rules! apply_method_all_arrow_series {
710    ($self:expr, $method:ident, $($args:expr),*) => {
711        match $self.dtype() {
712            DataType::Boolean => $self.bool().unwrap().$method($($args),*),
713            DataType::String => $self.str().unwrap().$method($($args),*),
714            #[cfg(feature = "dtype-u8")]
715            DataType::UInt8 => $self.u8().unwrap().$method($($args),*),
716            #[cfg(feature = "dtype-u16")]
717            DataType::UInt16 => $self.u16().unwrap().$method($($args),*),
718            DataType::UInt32 => $self.u32().unwrap().$method($($args),*),
719            DataType::UInt64 => $self.u64().unwrap().$method($($args),*),
720            #[cfg(feature = "dtype-u128")]
721            DataType::UInt128 => $self.u128().unwrap().$method($($args),*),
722            #[cfg(feature = "dtype-i8")]
723            DataType::Int8 => $self.i8().unwrap().$method($($args),*),
724            #[cfg(feature = "dtype-i16")]
725            DataType::Int16 => $self.i16().unwrap().$method($($args),*),
726            DataType::Int32 => $self.i32().unwrap().$method($($args),*),
727            DataType::Int64 => $self.i64().unwrap().$method($($args),*),
728            #[cfg(feature = "dtype-i128")]
729            DataType::Int128 => $self.i128().unwrap().$method($($args),*),
730            #[cfg(feature = "dtype-f16")]
731            DataType::Float16 => $self.f16().unwrap().$method($($args),*),
732            DataType::Float32 => $self.f32().unwrap().$method($($args),*),
733            DataType::Float64 => $self.f64().unwrap().$method($($args),*),
734            DataType::Time => $self.time().unwrap().$method($($args),*),
735            DataType::Date => $self.date().unwrap().$method($($args),*),
736            DataType::Datetime(_, _) => $self.datetime().unwrap().$method($($args),*),
737            DataType::List(_) => $self.list().unwrap().$method($($args),*),
738            DataType::Struct(_) => $self.struct_().unwrap().$method($($args),*),
739            dt => panic!("dtype {:?} not supported", dt)
740        }
741    }
742}
743
744#[macro_export]
745macro_rules! apply_method_physical_integer {
746    ($self:expr, $method:ident, $($args:expr),*) => {
747        match $self.dtype() {
748            #[cfg(feature = "dtype-u8")]
749            DataType::UInt8 => $self.u8().unwrap().$method($($args),*),
750            #[cfg(feature = "dtype-u16")]
751            DataType::UInt16 => $self.u16().unwrap().$method($($args),*),
752            DataType::UInt32 => $self.u32().unwrap().$method($($args),*),
753            DataType::UInt64 => $self.u64().unwrap().$method($($args),*),
754            #[cfg(feature = "dtype-u128")]
755            DataType::UInt128 => $self.u128().unwrap().$method($($args),*),
756            #[cfg(feature = "dtype-i8")]
757            DataType::Int8 => $self.i8().unwrap().$method($($args),*),
758            #[cfg(feature = "dtype-i16")]
759            DataType::Int16 => $self.i16().unwrap().$method($($args),*),
760            DataType::Int32 => $self.i32().unwrap().$method($($args),*),
761            DataType::Int64 => $self.i64().unwrap().$method($($args),*),
762            #[cfg(feature = "dtype-i128")]
763            DataType::Int128 => $self.i128().unwrap().$method($($args),*),
764            dt => panic!("not implemented for dtype {:?}", dt),
765        }
766    }
767}
768
769// doesn't include Bool and String
770#[macro_export]
771macro_rules! apply_method_physical_numeric {
772    ($self:expr, $method:ident, $($args:expr),*) => {
773        match $self.dtype() {
774            #[cfg(feature = "dtype-f16")]
775            DataType::Float16 => $self.f16().unwrap().$method($($args),*),
776            DataType::Float32 => $self.f32().unwrap().$method($($args),*),
777            DataType::Float64 => $self.f64().unwrap().$method($($args),*),
778            _ => apply_method_physical_integer!($self, $method, $($args),*),
779        }
780    }
781}
782
783#[macro_export]
784macro_rules! df {
785    ($($col_name:expr => $slice:expr), + $(,)?) => {
786        $crate::prelude::DataFrame::new_infer_height(vec![
787            $($crate::prelude::Column::from(<$crate::prelude::Series as $crate::prelude::NamedFrom::<_, _>>::new($col_name.into(), $slice)),)+
788        ])
789    }
790}
791
792pub fn get_time_units(tu_l: &TimeUnit, tu_r: &TimeUnit) -> TimeUnit {
793    use crate::datatypes::time_unit::TimeUnit::*;
794    match (tu_l, tu_r) {
795        (Nanoseconds, Microseconds) => Microseconds,
796        (_, Milliseconds) => Milliseconds,
797        _ => *tu_l,
798    }
799}
800
801#[cold]
802#[inline(never)]
803fn width_mismatch(df1: &DataFrame, df2: &DataFrame) -> PolarsError {
804    let mut df1_extra = Vec::new();
805    let mut df2_extra = Vec::new();
806
807    let s1 = df1.schema();
808    let s2 = df2.schema();
809
810    s1.field_compare(s2, &mut df1_extra, &mut df2_extra);
811
812    let df1_extra = df1_extra
813        .into_iter()
814        .map(|(_, (n, _))| n.as_str())
815        .collect::<Vec<_>>()
816        .join(", ");
817    let df2_extra = df2_extra
818        .into_iter()
819        .map(|(_, (n, _))| n.as_str())
820        .collect::<Vec<_>>()
821        .join(", ");
822
823    polars_err!(
824        SchemaMismatch: r#"unable to vstack, dataframes have different widths ({} != {}).
825One dataframe has additional columns: [{df1_extra}].
826Other dataframe has additional columns: [{df2_extra}]."#,
827        df1.width(),
828        df2.width(),
829    )
830}
831
832/// This takes ownership of the DataFrame so that drop is called earlier.
833/// Does not check if schema is correct
834pub fn accumulate_dataframes_vertical_unchecked<I>(dfs: I) -> DataFrame
835where
836    I: IntoIterator<Item = DataFrame>,
837{
838    let mut iter = dfs.into_iter();
839    let additional = iter.size_hint().0;
840    let mut acc_df = iter.next().unwrap();
841    acc_df.reserve_chunks(additional);
842
843    for df in iter {
844        if acc_df.width() != df.width() {
845            panic!("{}", width_mismatch(&acc_df, &df));
846        }
847
848        acc_df.vstack_mut_owned_unchecked(df);
849    }
850    acc_df
851}
852
853/// This takes ownership of the DataFrame so that drop is called earlier.
854/// # Panics
855/// Panics if `dfs` is empty.
856pub fn accumulate_dataframes_vertical<I>(dfs: I) -> PolarsResult<DataFrame>
857where
858    I: IntoIterator<Item = DataFrame>,
859{
860    let mut iter = dfs.into_iter();
861    let additional = iter.size_hint().0;
862    let mut acc_df = iter.next().unwrap();
863    acc_df.reserve_chunks(additional);
864    for df in iter {
865        if acc_df.width() != df.width() {
866            return Err(width_mismatch(&acc_df, &df));
867        }
868
869        acc_df.vstack_mut_owned(df)?;
870    }
871
872    Ok(acc_df)
873}
874
875/// Concat the DataFrames to a single DataFrame.
876pub fn concat_df<'a, I>(dfs: I) -> PolarsResult<DataFrame>
877where
878    I: IntoIterator<Item = &'a DataFrame>,
879{
880    let mut iter = dfs.into_iter();
881    let additional = iter.size_hint().0;
882    let mut acc_df = iter.next().unwrap().clone();
883    acc_df.reserve_chunks(additional);
884    for df in iter {
885        acc_df.vstack_mut(df)?;
886    }
887    Ok(acc_df)
888}
889
890/// Concat the DataFrames to a single DataFrame.
891pub fn concat_df_unchecked<'a, I>(dfs: I) -> DataFrame
892where
893    I: IntoIterator<Item = &'a DataFrame>,
894{
895    let mut iter = dfs.into_iter();
896    let additional = iter.size_hint().0;
897    let mut acc_df = iter.next().unwrap().clone();
898    acc_df.reserve_chunks(additional);
899    for df in iter {
900        acc_df.vstack_mut_unchecked(df);
901    }
902    acc_df
903}
904
905pub fn accumulate_dataframes_horizontal(dfs: Vec<DataFrame>) -> PolarsResult<DataFrame> {
906    let mut iter = dfs.into_iter();
907    let mut acc_df = iter.next().unwrap();
908    for df in iter {
909        acc_df.hstack_mut(df.columns())?;
910    }
911    Ok(acc_df)
912}
913
914/// Ensure the chunks in both ChunkedArrays have the same length.
915/// # Panics
916/// This will panic if `left.len() != right.len()` and array is chunked.
917pub fn align_chunks_binary<'a, T, B>(
918    left: &'a ChunkedArray<T>,
919    right: &'a ChunkedArray<B>,
920) -> (Cow<'a, ChunkedArray<T>>, Cow<'a, ChunkedArray<B>>)
921where
922    B: PolarsDataType,
923    T: PolarsDataType,
924{
925    let assert = || {
926        assert_eq!(
927            left.len(),
928            right.len(),
929            "expected arrays of the same length"
930        )
931    };
932    match (left.chunks.len(), right.chunks.len()) {
933        // All chunks are equal length
934        (1, 1) => (Cow::Borrowed(left), Cow::Borrowed(right)),
935        // All chunks are equal length
936        (a, b)
937            if a == b
938                && left
939                    .chunk_lengths()
940                    .zip(right.chunk_lengths())
941                    .all(|(l, r)| l == r) =>
942        {
943            (Cow::Borrowed(left), Cow::Borrowed(right))
944        },
945        (_, 1) => {
946            assert();
947            (
948                Cow::Borrowed(left),
949                Cow::Owned(right.match_chunks(left.chunk_lengths())),
950            )
951        },
952        (1, _) => {
953            assert();
954            (
955                Cow::Owned(left.match_chunks(right.chunk_lengths())),
956                Cow::Borrowed(right),
957            )
958        },
959        (_, _) => {
960            assert();
961            // could optimize to choose to rechunk a primitive and not a string or list type
962            let left = left.rechunk();
963            (
964                Cow::Owned(left.match_chunks(right.chunk_lengths())),
965                Cow::Borrowed(right),
966            )
967        },
968    }
969}
970
971/// Ensure the chunks in ChunkedArray and Series have the same length.
972/// # Panics
973/// This will panic if `left.len() != right.len()` and array is chunked.
974pub fn align_chunks_binary_ca_series<'a, T>(
975    left: &'a ChunkedArray<T>,
976    right: &'a Series,
977) -> (Cow<'a, ChunkedArray<T>>, Cow<'a, Series>)
978where
979    T: PolarsDataType,
980{
981    let assert = || {
982        assert_eq!(
983            left.len(),
984            right.len(),
985            "expected arrays of the same length"
986        )
987    };
988    match (left.chunks.len(), right.chunks().len()) {
989        // All chunks are equal length
990        (1, 1) => (Cow::Borrowed(left), Cow::Borrowed(right)),
991        // All chunks are equal length
992        (a, b)
993            if a == b
994                && left
995                    .chunk_lengths()
996                    .zip(right.chunk_lengths())
997                    .all(|(l, r)| l == r) =>
998        {
999            assert();
1000            (Cow::Borrowed(left), Cow::Borrowed(right))
1001        },
1002        (_, 1) => (left.rechunk(), Cow::Borrowed(right)),
1003        (1, _) => (Cow::Borrowed(left), Cow::Owned(right.rechunk())),
1004        (_, _) => {
1005            assert();
1006            (left.rechunk(), Cow::Owned(right.rechunk()))
1007        },
1008    }
1009}
1010
1011#[cfg(feature = "performant")]
1012pub(crate) fn align_chunks_binary_owned_series(left: Series, right: Series) -> (Series, Series) {
1013    match (left.chunks().len(), right.chunks().len()) {
1014        (1, 1) => (left, right),
1015        // All chunks are equal length
1016        (a, b)
1017            if a == b
1018                && left
1019                    .chunk_lengths()
1020                    .zip(right.chunk_lengths())
1021                    .all(|(l, r)| l == r) =>
1022        {
1023            (left, right)
1024        },
1025        (_, 1) => (left.rechunk(), right),
1026        (1, _) => (left, right.rechunk()),
1027        (_, _) => (left.rechunk(), right.rechunk()),
1028    }
1029}
1030
1031pub(crate) fn align_chunks_binary_owned<T, B>(
1032    left: ChunkedArray<T>,
1033    right: ChunkedArray<B>,
1034) -> (ChunkedArray<T>, ChunkedArray<B>)
1035where
1036    B: PolarsDataType,
1037    T: PolarsDataType,
1038{
1039    match (left.chunks.len(), right.chunks.len()) {
1040        (1, 1) => (left, right),
1041        // All chunks are equal length
1042        (a, b)
1043            if a == b
1044                && left
1045                    .chunk_lengths()
1046                    .zip(right.chunk_lengths())
1047                    .all(|(l, r)| l == r) =>
1048        {
1049            (left, right)
1050        },
1051        (_, 1) => (left.rechunk().into_owned(), right),
1052        (1, _) => (left, right.rechunk().into_owned()),
1053        (_, _) => (left.rechunk().into_owned(), right.rechunk().into_owned()),
1054    }
1055}
1056
1057/// # Panics
1058/// This will panic if `a.len() != b.len() || b.len() != c.len()` and array is chunked.
1059#[allow(clippy::type_complexity)]
1060pub fn align_chunks_ternary<'a, A, B, C>(
1061    a: &'a ChunkedArray<A>,
1062    b: &'a ChunkedArray<B>,
1063    c: &'a ChunkedArray<C>,
1064) -> (
1065    Cow<'a, ChunkedArray<A>>,
1066    Cow<'a, ChunkedArray<B>>,
1067    Cow<'a, ChunkedArray<C>>,
1068)
1069where
1070    A: PolarsDataType,
1071    B: PolarsDataType,
1072    C: PolarsDataType,
1073{
1074    if a.chunks.len() == 1 && b.chunks.len() == 1 && c.chunks.len() == 1 {
1075        return (Cow::Borrowed(a), Cow::Borrowed(b), Cow::Borrowed(c));
1076    }
1077
1078    assert!(
1079        a.len() == b.len() && b.len() == c.len(),
1080        "expected arrays of the same length"
1081    );
1082
1083    match (a.chunks.len(), b.chunks.len(), c.chunks.len()) {
1084        (_, 1, 1) => (
1085            Cow::Borrowed(a),
1086            Cow::Owned(b.match_chunks(a.chunk_lengths())),
1087            Cow::Owned(c.match_chunks(a.chunk_lengths())),
1088        ),
1089        (1, 1, _) => (
1090            Cow::Owned(a.match_chunks(c.chunk_lengths())),
1091            Cow::Owned(b.match_chunks(c.chunk_lengths())),
1092            Cow::Borrowed(c),
1093        ),
1094        (1, _, 1) => (
1095            Cow::Owned(a.match_chunks(b.chunk_lengths())),
1096            Cow::Borrowed(b),
1097            Cow::Owned(c.match_chunks(b.chunk_lengths())),
1098        ),
1099        (1, _, _) => {
1100            let b = b.rechunk();
1101            (
1102                Cow::Owned(a.match_chunks(c.chunk_lengths())),
1103                Cow::Owned(b.match_chunks(c.chunk_lengths())),
1104                Cow::Borrowed(c),
1105            )
1106        },
1107        (_, 1, _) => {
1108            let a = a.rechunk();
1109            (
1110                Cow::Owned(a.match_chunks(c.chunk_lengths())),
1111                Cow::Owned(b.match_chunks(c.chunk_lengths())),
1112                Cow::Borrowed(c),
1113            )
1114        },
1115        (_, _, 1) => {
1116            let b = b.rechunk();
1117            (
1118                Cow::Borrowed(a),
1119                Cow::Owned(b.match_chunks(a.chunk_lengths())),
1120                Cow::Owned(c.match_chunks(a.chunk_lengths())),
1121            )
1122        },
1123        (len_a, len_b, len_c)
1124            if len_a == len_b
1125                && len_b == len_c
1126                && a.chunk_lengths()
1127                    .zip(b.chunk_lengths())
1128                    .zip(c.chunk_lengths())
1129                    .all(|((a, b), c)| a == b && b == c) =>
1130        {
1131            (Cow::Borrowed(a), Cow::Borrowed(b), Cow::Borrowed(c))
1132        },
1133        _ => {
1134            // could optimize to choose to rechunk a primitive and not a string or list type
1135            let a = a.rechunk();
1136            let b = b.rechunk();
1137            (
1138                Cow::Owned(a.match_chunks(c.chunk_lengths())),
1139                Cow::Owned(b.match_chunks(c.chunk_lengths())),
1140                Cow::Borrowed(c),
1141            )
1142        },
1143    }
1144}
1145
1146pub fn binary_concatenate_validities<'a, T, B>(
1147    left: &'a ChunkedArray<T>,
1148    right: &'a ChunkedArray<B>,
1149) -> Option<Bitmap>
1150where
1151    B: PolarsDataType,
1152    T: PolarsDataType,
1153{
1154    let (left, right) = align_chunks_binary(left, right);
1155    let left_validity = concatenate_validities(left.chunks());
1156    let right_validity = concatenate_validities(right.chunks());
1157    combine_validities_and(left_validity.as_ref(), right_validity.as_ref())
1158}
1159
1160/// Convenience for `x.into_iter().map(Into::into).collect()` using an `into_vec()` function.
1161pub trait IntoVec<T> {
1162    fn into_vec(self) -> Vec<T>;
1163}
1164
1165impl<I, S> IntoVec<PlSmallStr> for I
1166where
1167    I: IntoIterator<Item = S>,
1168    S: Into<PlSmallStr>,
1169{
1170    fn into_vec(self) -> Vec<PlSmallStr> {
1171        self.into_iter().map(|s| s.into()).collect()
1172    }
1173}
1174
1175/// This logic is same as the impl on ChunkedArray
1176/// The difference is that there is less indirection because the caller should preallocate
1177/// `chunk_lens` once. On the `ChunkedArray` we indirect through an `ArrayRef` which is an indirection
1178/// and a vtable.
1179#[inline]
1180pub(crate) fn index_to_chunked_index<
1181    I: Iterator<Item = Idx>,
1182    Idx: PartialOrd + std::ops::AddAssign + std::ops::SubAssign + Zero + One,
1183>(
1184    chunk_lens: I,
1185    index: Idx,
1186) -> (Idx, Idx) {
1187    let mut index_remainder = index;
1188    let mut current_chunk_idx = Zero::zero();
1189
1190    for chunk_len in chunk_lens {
1191        if chunk_len > index_remainder {
1192            break;
1193        } else {
1194            index_remainder -= chunk_len;
1195            current_chunk_idx += One::one();
1196        }
1197    }
1198    (current_chunk_idx, index_remainder)
1199}
1200
1201pub(crate) fn index_to_chunked_index_rev<
1202    I: Iterator<Item = Idx>,
1203    Idx: PartialOrd
1204        + std::ops::AddAssign
1205        + std::ops::SubAssign
1206        + std::ops::Sub<Output = Idx>
1207        + Zero
1208        + One
1209        + Copy
1210        + std::fmt::Debug,
1211>(
1212    chunk_lens_rev: I,
1213    index_from_back: Idx,
1214    total_chunks: Idx,
1215) -> (Idx, Idx) {
1216    debug_assert!(index_from_back > Zero::zero(), "at least -1");
1217    let mut index_remainder = index_from_back;
1218    let mut current_chunk_idx = One::one();
1219    let mut current_chunk_len = Zero::zero();
1220
1221    for chunk_len in chunk_lens_rev {
1222        current_chunk_len = chunk_len;
1223        if chunk_len >= index_remainder {
1224            break;
1225        } else {
1226            index_remainder -= chunk_len;
1227            current_chunk_idx += One::one();
1228        }
1229    }
1230    (
1231        total_chunks - current_chunk_idx,
1232        current_chunk_len - index_remainder,
1233    )
1234}
1235
1236pub fn first_null<'a, I>(iter: I) -> Option<usize>
1237where
1238    I: Iterator<Item = &'a dyn Array>,
1239{
1240    let mut offset = 0;
1241    for arr in iter {
1242        if let Some(mask) = arr.validity() {
1243            let len_mask = mask.len();
1244            let n = mask.leading_ones();
1245            if n < len_mask {
1246                return Some(offset + n);
1247            }
1248            offset += len_mask
1249        } else {
1250            offset += arr.len();
1251        }
1252    }
1253    None
1254}
1255
1256pub fn first_non_null<'a, I>(iter: I) -> Option<usize>
1257where
1258    I: Iterator<Item = &'a dyn Array>,
1259{
1260    let mut offset = 0;
1261    for arr in iter {
1262        if let Some(mask) = arr.validity() {
1263            let len_mask = mask.len();
1264            let n = mask.leading_zeros();
1265            if n < len_mask {
1266                return Some(offset + n);
1267            }
1268            offset += len_mask
1269        } else if !arr.is_empty() {
1270            return Some(offset);
1271        }
1272    }
1273    None
1274}
1275
1276pub fn last_non_null<'a, I>(iter: I, len: usize) -> Option<usize>
1277where
1278    I: DoubleEndedIterator<Item = &'a dyn Array>,
1279{
1280    if len == 0 {
1281        return None;
1282    }
1283    let mut offset = 0;
1284    for arr in iter.rev() {
1285        if let Some(mask) = arr.validity() {
1286            let len_mask = mask.len();
1287            let n = mask.trailing_zeros();
1288            if n < len_mask {
1289                return Some(len - offset - n - 1);
1290            }
1291            offset += len_mask;
1292        } else if !arr.is_empty() {
1293            return Some(len - offset - 1);
1294        }
1295    }
1296    None
1297}
1298
1299pub fn coalesce_nulls_columns(a: &Column, b: &Column) -> (Column, Column) {
1300    if a.null_count() > 0 || b.null_count() > 0 {
1301        let mut a = a.as_materialized_series().rechunk();
1302        let mut b = b.as_materialized_series().rechunk();
1303        for (arr_a, arr_b) in unsafe { a.chunks_mut().iter_mut().zip(b.chunks_mut()) } {
1304            let validity = match (arr_a.validity(), arr_b.validity()) {
1305                (None, Some(b)) => Some(b.clone()),
1306                (Some(a), Some(b)) => Some(a & b),
1307                (Some(a), None) => Some(a.clone()),
1308                (None, None) => None,
1309            };
1310            *arr_a = arr_a.with_validity(validity.clone());
1311            *arr_b = arr_b.with_validity(validity);
1312        }
1313        a.compute_len();
1314        b.compute_len();
1315        (a.into(), b.into())
1316    } else {
1317        (a.clone(), b.clone())
1318    }
1319}
1320
1321#[cfg(test)]
1322mod test {
1323    use super::*;
1324
1325    #[test]
1326    fn test_split() {
1327        let ca: Int32Chunked = (0..10).collect_ca("a".into());
1328
1329        let out = split(&ca, 3);
1330        assert_eq!(out[0].len(), 3);
1331        assert_eq!(out[1].len(), 3);
1332        assert_eq!(out[2].len(), 4);
1333    }
1334
1335    #[test]
1336    fn test_align_chunks() -> PolarsResult<()> {
1337        let a = Int32Chunked::new(PlSmallStr::EMPTY, &[1, 2, 3, 4]);
1338        let mut b = Int32Chunked::new(PlSmallStr::EMPTY, &[1]);
1339        let b2 = Int32Chunked::new(PlSmallStr::EMPTY, &[2, 3, 4]);
1340
1341        b.append(&b2)?;
1342        let (a, b) = align_chunks_binary(&a, &b);
1343        assert_eq!(
1344            a.chunk_lengths().collect::<Vec<_>>(),
1345            b.chunk_lengths().collect::<Vec<_>>()
1346        );
1347
1348        let a = Int32Chunked::new(PlSmallStr::EMPTY, &[1, 2, 3, 4]);
1349        let mut b = Int32Chunked::new(PlSmallStr::EMPTY, &[1]);
1350        let b1 = b.clone();
1351        b.append(&b1)?;
1352        b.append(&b1)?;
1353        b.append(&b1)?;
1354        let (a, b) = align_chunks_binary(&a, &b);
1355        assert_eq!(
1356            a.chunk_lengths().collect::<Vec<_>>(),
1357            b.chunk_lengths().collect::<Vec<_>>()
1358        );
1359
1360        Ok(())
1361    }
1362}