polars_core/chunked_array/ops/
mod.rs

1//! Traits for miscellaneous operations on ChunkedArray
2use arrow::offset::OffsetsBuffer;
3use polars_compute::rolling::QuantileMethod;
4
5use crate::prelude::*;
6
7pub(crate) mod aggregate;
8pub(crate) mod any_value;
9pub(crate) mod append;
10mod apply;
11#[cfg(feature = "approx_unique")]
12mod approx_n_unique;
13pub mod arity;
14mod bit_repr;
15mod bits;
16#[cfg(feature = "bitwise")]
17mod bitwise_reduce;
18pub(crate) mod chunkops;
19pub(crate) mod compare_inner;
20#[cfg(feature = "dtype-decimal")]
21mod decimal;
22pub(crate) mod downcast;
23pub(crate) mod explode;
24mod explode_and_offsets;
25mod extend;
26pub mod fill_null;
27mod filter;
28pub mod float_sorted_arg_max;
29mod for_each;
30pub mod full;
31pub mod gather;
32mod nesting_utils;
33pub(crate) mod nulls;
34mod reverse;
35#[cfg(feature = "rolling_window")]
36pub(crate) mod rolling_window;
37pub mod row_encode;
38pub mod search_sorted;
39mod set;
40mod shift;
41pub mod sort;
42#[cfg(feature = "algorithm_group_by")]
43pub(crate) mod unique;
44#[cfg(feature = "zip_with")]
45pub mod zip;
46
47pub use chunkops::_set_check_length;
48pub use nesting_utils::ChunkNestingUtils;
49#[cfg(feature = "serde-lazy")]
50use serde::{Deserialize, Serialize};
51pub use sort::options::*;
52
53use crate::chunked_array::cast::CastOptions;
54use crate::series::{BitRepr, IsSorted};
55#[cfg(feature = "reinterpret")]
56pub trait Reinterpret {
57    fn reinterpret_signed(&self) -> Series {
58        unimplemented!()
59    }
60
61    fn reinterpret_unsigned(&self) -> Series {
62        unimplemented!()
63    }
64}
65
66/// Transmute [`ChunkedArray`] to bit representation.
67/// This is useful in hashing context and reduces no.
68/// of compiled code paths.
69pub(crate) trait ToBitRepr {
70    fn to_bit_repr(&self) -> BitRepr;
71}
72
73pub trait ChunkAnyValue {
74    /// Get a single value. Beware this is slow.
75    /// If you need to use this slightly performant, cast Categorical to UInt32
76    ///
77    /// # Safety
78    /// Does not do any bounds checking.
79    unsafe fn get_any_value_unchecked(&self, index: usize) -> AnyValue;
80
81    /// Get a single value. Beware this is slow.
82    fn get_any_value(&self, index: usize) -> PolarsResult<AnyValue>;
83}
84
85/// Explode/flatten a List or String Series
86pub trait ChunkExplode {
87    fn explode(&self, skip_empty: bool) -> PolarsResult<Series> {
88        self.explode_and_offsets(skip_empty).map(|t| t.0)
89    }
90    fn offsets(&self) -> PolarsResult<OffsetsBuffer<i64>>;
91    fn explode_and_offsets(&self, skip_empty: bool) -> PolarsResult<(Series, OffsetsBuffer<i64>)>;
92}
93
94pub trait ChunkBytes {
95    fn to_byte_slices(&self) -> Vec<&[u8]>;
96}
97
98/// This differs from ChunkWindowCustom and ChunkWindow
99/// by not using a fold aggregator, but reusing a `Series` wrapper and calling `Series` aggregators.
100/// This likely is a bit slower than ChunkWindow
101#[cfg(feature = "rolling_window")]
102pub trait ChunkRollApply: AsRefDataType {
103    fn rolling_map(
104        &self,
105        _f: &dyn Fn(&Series) -> Series,
106        _options: RollingOptionsFixedWindow,
107    ) -> PolarsResult<Series>
108    where
109        Self: Sized,
110    {
111        polars_bail!(opq = rolling_map, self.as_ref_dtype());
112    }
113}
114
115pub trait ChunkTake<Idx: ?Sized>: ChunkTakeUnchecked<Idx> {
116    /// Gather values from ChunkedArray by index.
117    fn take(&self, indices: &Idx) -> PolarsResult<Self>
118    where
119        Self: Sized;
120}
121
122pub trait ChunkTakeUnchecked<Idx: ?Sized> {
123    /// Gather values from ChunkedArray by index.
124    ///
125    /// # Safety
126    /// The non-null indices must be valid.
127    unsafe fn take_unchecked(&self, indices: &Idx) -> Self;
128}
129
130/// Create a `ChunkedArray` with new values by index or by boolean mask.
131///
132/// Note that these operations clone data. This is however the only way we can modify at mask or
133/// index level as the underlying Arrow arrays are immutable.
134pub trait ChunkSet<'a, A, B> {
135    /// Set the values at indexes `idx` to some optional value `Option<T>`.
136    ///
137    /// # Example
138    ///
139    /// ```rust
140    /// # use polars_core::prelude::*;
141    /// let ca = UInt32Chunked::new("a".into(), &[1, 2, 3]);
142    /// let new = ca.scatter_single(vec![0, 1], Some(10)).unwrap();
143    ///
144    /// assert_eq!(Vec::from(&new), &[Some(10), Some(10), Some(3)]);
145    /// ```
146    fn scatter_single<I: IntoIterator<Item = IdxSize>>(
147        &'a self,
148        idx: I,
149        opt_value: Option<A>,
150    ) -> PolarsResult<Self>
151    where
152        Self: Sized;
153
154    /// Set the values at indexes `idx` by applying a closure to these values.
155    ///
156    /// # Example
157    ///
158    /// ```rust
159    /// # use polars_core::prelude::*;
160    /// let ca = Int32Chunked::new("a".into(), &[1, 2, 3]);
161    /// let new = ca.scatter_with(vec![0, 1], |opt_v| opt_v.map(|v| v - 5)).unwrap();
162    ///
163    /// assert_eq!(Vec::from(&new), &[Some(-4), Some(-3), Some(3)]);
164    /// ```
165    fn scatter_with<I: IntoIterator<Item = IdxSize>, F>(
166        &'a self,
167        idx: I,
168        f: F,
169    ) -> PolarsResult<Self>
170    where
171        Self: Sized,
172        F: Fn(Option<A>) -> Option<B>;
173    /// Set the values where the mask evaluates to `true` to some optional value `Option<T>`.
174    ///
175    /// # Example
176    ///
177    /// ```rust
178    /// # use polars_core::prelude::*;
179    /// let ca = Int32Chunked::new("a".into(), &[1, 2, 3]);
180    /// let mask = BooleanChunked::new("mask".into(), &[false, true, false]);
181    /// let new = ca.set(&mask, Some(5)).unwrap();
182    /// assert_eq!(Vec::from(&new), &[Some(1), Some(5), Some(3)]);
183    /// ```
184    fn set(&'a self, mask: &BooleanChunked, opt_value: Option<A>) -> PolarsResult<Self>
185    where
186        Self: Sized;
187}
188
189/// Cast `ChunkedArray<T>` to `ChunkedArray<N>`
190pub trait ChunkCast {
191    /// Cast a [`ChunkedArray`] to [`DataType`]
192    fn cast(&self, dtype: &DataType) -> PolarsResult<Series> {
193        self.cast_with_options(dtype, CastOptions::NonStrict)
194    }
195
196    /// Cast a [`ChunkedArray`] to [`DataType`]
197    fn cast_with_options(&self, dtype: &DataType, options: CastOptions) -> PolarsResult<Series>;
198
199    /// Does not check if the cast is a valid one and may over/underflow
200    ///
201    /// # Safety
202    /// - This doesn't do utf8 validation checking when casting from binary
203    /// - This doesn't do categorical bound checking when casting from UInt32
204    unsafe fn cast_unchecked(&self, dtype: &DataType) -> PolarsResult<Series>;
205}
206
207/// Fastest way to do elementwise operations on a [`ChunkedArray<T>`] when the operation is cheaper than
208/// branching due to null checking.
209pub trait ChunkApply<'a, T> {
210    type FuncRet;
211
212    /// Apply a closure elementwise. This is fastest when the null check branching is more expensive
213    /// than the closure application. Often it is.
214    ///
215    /// Null values remain null.
216    ///
217    /// # Example
218    ///
219    /// ```
220    /// use polars_core::prelude::*;
221    /// fn double(ca: &UInt32Chunked) -> UInt32Chunked {
222    ///     ca.apply_values(|v| v * 2)
223    /// }
224    /// ```
225    #[must_use]
226    fn apply_values<F>(&'a self, f: F) -> Self
227    where
228        F: Fn(T) -> Self::FuncRet + Copy;
229
230    /// Apply a closure elementwise including null values.
231    #[must_use]
232    fn apply<F>(&'a self, f: F) -> Self
233    where
234        F: Fn(Option<T>) -> Option<Self::FuncRet> + Copy;
235
236    /// Apply a closure elementwise and write results to a mutable slice.
237    fn apply_to_slice<F, S>(&'a self, f: F, slice: &mut [S])
238    // (value of chunkedarray, value of slice) -> value of slice
239    where
240        F: Fn(Option<T>, &S) -> S;
241}
242
243/// Aggregation operations.
244pub trait ChunkAgg<T> {
245    /// Aggregate the sum of the ChunkedArray.
246    /// Returns `None` if not implemented for `T`.
247    /// If the array is empty, `0` is returned
248    fn sum(&self) -> Option<T> {
249        None
250    }
251
252    fn _sum_as_f64(&self) -> f64;
253
254    fn min(&self) -> Option<T> {
255        None
256    }
257
258    /// Returns the maximum value in the array, according to the natural order.
259    /// Returns `None` if the array is empty or only contains null values.
260    fn max(&self) -> Option<T> {
261        None
262    }
263
264    fn min_max(&self) -> Option<(T, T)> {
265        Some((self.min()?, self.max()?))
266    }
267
268    /// Returns the mean value in the array.
269    /// Returns `None` if the array is empty or only contains null values.
270    fn mean(&self) -> Option<f64> {
271        None
272    }
273}
274
275/// Quantile and median aggregation.
276pub trait ChunkQuantile<T> {
277    /// Returns the mean value in the array.
278    /// Returns `None` if the array is empty or only contains null values.
279    fn median(&self) -> Option<T> {
280        None
281    }
282    /// Aggregate a given quantile of the ChunkedArray.
283    /// Returns `None` if the array is empty or only contains null values.
284    fn quantile(&self, _quantile: f64, _method: QuantileMethod) -> PolarsResult<Option<T>> {
285        Ok(None)
286    }
287}
288
289/// Variance and standard deviation aggregation.
290pub trait ChunkVar {
291    /// Compute the variance of this ChunkedArray/Series.
292    fn var(&self, _ddof: u8) -> Option<f64> {
293        None
294    }
295
296    /// Compute the standard deviation of this ChunkedArray/Series.
297    fn std(&self, _ddof: u8) -> Option<f64> {
298        None
299    }
300}
301
302/// Bitwise Reduction Operations.
303#[cfg(feature = "bitwise")]
304pub trait ChunkBitwiseReduce {
305    type Physical;
306
307    fn and_reduce(&self) -> Option<Self::Physical>;
308    fn or_reduce(&self) -> Option<Self::Physical>;
309    fn xor_reduce(&self) -> Option<Self::Physical>;
310}
311
312/// Compare [`Series`] and [`ChunkedArray`]'s and get a `boolean` mask that
313/// can be used to filter rows.
314///
315/// # Example
316///
317/// ```
318/// use polars_core::prelude::*;
319/// fn filter_all_ones(df: &DataFrame) -> PolarsResult<DataFrame> {
320///     let mask = df
321///     .column("column_a")?
322///     .as_materialized_series()
323///     .equal(1)?;
324///
325///     df.filter(&mask)
326/// }
327/// ```
328pub trait ChunkCompareEq<Rhs> {
329    type Item;
330
331    /// Check for equality.
332    fn equal(&self, rhs: Rhs) -> Self::Item;
333
334    /// Check for equality where `None == None`.
335    fn equal_missing(&self, rhs: Rhs) -> Self::Item;
336
337    /// Check for inequality.
338    fn not_equal(&self, rhs: Rhs) -> Self::Item;
339
340    /// Check for inequality where `None == None`.
341    fn not_equal_missing(&self, rhs: Rhs) -> Self::Item;
342}
343
344/// Compare [`Series`] and [`ChunkedArray`]'s using inequality operators (`<`, `>=`, etc.) and get
345/// a `boolean` mask that can be used to filter rows.
346pub trait ChunkCompareIneq<Rhs> {
347    type Item;
348
349    /// Greater than comparison.
350    fn gt(&self, rhs: Rhs) -> Self::Item;
351
352    /// Greater than or equal comparison.
353    fn gt_eq(&self, rhs: Rhs) -> Self::Item;
354
355    /// Less than comparison.
356    fn lt(&self, rhs: Rhs) -> Self::Item;
357
358    /// Less than or equal comparison
359    fn lt_eq(&self, rhs: Rhs) -> Self::Item;
360}
361
362/// Get unique values in a `ChunkedArray`
363pub trait ChunkUnique {
364    // We don't return Self to be able to use AutoRef specialization
365    /// Get unique values of a ChunkedArray
366    fn unique(&self) -> PolarsResult<Self>
367    where
368        Self: Sized;
369
370    /// Get first index of the unique values in a `ChunkedArray`.
371    /// This Vec is sorted.
372    fn arg_unique(&self) -> PolarsResult<IdxCa>;
373
374    /// Number of unique values in the `ChunkedArray`
375    fn n_unique(&self) -> PolarsResult<usize> {
376        self.arg_unique().map(|v| v.len())
377    }
378}
379
380#[cfg(feature = "approx_unique")]
381pub trait ChunkApproxNUnique {
382    fn approx_n_unique(&self) -> IdxSize;
383}
384
385/// Sort operations on `ChunkedArray`.
386pub trait ChunkSort<T: PolarsDataType> {
387    #[allow(unused_variables)]
388    fn sort_with(&self, options: SortOptions) -> ChunkedArray<T>;
389
390    /// Returned a sorted `ChunkedArray`.
391    fn sort(&self, descending: bool) -> ChunkedArray<T>;
392
393    /// Retrieve the indexes needed to sort this array.
394    fn arg_sort(&self, options: SortOptions) -> IdxCa;
395
396    /// Retrieve the indexes need to sort this and the other arrays.
397    #[allow(unused_variables)]
398    fn arg_sort_multiple(
399        &self,
400        by: &[Column],
401        _options: &SortMultipleOptions,
402    ) -> PolarsResult<IdxCa> {
403        polars_bail!(opq = arg_sort_multiple, T::get_static_dtype());
404    }
405}
406
407pub type FillNullLimit = Option<IdxSize>;
408
409#[derive(Copy, Clone, Debug, PartialEq, Hash)]
410#[cfg_attr(feature = "serde-lazy", derive(Serialize, Deserialize))]
411#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
412pub enum FillNullStrategy {
413    /// previous value in array
414    Backward(FillNullLimit),
415    /// next value in array
416    Forward(FillNullLimit),
417    /// mean value of array
418    Mean,
419    /// minimal value in array
420    Min,
421    /// maximum value in array
422    Max,
423    /// replace with the value zero
424    Zero,
425    /// replace with the value one
426    One,
427}
428
429impl FillNullStrategy {
430    pub fn is_elementwise(&self) -> bool {
431        matches!(self, Self::One | Self::Zero)
432    }
433}
434
435/// Replace None values with a value
436pub trait ChunkFillNullValue<T> {
437    /// Replace None values with a give value `T`.
438    fn fill_null_with_values(&self, value: T) -> PolarsResult<Self>
439    where
440        Self: Sized;
441}
442
443/// Fill a ChunkedArray with one value.
444pub trait ChunkFull<T> {
445    /// Create a ChunkedArray with a single value.
446    fn full(name: PlSmallStr, value: T, length: usize) -> Self
447    where
448        Self: Sized;
449}
450
451pub trait ChunkFullNull {
452    fn full_null(_name: PlSmallStr, _length: usize) -> Self
453    where
454        Self: Sized;
455}
456
457/// Reverse a [`ChunkedArray<T>`]
458pub trait ChunkReverse {
459    /// Return a reversed version of this array.
460    fn reverse(&self) -> Self;
461}
462
463/// Filter values by a boolean mask.
464pub trait ChunkFilter<T: PolarsDataType> {
465    /// Filter values in the ChunkedArray with a boolean mask.
466    ///
467    /// ```rust
468    /// # use polars_core::prelude::*;
469    /// let array = Int32Chunked::new("array".into(), &[1, 2, 3]);
470    /// let mask = BooleanChunked::new("mask".into(), &[true, false, true]);
471    ///
472    /// let filtered = array.filter(&mask).unwrap();
473    /// assert_eq!(Vec::from(&filtered), [Some(1), Some(3)])
474    /// ```
475    fn filter(&self, filter: &BooleanChunked) -> PolarsResult<ChunkedArray<T>>
476    where
477        Self: Sized;
478}
479
480/// Create a new ChunkedArray filled with values at that index.
481pub trait ChunkExpandAtIndex<T: PolarsDataType> {
482    /// Create a new ChunkedArray filled with values at that index.
483    fn new_from_index(&self, index: usize, length: usize) -> ChunkedArray<T>;
484}
485
486macro_rules! impl_chunk_expand {
487    ($self:ident, $length:ident, $index:ident) => {{
488        if $self.is_empty() {
489            return $self.clone();
490        }
491        let opt_val = $self.get($index);
492        match opt_val {
493            Some(val) => ChunkedArray::full($self.name().clone(), val, $length),
494            None => ChunkedArray::full_null($self.name().clone(), $length),
495        }
496    }};
497}
498
499impl<T: PolarsNumericType> ChunkExpandAtIndex<T> for ChunkedArray<T>
500where
501    ChunkedArray<T>: ChunkFull<T::Native>,
502{
503    fn new_from_index(&self, index: usize, length: usize) -> ChunkedArray<T> {
504        let mut out = impl_chunk_expand!(self, length, index);
505        out.set_sorted_flag(IsSorted::Ascending);
506        out
507    }
508}
509
510impl ChunkExpandAtIndex<BooleanType> for BooleanChunked {
511    fn new_from_index(&self, index: usize, length: usize) -> BooleanChunked {
512        let mut out = impl_chunk_expand!(self, length, index);
513        out.set_sorted_flag(IsSorted::Ascending);
514        out
515    }
516}
517
518impl ChunkExpandAtIndex<StringType> for StringChunked {
519    fn new_from_index(&self, index: usize, length: usize) -> StringChunked {
520        let mut out = impl_chunk_expand!(self, length, index);
521        out.set_sorted_flag(IsSorted::Ascending);
522        out
523    }
524}
525
526impl ChunkExpandAtIndex<BinaryType> for BinaryChunked {
527    fn new_from_index(&self, index: usize, length: usize) -> BinaryChunked {
528        let mut out = impl_chunk_expand!(self, length, index);
529        out.set_sorted_flag(IsSorted::Ascending);
530        out
531    }
532}
533
534impl ChunkExpandAtIndex<BinaryOffsetType> for BinaryOffsetChunked {
535    fn new_from_index(&self, index: usize, length: usize) -> BinaryOffsetChunked {
536        let mut out = impl_chunk_expand!(self, length, index);
537        out.set_sorted_flag(IsSorted::Ascending);
538        out
539    }
540}
541
542impl ChunkExpandAtIndex<ListType> for ListChunked {
543    fn new_from_index(&self, index: usize, length: usize) -> ListChunked {
544        let opt_val = self.get_as_series(index);
545        match opt_val {
546            Some(val) => {
547                let mut ca = ListChunked::full(self.name().clone(), &val, length);
548                unsafe { ca.to_logical(self.inner_dtype().clone()) };
549                ca
550            },
551            None => {
552                ListChunked::full_null_with_dtype(self.name().clone(), length, self.inner_dtype())
553            },
554        }
555    }
556}
557
558#[cfg(feature = "dtype-struct")]
559impl ChunkExpandAtIndex<StructType> for StructChunked {
560    fn new_from_index(&self, index: usize, length: usize) -> ChunkedArray<StructType> {
561        let (chunk_idx, idx) = self.index_to_chunked_index(index);
562        let chunk = self.downcast_chunks().get(chunk_idx).unwrap();
563        let chunk = if chunk.is_null(idx) {
564            new_null_array(chunk.dtype().clone(), length)
565        } else {
566            let values = chunk
567                .values()
568                .iter()
569                .map(|arr| {
570                    let s = Series::try_from((PlSmallStr::EMPTY, arr.clone())).unwrap();
571                    let s = s.new_from_index(idx, length);
572                    s.chunks()[0].clone()
573                })
574                .collect::<Vec<_>>();
575
576            StructArray::new(chunk.dtype().clone(), length, values, None).boxed()
577        };
578
579        // SAFETY: chunks are from self.
580        unsafe { self.copy_with_chunks(vec![chunk]) }
581    }
582}
583
584#[cfg(feature = "dtype-array")]
585impl ChunkExpandAtIndex<FixedSizeListType> for ArrayChunked {
586    fn new_from_index(&self, index: usize, length: usize) -> ArrayChunked {
587        let opt_val = self.get_as_series(index);
588        match opt_val {
589            Some(val) => {
590                let mut ca = ArrayChunked::full(self.name().clone(), &val, length);
591                unsafe { ca.to_logical(self.inner_dtype().clone()) };
592                ca
593            },
594            None => ArrayChunked::full_null_with_dtype(
595                self.name().clone(),
596                length,
597                self.inner_dtype(),
598                self.width(),
599            ),
600        }
601    }
602}
603
604#[cfg(feature = "object")]
605impl<T: PolarsObject> ChunkExpandAtIndex<ObjectType<T>> for ObjectChunked<T> {
606    fn new_from_index(&self, index: usize, length: usize) -> ObjectChunked<T> {
607        let opt_val = self.get(index);
608        match opt_val {
609            Some(val) => ObjectChunked::<T>::full(self.name().clone(), val.clone(), length),
610            None => ObjectChunked::<T>::full_null(self.name().clone(), length),
611        }
612    }
613}
614
615/// Shift the values of a [`ChunkedArray`] by a number of periods.
616pub trait ChunkShiftFill<T: PolarsDataType, V> {
617    /// Shift the values by a given period and fill the parts that will be empty due to this operation
618    /// with `fill_value`.
619    fn shift_and_fill(&self, periods: i64, fill_value: V) -> ChunkedArray<T>;
620}
621
622pub trait ChunkShift<T: PolarsDataType> {
623    fn shift(&self, periods: i64) -> ChunkedArray<T>;
624}
625
626/// Combine two [`ChunkedArray`] based on some predicate.
627pub trait ChunkZip<T: PolarsDataType> {
628    /// Create a new ChunkedArray with values from self where the mask evaluates `true` and values
629    /// from `other` where the mask evaluates `false`
630    fn zip_with(
631        &self,
632        mask: &BooleanChunked,
633        other: &ChunkedArray<T>,
634    ) -> PolarsResult<ChunkedArray<T>>;
635}
636
637/// Apply kernels on the arrow array chunks in a ChunkedArray.
638pub trait ChunkApplyKernel<A: Array> {
639    /// Apply kernel and return result as a new ChunkedArray.
640    #[must_use]
641    fn apply_kernel(&self, f: &dyn Fn(&A) -> ArrayRef) -> Self;
642
643    /// Apply a kernel that outputs an array of different type.
644    fn apply_kernel_cast<S>(&self, f: &dyn Fn(&A) -> ArrayRef) -> ChunkedArray<S>
645    where
646        S: PolarsDataType;
647}
648
649#[cfg(feature = "is_first_distinct")]
650/// Mask the first unique values as `true`
651pub trait IsFirstDistinct<T: PolarsDataType> {
652    fn is_first_distinct(&self) -> PolarsResult<BooleanChunked> {
653        polars_bail!(opq = is_first_distinct, T::get_static_dtype());
654    }
655}
656
657#[cfg(feature = "is_last_distinct")]
658/// Mask the last unique values as `true`
659pub trait IsLastDistinct<T: PolarsDataType> {
660    fn is_last_distinct(&self) -> PolarsResult<BooleanChunked> {
661        polars_bail!(opq = is_last_distinct, T::get_static_dtype());
662    }
663}
polars_core/chunked_array/ops/mod.rs

polars_core/chunked_array/ops/
mod.rs
