Skip to main content

polars_ops/series/ops/
horizontal.rs

1use std::borrow::Cow;
2
3use polars_arrow::array::PrimitiveArray;
4use polars_arrow::compute::utils::combine_validities_or;
5use polars_core::chunked_array::cast::CastOptions;
6use polars_core::prelude::*;
7use polars_core::runtime::RAYON;
8use polars_core::series::arithmetic::coerce_lhs_rhs;
9use polars_core::utils::{CustomIterTools, align_chunks_binary, dtypes_to_supertype};
10use polars_core::with_match_physical_numeric_polars_type;
11use polars_utils::broadcast::broadcast_len;
12use polars_utils::min_max::MinMax;
13use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
14
15pub trait MinMaxHorizontal {
16    /// Aggregate the column horizontally to their min values.
17    fn min_horizontal(&self) -> PolarsResult<Option<Column>>;
18    /// Aggregate the column horizontally to their max values.
19    fn max_horizontal(&self) -> PolarsResult<Option<Column>>;
20}
21
22impl MinMaxHorizontal for DataFrame {
23    fn min_horizontal(&self) -> PolarsResult<Option<Column>> {
24        min_horizontal(self.columns())
25    }
26    fn max_horizontal(&self) -> PolarsResult<Option<Column>> {
27        max_horizontal(self.columns())
28    }
29}
30
31#[derive(Copy, Clone, Debug, PartialEq)]
32pub enum NullStrategy {
33    Ignore,
34    Propagate,
35}
36
37pub trait SumMeanHorizontal {
38    /// Sum all values horizontally across columns.
39    fn sum_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>>;
40
41    /// Compute the mean of all numeric values horizontally across columns.
42    fn mean_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>>;
43}
44
45impl SumMeanHorizontal for DataFrame {
46    fn sum_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>> {
47        sum_horizontal(self.columns(), null_strategy)
48    }
49    fn mean_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>> {
50        mean_horizontal(self.columns(), null_strategy)
51    }
52}
53
54/// Replace the nulls in `s` with the values of `other` at the same position.
55fn fill_null_from<'a>(s: &'a Series, other: &Series) -> PolarsResult<Cow<'a, Series>> {
56    if s.has_nulls() {
57        s.zip_with_same_type(&s.is_not_null(), other)
58            .map(Cow::Owned)
59    } else {
60        Ok(Cow::Borrowed(s))
61    }
62}
63
64/// Elementwise `op` where a null on one side takes the value of the other side.
65fn binary_ignore_nulls<T, F>(
66    left: &ChunkedArray<T>,
67    right: &ChunkedArray<T>,
68    op: F,
69) -> ChunkedArray<T>
70where
71    T: PolarsNumericType,
72    F: Fn(T::Native, T::Native) -> T::Native,
73{
74    let (left, right) = align_chunks_binary(left, right);
75    let chunks = left
76        .downcast_iter()
77        .zip(right.downcast_iter())
78        .map(|(l, r)| {
79            let lv = l.values().iter().copied();
80            let rv = r.values().iter().copied();
81            let values: Vec<T::Native> = match (l.validity(), r.validity()) {
82                (Some(lm), Some(rm)) => lv
83                    .zip(rv)
84                    .zip(lm.iter().zip(rm.iter()))
85                    .map(|((a, b), (a_valid, b_valid))| {
86                        op(if a_valid { a } else { b }, if b_valid { b } else { a })
87                    })
88                    .collect_trusted(),
89                (Some(lm), None) => lv
90                    .zip(rv)
91                    .zip(lm.iter())
92                    .map(|((a, b), a_valid)| op(if a_valid { a } else { b }, b))
93                    .collect_trusted(),
94                (None, Some(rm)) => lv
95                    .zip(rv)
96                    .zip(rm.iter())
97                    .map(|((a, b), b_valid)| op(a, if b_valid { b } else { a }))
98                    .collect_trusted(),
99                (None, None) => lv.zip(rv).map(|(a, b)| op(a, b)).collect_trusted(),
100            };
101            let validity = combine_validities_or(l.validity(), r.validity());
102            PrimitiveArray::from_vec(values).with_validity(validity)
103        });
104    ChunkedArray::from_chunk_iter(left.name().clone(), chunks)
105}
106
107fn min_max_binary_columns(left: &Column, right: &Column, min: bool) -> PolarsResult<Column> {
108    if left.dtype().to_physical().is_primitive_numeric()
109        && right.dtype().to_physical().is_primitive_numeric()
110    {
111        let height = broadcast_len([left, right])?;
112        let left_s = left.as_materialized_series_maintain_scalar();
113        let right_s = right.as_materialized_series_maintain_scalar();
114        let (lhs, rhs) = coerce_lhs_rhs(&left_s, &right_s)?;
115        let logical = lhs.dtype();
116
117        let lhs = lhs.to_physical_repr();
118        let rhs = rhs.to_physical_repr();
119        let has_nulls = lhs.has_nulls() || rhs.has_nulls();
120        let same_len = lhs.len() == rhs.len();
121        let (lhs, rhs) = if has_nulls && !same_len {
122            (fill_null_from(&lhs, &rhs)?, fill_null_from(&rhs, &lhs)?)
123        } else {
124            (lhs, rhs)
125        };
126
127        with_match_physical_numeric_polars_type!(lhs.dtype(), |$T| {
128            let a: &ChunkedArray<$T> = lhs.as_ref().as_ref().as_ref();
129            let b: &ChunkedArray<$T> = rhs.as_ref().as_ref().as_ref();
130
131            let out: ChunkedArray<$T> = match (has_nulls && same_len, min) {
132                (true, true) => binary_ignore_nulls(a, b, MinMax::min_ignore_nan),
133                (true, false) => binary_ignore_nulls(a, b, MinMax::max_ignore_nan),
134                (false, true) => {
135                    arity::broadcast_binary_elementwise_values(a, b, MinMax::min_ignore_nan)
136                },
137                (false, false) => {
138                    arity::broadcast_binary_elementwise_values(a, b, MinMax::max_ignore_nan)
139                },
140            };
141            unsafe { out.into_series().from_physical_unchecked(logical) }
142        })
143        .and_then(|s| s.broadcast_owned_to(height))
144        .map(Column::from)
145    } else {
146        let mut mask = if min {
147            left.lt(right)?
148        } else {
149            left.gt(right)?
150        };
151        if left.has_nulls() {
152            mask = mask & left.is_not_null();
153        }
154        if right.has_nulls() {
155            mask = mask | right.is_null();
156        }
157        left.zip_with(&mask, right)
158    }
159}
160
161pub fn max_horizontal(columns: &[Column]) -> PolarsResult<Option<Column>> {
162    broadcast_len(columns.iter()).context("max_horizontal")?;
163
164    let max_fn = |acc: &Column, s: &Column| min_max_binary_columns(acc, s, false);
165
166    match columns.len() {
167        0 => Ok(None),
168        1 => Ok(Some(columns[0].clone())),
169        2 => max_fn(&columns[0], &columns[1]).map(Some),
170        _ => {
171            // the try_reduce_with is a bit slower in parallelism,
172            // but I don't think it matters here as we parallelize over columns, not over elements
173            RAYON.install(|| {
174                columns
175                    .par_iter()
176                    .map(|s| Ok(Cow::Borrowed(s)))
177                    .try_reduce_with(|l, r| max_fn(&l, &r).map(Cow::Owned))
178                    // we can unwrap the option, because we are certain there is a column
179                    // we started this operation on 3 columns
180                    .unwrap()
181                    .map(|cow| Some(cow.into_owned()))
182            })
183        },
184    }
185}
186
187pub fn min_horizontal(columns: &[Column]) -> PolarsResult<Option<Column>> {
188    broadcast_len(columns.iter()).context("min_horizontal")?;
189
190    let min_fn = |acc: &Column, s: &Column| min_max_binary_columns(acc, s, true);
191
192    match columns.len() {
193        0 => Ok(None),
194        1 => Ok(Some(columns[0].clone())),
195        2 => min_fn(&columns[0], &columns[1]).map(Some),
196        _ => {
197            // the try_reduce_with is a bit slower in parallelism,
198            // but I don't think it matters here as we parallelize over columns, not over elements
199            RAYON.install(|| {
200                columns
201                    .par_iter()
202                    .map(|s| Ok(Cow::Borrowed(s)))
203                    .try_reduce_with(|l, r| min_fn(&l, &r).map(Cow::Owned))
204                    // we can unwrap the option, because we are certain there is a column
205                    // we started this operation on 3 columns
206                    .unwrap()
207                    .map(|cow| Some(cow.into_owned()))
208            })
209        },
210    }
211}
212
213pub fn sum_horizontal(
214    columns: &[Column],
215    null_strategy: NullStrategy,
216) -> PolarsResult<Option<Column>> {
217    broadcast_len(columns.iter()).context("sum_horizontal")?;
218    let ignore_nulls = null_strategy == NullStrategy::Ignore;
219
220    let apply_null_strategy = |s: Series| -> PolarsResult<Series> {
221        if ignore_nulls && s.null_count() > 0 {
222            s.fill_null(FillNullStrategy::Zero)
223        } else {
224            Ok(s)
225        }
226    };
227
228    let sum_fn = |acc: Series, s: Series| -> PolarsResult<Series> {
229        let acc: Series = apply_null_strategy(acc)?;
230        let s = apply_null_strategy(s)?;
231        // This will do owned arithmetic and can be mutable
232        std::ops::Add::add(acc, s)
233    };
234
235    // @scalar-opt
236    let non_null_cols = columns
237        .iter()
238        .filter(|x| x.dtype() != &DataType::Null)
239        .collect::<Vec<_>>();
240    let out_dtype = sum_output_dtype(&dtypes_to_supertype(
241        non_null_cols.iter().map(|c| c.dtype()),
242    )?);
243
244    // If we have any null columns and null strategy is not `Ignore`, we can return immediately.
245    if !ignore_nulls && non_null_cols.len() < columns.len() {
246        return Ok(Some(Column::full_null(
247            columns[0].name().clone(),
248            columns[0].len(),
249            &out_dtype,
250        )));
251    }
252
253    let non_null_cols = non_null_cols
254        .into_iter()
255        .map(|c| c.as_materialized_series().cast(&out_dtype))
256        .collect::<PolarsResult<Vec<_>>>()?;
257
258    match non_null_cols.len() {
259        0 => {
260            if columns.is_empty() {
261                Ok(None)
262            } else {
263                // all columns are null dtype, so result is null dtype
264                Ok(Some(columns[0].clone()))
265            }
266        },
267        1 => {
268            let [s] = <[Series; 1]>::try_from(non_null_cols).unwrap();
269            Ok(Some(apply_null_strategy(s)?.into()))
270        },
271        2 => {
272            let [a, b] = <[Series; 2]>::try_from(non_null_cols).unwrap();
273            sum_fn(a, b).map(Column::from).map(Some)
274        },
275        _ => {
276            // the try_reduce_with is a bit slower in parallelism,
277            // but I don't think it matters here as we parallelize over columns, not over elements
278            let out = RAYON.install(|| {
279                non_null_cols
280                    .into_par_iter()
281                    .map(Ok)
282                    .try_reduce_with(sum_fn)
283                    // We can unwrap because we started with at least 3 columns, so we always get a Some
284                    .unwrap()
285            });
286            out.map(Column::from).map(Some)
287        },
288    }
289}
290
291pub fn mean_horizontal(
292    columns: &[Column],
293    null_strategy: NullStrategy,
294) -> PolarsResult<Option<Column>> {
295    broadcast_len(columns.iter()).context("mean_horizontal")?;
296
297    let (numeric_columns, non_numeric_columns): (Vec<_>, Vec<_>) = columns.iter().partition(|s| {
298        let dtype = s.dtype();
299        dtype.is_primitive_numeric() || dtype.is_decimal() || dtype.is_bool() || dtype.is_null()
300    });
301
302    if !non_numeric_columns.is_empty() {
303        let col = non_numeric_columns.first().cloned();
304        polars_bail!(
305            InvalidOperation: "'horizontal_mean' expects numeric expressions, found {:?} (dtype={})",
306            col.unwrap().name(),
307            col.unwrap().dtype(),
308        );
309    }
310    let columns = numeric_columns.into_iter().cloned().collect::<Vec<_>>();
311    let num_rows = columns.len();
312    match num_rows {
313        0 => Ok(None),
314        1 => Ok(Some(match columns[0].dtype() {
315            dt if !matches!(dt, DataType::Float16 | DataType::Float32) && !dt.is_decimal() => {
316                columns[0].cast(&DataType::Float64)?
317            },
318            _ => columns[0].clone(),
319        })),
320        _ => {
321            let sum = || sum_horizontal(columns.as_slice(), null_strategy);
322            let null_count = || {
323                columns
324                    .par_iter()
325                    .map(|c| {
326                        c.is_null()
327                            .into_column()
328                            .cast_with_options(&DataType::UInt32, CastOptions::NonStrict)
329                    })
330                    .reduce_with(|l, r| {
331                        let l = l?;
332                        let r = r?;
333                        let result = std::ops::Add::add(&l, &r)?;
334                        PolarsResult::Ok(result)
335                    })
336                    // we can unwrap the option, because we are certain there is a column
337                    // we started this operation on 2 columns
338                    .unwrap()
339            };
340
341            let (sum, null_count) = RAYON.install(|| rayon::join(sum, null_count));
342            let sum = sum?;
343            let null_count = null_count?;
344
345            // value lengths: len - null_count
346            let value_length: UInt32Chunked = (Column::new_scalar(
347                PlSmallStr::EMPTY,
348                Scalar::from(num_rows as u32),
349                null_count.len(),
350            ) - null_count)?
351                .u32()
352                .unwrap()
353                .clone();
354
355            // make sure that we do not divide by zero
356            // by replacing with None
357            let dt = sum
358                .as_ref()
359                .map(Column::dtype)
360                .filter(|dt| matches!(dt, DataType::Float16 | DataType::Float32))
361                .unwrap_or(&DataType::Float64);
362            let value_length = value_length
363                .set(&value_length.equal(0), None)?
364                .into_column()
365                .cast(dt)?;
366
367            sum.map(|sum| std::ops::Div::div(&sum, &value_length))
368                .transpose()
369        },
370    }
371}
372
373pub fn coalesce_columns(s: &[Column]) -> PolarsResult<Column> {
374    // Each iteration recomputes the null mask and allocates a new `out`; a single
375    // pass over all inputs would be faster for many inputs with many nulls.
376    polars_ensure!(!s.is_empty(), NoData: "cannot coalesce empty list");
377    let len = broadcast_len(s)?;
378    let mut out = s[0].clone();
379    for next in &s[1..] {
380        if !out.has_nulls() {
381            break;
382        }
383        if out.null_count() == out.len() {
384            out = next.clone().with_name(out.name().clone());
385            continue;
386        }
387        let mask = out.is_not_null();
388        out = out
389            .as_materialized_series()
390            .zip_with_same_type(&mask, next.as_materialized_series())?
391            .into();
392    }
393    // The early exits skip the broadcast `zip_with` would have done.
394    if out.len() != len {
395        out = out.new_from_index(0, len);
396    }
397    Ok(out)
398}
399
400#[cfg(test)]
401mod tests {
402    use super::*;
403
404    #[test]
405    #[cfg_attr(miri, ignore)]
406    fn test_horizontal_agg() {
407        let a = Column::new("a".into(), [1, 2, 6]);
408        let b = Column::new("b".into(), [Some(1), None, None]);
409        let c = Column::new("c".into(), [Some(4), None, Some(3)]);
410
411        let df = DataFrame::new_infer_height(vec![a, b, c]).unwrap();
412        assert_eq!(
413            Vec::from(
414                df.mean_horizontal(NullStrategy::Ignore)
415                    .unwrap()
416                    .unwrap()
417                    .f64()
418                    .unwrap()
419            ),
420            &[Some(2.0), Some(2.0), Some(4.5)]
421        );
422        assert_eq!(
423            Vec::from(
424                df.sum_horizontal(NullStrategy::Ignore)
425                    .unwrap()
426                    .unwrap()
427                    .i32()
428                    .unwrap()
429            ),
430            &[Some(6), Some(2), Some(9)]
431        );
432        assert_eq!(
433            Vec::from(df.min_horizontal().unwrap().unwrap().i32().unwrap()),
434            &[Some(1), Some(2), Some(3)]
435        );
436        assert_eq!(
437            Vec::from(df.max_horizontal().unwrap().unwrap().i32().unwrap()),
438            &[Some(4), Some(2), Some(6)]
439        );
440    }
441}