1use std::borrow::Cow;
2
3use polars_arrow::array::PrimitiveArray;
4use polars_arrow::compute::utils::combine_validities_or;
5use polars_core::chunked_array::cast::CastOptions;
6use polars_core::prelude::*;
7use polars_core::runtime::RAYON;
8use polars_core::series::arithmetic::coerce_lhs_rhs;
9use polars_core::utils::{CustomIterTools, align_chunks_binary, dtypes_to_supertype};
10use polars_core::with_match_physical_numeric_polars_type;
11use polars_utils::broadcast::broadcast_len;
12use polars_utils::min_max::MinMax;
13use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
14
15pub trait MinMaxHorizontal {
16 fn min_horizontal(&self) -> PolarsResult<Option<Column>>;
18 fn max_horizontal(&self) -> PolarsResult<Option<Column>>;
20}
21
22impl MinMaxHorizontal for DataFrame {
23 fn min_horizontal(&self) -> PolarsResult<Option<Column>> {
24 min_horizontal(self.columns())
25 }
26 fn max_horizontal(&self) -> PolarsResult<Option<Column>> {
27 max_horizontal(self.columns())
28 }
29}
30
31#[derive(Copy, Clone, Debug, PartialEq)]
32pub enum NullStrategy {
33 Ignore,
34 Propagate,
35}
36
37pub trait SumMeanHorizontal {
38 fn sum_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>>;
40
41 fn mean_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>>;
43}
44
45impl SumMeanHorizontal for DataFrame {
46 fn sum_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>> {
47 sum_horizontal(self.columns(), null_strategy)
48 }
49 fn mean_horizontal(&self, null_strategy: NullStrategy) -> PolarsResult<Option<Column>> {
50 mean_horizontal(self.columns(), null_strategy)
51 }
52}
53
54fn fill_null_from<'a>(s: &'a Series, other: &Series) -> PolarsResult<Cow<'a, Series>> {
56 if s.has_nulls() {
57 s.zip_with_same_type(&s.is_not_null(), other)
58 .map(Cow::Owned)
59 } else {
60 Ok(Cow::Borrowed(s))
61 }
62}
63
64fn binary_ignore_nulls<T, F>(
66 left: &ChunkedArray<T>,
67 right: &ChunkedArray<T>,
68 op: F,
69) -> ChunkedArray<T>
70where
71 T: PolarsNumericType,
72 F: Fn(T::Native, T::Native) -> T::Native,
73{
74 let (left, right) = align_chunks_binary(left, right);
75 let chunks = left
76 .downcast_iter()
77 .zip(right.downcast_iter())
78 .map(|(l, r)| {
79 let lv = l.values().iter().copied();
80 let rv = r.values().iter().copied();
81 let values: Vec<T::Native> = match (l.validity(), r.validity()) {
82 (Some(lm), Some(rm)) => lv
83 .zip(rv)
84 .zip(lm.iter().zip(rm.iter()))
85 .map(|((a, b), (a_valid, b_valid))| {
86 op(if a_valid { a } else { b }, if b_valid { b } else { a })
87 })
88 .collect_trusted(),
89 (Some(lm), None) => lv
90 .zip(rv)
91 .zip(lm.iter())
92 .map(|((a, b), a_valid)| op(if a_valid { a } else { b }, b))
93 .collect_trusted(),
94 (None, Some(rm)) => lv
95 .zip(rv)
96 .zip(rm.iter())
97 .map(|((a, b), b_valid)| op(a, if b_valid { b } else { a }))
98 .collect_trusted(),
99 (None, None) => lv.zip(rv).map(|(a, b)| op(a, b)).collect_trusted(),
100 };
101 let validity = combine_validities_or(l.validity(), r.validity());
102 PrimitiveArray::from_vec(values).with_validity(validity)
103 });
104 ChunkedArray::from_chunk_iter(left.name().clone(), chunks)
105}
106
107fn min_max_binary_columns(left: &Column, right: &Column, min: bool) -> PolarsResult<Column> {
108 if left.dtype().to_physical().is_primitive_numeric()
109 && right.dtype().to_physical().is_primitive_numeric()
110 {
111 let height = broadcast_len([left, right])?;
112 let left_s = left.as_materialized_series_maintain_scalar();
113 let right_s = right.as_materialized_series_maintain_scalar();
114 let (lhs, rhs) = coerce_lhs_rhs(&left_s, &right_s)?;
115 let logical = lhs.dtype();
116
117 let lhs = lhs.to_physical_repr();
118 let rhs = rhs.to_physical_repr();
119 let has_nulls = lhs.has_nulls() || rhs.has_nulls();
120 let same_len = lhs.len() == rhs.len();
121 let (lhs, rhs) = if has_nulls && !same_len {
122 (fill_null_from(&lhs, &rhs)?, fill_null_from(&rhs, &lhs)?)
123 } else {
124 (lhs, rhs)
125 };
126
127 with_match_physical_numeric_polars_type!(lhs.dtype(), |$T| {
128 let a: &ChunkedArray<$T> = lhs.as_ref().as_ref().as_ref();
129 let b: &ChunkedArray<$T> = rhs.as_ref().as_ref().as_ref();
130
131 let out: ChunkedArray<$T> = match (has_nulls && same_len, min) {
132 (true, true) => binary_ignore_nulls(a, b, MinMax::min_ignore_nan),
133 (true, false) => binary_ignore_nulls(a, b, MinMax::max_ignore_nan),
134 (false, true) => {
135 arity::broadcast_binary_elementwise_values(a, b, MinMax::min_ignore_nan)
136 },
137 (false, false) => {
138 arity::broadcast_binary_elementwise_values(a, b, MinMax::max_ignore_nan)
139 },
140 };
141 unsafe { out.into_series().from_physical_unchecked(logical) }
142 })
143 .and_then(|s| s.broadcast_owned_to(height))
144 .map(Column::from)
145 } else {
146 let mut mask = if min {
147 left.lt(right)?
148 } else {
149 left.gt(right)?
150 };
151 if left.has_nulls() {
152 mask = mask & left.is_not_null();
153 }
154 if right.has_nulls() {
155 mask = mask | right.is_null();
156 }
157 left.zip_with(&mask, right)
158 }
159}
160
161pub fn max_horizontal(columns: &[Column]) -> PolarsResult<Option<Column>> {
162 broadcast_len(columns.iter()).context("max_horizontal")?;
163
164 let max_fn = |acc: &Column, s: &Column| min_max_binary_columns(acc, s, false);
165
166 match columns.len() {
167 0 => Ok(None),
168 1 => Ok(Some(columns[0].clone())),
169 2 => max_fn(&columns[0], &columns[1]).map(Some),
170 _ => {
171 RAYON.install(|| {
174 columns
175 .par_iter()
176 .map(|s| Ok(Cow::Borrowed(s)))
177 .try_reduce_with(|l, r| max_fn(&l, &r).map(Cow::Owned))
178 .unwrap()
181 .map(|cow| Some(cow.into_owned()))
182 })
183 },
184 }
185}
186
187pub fn min_horizontal(columns: &[Column]) -> PolarsResult<Option<Column>> {
188 broadcast_len(columns.iter()).context("min_horizontal")?;
189
190 let min_fn = |acc: &Column, s: &Column| min_max_binary_columns(acc, s, true);
191
192 match columns.len() {
193 0 => Ok(None),
194 1 => Ok(Some(columns[0].clone())),
195 2 => min_fn(&columns[0], &columns[1]).map(Some),
196 _ => {
197 RAYON.install(|| {
200 columns
201 .par_iter()
202 .map(|s| Ok(Cow::Borrowed(s)))
203 .try_reduce_with(|l, r| min_fn(&l, &r).map(Cow::Owned))
204 .unwrap()
207 .map(|cow| Some(cow.into_owned()))
208 })
209 },
210 }
211}
212
213pub fn sum_horizontal(
214 columns: &[Column],
215 null_strategy: NullStrategy,
216) -> PolarsResult<Option<Column>> {
217 broadcast_len(columns.iter()).context("sum_horizontal")?;
218 let ignore_nulls = null_strategy == NullStrategy::Ignore;
219
220 let apply_null_strategy = |s: Series| -> PolarsResult<Series> {
221 if ignore_nulls && s.null_count() > 0 {
222 s.fill_null(FillNullStrategy::Zero)
223 } else {
224 Ok(s)
225 }
226 };
227
228 let sum_fn = |acc: Series, s: Series| -> PolarsResult<Series> {
229 let acc: Series = apply_null_strategy(acc)?;
230 let s = apply_null_strategy(s)?;
231 std::ops::Add::add(acc, s)
233 };
234
235 let non_null_cols = columns
237 .iter()
238 .filter(|x| x.dtype() != &DataType::Null)
239 .collect::<Vec<_>>();
240 let out_dtype = sum_output_dtype(&dtypes_to_supertype(
241 non_null_cols.iter().map(|c| c.dtype()),
242 )?);
243
244 if !ignore_nulls && non_null_cols.len() < columns.len() {
246 return Ok(Some(Column::full_null(
247 columns[0].name().clone(),
248 columns[0].len(),
249 &out_dtype,
250 )));
251 }
252
253 let non_null_cols = non_null_cols
254 .into_iter()
255 .map(|c| c.as_materialized_series().cast(&out_dtype))
256 .collect::<PolarsResult<Vec<_>>>()?;
257
258 match non_null_cols.len() {
259 0 => {
260 if columns.is_empty() {
261 Ok(None)
262 } else {
263 Ok(Some(columns[0].clone()))
265 }
266 },
267 1 => {
268 let [s] = <[Series; 1]>::try_from(non_null_cols).unwrap();
269 Ok(Some(apply_null_strategy(s)?.into()))
270 },
271 2 => {
272 let [a, b] = <[Series; 2]>::try_from(non_null_cols).unwrap();
273 sum_fn(a, b).map(Column::from).map(Some)
274 },
275 _ => {
276 let out = RAYON.install(|| {
279 non_null_cols
280 .into_par_iter()
281 .map(Ok)
282 .try_reduce_with(sum_fn)
283 .unwrap()
285 });
286 out.map(Column::from).map(Some)
287 },
288 }
289}
290
291pub fn mean_horizontal(
292 columns: &[Column],
293 null_strategy: NullStrategy,
294) -> PolarsResult<Option<Column>> {
295 broadcast_len(columns.iter()).context("mean_horizontal")?;
296
297 let (numeric_columns, non_numeric_columns): (Vec<_>, Vec<_>) = columns.iter().partition(|s| {
298 let dtype = s.dtype();
299 dtype.is_primitive_numeric() || dtype.is_decimal() || dtype.is_bool() || dtype.is_null()
300 });
301
302 if !non_numeric_columns.is_empty() {
303 let col = non_numeric_columns.first().cloned();
304 polars_bail!(
305 InvalidOperation: "'horizontal_mean' expects numeric expressions, found {:?} (dtype={})",
306 col.unwrap().name(),
307 col.unwrap().dtype(),
308 );
309 }
310 let columns = numeric_columns.into_iter().cloned().collect::<Vec<_>>();
311 let num_rows = columns.len();
312 match num_rows {
313 0 => Ok(None),
314 1 => Ok(Some(match columns[0].dtype() {
315 dt if !matches!(dt, DataType::Float16 | DataType::Float32) && !dt.is_decimal() => {
316 columns[0].cast(&DataType::Float64)?
317 },
318 _ => columns[0].clone(),
319 })),
320 _ => {
321 let sum = || sum_horizontal(columns.as_slice(), null_strategy);
322 let null_count = || {
323 columns
324 .par_iter()
325 .map(|c| {
326 c.is_null()
327 .into_column()
328 .cast_with_options(&DataType::UInt32, CastOptions::NonStrict)
329 })
330 .reduce_with(|l, r| {
331 let l = l?;
332 let r = r?;
333 let result = std::ops::Add::add(&l, &r)?;
334 PolarsResult::Ok(result)
335 })
336 .unwrap()
339 };
340
341 let (sum, null_count) = RAYON.install(|| rayon::join(sum, null_count));
342 let sum = sum?;
343 let null_count = null_count?;
344
345 let value_length: UInt32Chunked = (Column::new_scalar(
347 PlSmallStr::EMPTY,
348 Scalar::from(num_rows as u32),
349 null_count.len(),
350 ) - null_count)?
351 .u32()
352 .unwrap()
353 .clone();
354
355 let dt = sum
358 .as_ref()
359 .map(Column::dtype)
360 .filter(|dt| matches!(dt, DataType::Float16 | DataType::Float32))
361 .unwrap_or(&DataType::Float64);
362 let value_length = value_length
363 .set(&value_length.equal(0), None)?
364 .into_column()
365 .cast(dt)?;
366
367 sum.map(|sum| std::ops::Div::div(&sum, &value_length))
368 .transpose()
369 },
370 }
371}
372
373pub fn coalesce_columns(s: &[Column]) -> PolarsResult<Column> {
374 polars_ensure!(!s.is_empty(), NoData: "cannot coalesce empty list");
377 let len = broadcast_len(s)?;
378 let mut out = s[0].clone();
379 for next in &s[1..] {
380 if !out.has_nulls() {
381 break;
382 }
383 if out.null_count() == out.len() {
384 out = next.clone().with_name(out.name().clone());
385 continue;
386 }
387 let mask = out.is_not_null();
388 out = out
389 .as_materialized_series()
390 .zip_with_same_type(&mask, next.as_materialized_series())?
391 .into();
392 }
393 if out.len() != len {
395 out = out.new_from_index(0, len);
396 }
397 Ok(out)
398}
399
400#[cfg(test)]
401mod tests {
402 use super::*;
403
404 #[test]
405 #[cfg_attr(miri, ignore)]
406 fn test_horizontal_agg() {
407 let a = Column::new("a".into(), [1, 2, 6]);
408 let b = Column::new("b".into(), [Some(1), None, None]);
409 let c = Column::new("c".into(), [Some(4), None, Some(3)]);
410
411 let df = DataFrame::new_infer_height(vec![a, b, c]).unwrap();
412 assert_eq!(
413 Vec::from(
414 df.mean_horizontal(NullStrategy::Ignore)
415 .unwrap()
416 .unwrap()
417 .f64()
418 .unwrap()
419 ),
420 &[Some(2.0), Some(2.0), Some(4.5)]
421 );
422 assert_eq!(
423 Vec::from(
424 df.sum_horizontal(NullStrategy::Ignore)
425 .unwrap()
426 .unwrap()
427 .i32()
428 .unwrap()
429 ),
430 &[Some(6), Some(2), Some(9)]
431 );
432 assert_eq!(
433 Vec::from(df.min_horizontal().unwrap().unwrap().i32().unwrap()),
434 &[Some(1), Some(2), Some(3)]
435 );
436 assert_eq!(
437 Vec::from(df.max_horizontal().unwrap().unwrap().i32().unwrap()),
438 &[Some(4), Some(2), Some(6)]
439 );
440 }
441}