polars_core/frame/group_by/
mod.rs1use std::fmt::{Debug, Display, Formatter};
2use std::hash::Hash;
3
4use num_traits::NumCast;
5use polars_compute::rolling::QuantileMethod;
6use polars_utils::broadcast::broadcast_len;
7use polars_utils::format_pl_smallstr;
8use polars_utils::hashing::DirtyHash;
9use polars_utils::slice_enum::Slice;
10use rayon::prelude::*;
11
12use self::hashing::*;
13use crate::prelude::*;
14use crate::runtime::RAYON;
15use crate::utils::{_set_partition_size, accumulate_dataframes_vertical};
16
17pub mod aggregations;
18pub(crate) mod hashing;
19mod into_groups;
20mod position;
21
22pub use into_groups::*;
23pub use position::*;
24
25use crate::chunked_array::ops::row_encode::{
26 encode_rows_with_layout, encode_rows_with_layout_par, get_optimal_row_encoding_layout,
27};
28
29impl DataFrame {
30 pub fn group_by_with_series(
31 &self,
32 mut by: Vec<Column>,
33 multithreaded: bool,
34 sorted: bool,
35 ) -> PolarsResult<GroupBy<'_>> {
36 polars_ensure!(
37 !by.is_empty(),
38 ComputeError: "at least one key is required in a group_by operation"
39 );
40
41 let common_height = if self.width() > 0 {
45 self.height()
46 } else {
47 broadcast_len(by.iter()).context("group_by key")?
48 };
49 for by_key in by.iter_mut() {
50 by_key
51 .broadcast_in_place_to(common_height)
52 .context("group_by keys should have the same length as the DataFrame")?;
53 }
54
55 let groups = if by.len() == 1 {
56 let column = &by[0];
57 column
58 .as_materialized_series()
59 .group_tuples(multithreaded, sorted)
60 } else if by.iter().any(|s| s.dtype().is_object()) {
61 #[cfg(feature = "object")]
62 {
63 let mut df = DataFrame::new(self.height(), by.clone()).unwrap();
64 let n = df.height();
65 let rows = df.to_av_rows();
66 let iter = (0..n).map(|i| rows.get(i));
67 Ok(group_by(iter, sorted))
68 }
69 #[cfg(not(feature = "object"))]
70 {
71 unreachable!()
72 }
73 } else {
74 let by = by
76 .iter()
77 .filter(|s| !s.dtype().is_null())
78 .cloned()
79 .collect::<Vec<_>>();
80 if by.is_empty() {
81 let groups = if self.height() == 0 {
82 vec![]
83 } else {
84 vec![[0, self.height() as IdxSize]]
85 };
86
87 Ok(GroupsType::new_slice(groups, false, true))
88 } else {
89 let opts = vec![RowEncodingOptions::new_unsorted(); by.len()];
90 let layout = get_optimal_row_encoding_layout(&by, &[], &opts);
91 let rows = if multithreaded {
92 encode_rows_with_layout_par(&by, &layout, false)?
93 } else {
94 encode_rows_with_layout(&by, &layout, false)?
95 };
96 rows.group_tuples(multithreaded, sorted)
97 }
98 };
99 Ok(GroupBy::new(self, by, groups?.into_sliceable(), None))
100 }
101
102 pub fn group_by<I, S>(&self, by: I) -> PolarsResult<GroupBy<'_>>
115 where
116 I: IntoIterator<Item = S>,
117 S: AsRef<str>,
118 {
119 let selected_keys = self.select_to_vec(by)?;
120 self.group_by_with_series(selected_keys, true, false)
121 }
122
123 pub fn group_by_stable<I, S>(&self, by: I) -> PolarsResult<GroupBy<'_>>
126 where
127 I: IntoIterator<Item = S>,
128 S: AsRef<str>,
129 {
130 let selected_keys = self.select_to_vec(by)?;
131 self.group_by_with_series(selected_keys, true, true)
132 }
133}
134
135#[derive(Debug, Clone)]
185pub struct GroupBy<'a> {
186 pub df: &'a DataFrame,
187 pub(crate) selected_keys: Vec<Column>,
188 groups: GroupPositions,
190 pub(crate) selected_agg: Option<Vec<PlSmallStr>>,
192}
193
194impl<'a> GroupBy<'a> {
195 pub fn new(
196 df: &'a DataFrame,
197 by: Vec<Column>,
198 groups: GroupPositions,
199 selected_agg: Option<Vec<PlSmallStr>>,
200 ) -> Self {
201 GroupBy {
202 df,
203 selected_keys: by,
204 groups,
205 selected_agg,
206 }
207 }
208
209 #[must_use]
215 pub fn select<I: IntoIterator<Item = S>, S: Into<PlSmallStr>>(mut self, selection: I) -> Self {
216 self.selected_agg = Some(selection.into_iter().map(|s| s.into()).collect());
217 self
218 }
219
220 pub fn get_groups(&self) -> &GroupPositions {
225 &self.groups
226 }
227
228 pub fn into_groups(self) -> GroupPositions {
229 self.groups
230 }
231
232 pub fn keys_sliced(&self, slice: Option<Slice>) -> Vec<Column> {
233 #[allow(unused_assignments)]
234 let mut groups_owned = None;
236
237 let groups = if let Some(slice) = slice {
238 groups_owned = Some(self.groups.slice(slice));
239 groups_owned.as_deref().unwrap()
240 } else {
241 &self.groups
242 };
243 RAYON.install(|| {
244 self.selected_keys
245 .par_iter()
246 .map(Column::as_materialized_series)
247 .map(|s| {
248 match groups {
249 GroupsType::Idx(groups) => {
250 let mut out = unsafe { s.take_slice_unchecked(groups.first()) };
252 if groups.groups_sorted_by_first_idx {
253 out.set_sorted_flag(s.is_sorted_flag());
254 };
255 out
256 },
257 GroupsType::Slice {
258 groups,
259 overlapping,
260 monotonic: _,
261 } => {
262 if *overlapping && !groups.is_empty() {
263 let offset = groups[0][0];
265 let [upper_offset, upper_len] = groups[groups.len() - 1];
266 return s.slice((
267 offset as usize,
268 ((upper_offset + upper_len) - offset) as usize,
269 ));
270 }
271
272 let indices = groups
273 .iter()
274 .map(|&[first, _len]| first)
275 .collect_ca(PlSmallStr::EMPTY);
276 let mut out = unsafe { s.take_unchecked(&indices) };
278 out.set_sorted_flag(s.is_sorted_flag());
280 out
281 },
282 }
283 })
284 .map(Column::from)
285 .collect()
286 })
287 }
288
289 pub fn keys(&self) -> Vec<Column> {
290 self.keys_sliced(None)
291 }
292
293 fn prepare_agg(&self) -> PolarsResult<(Vec<Column>, Vec<Column>)> {
294 let keys = self.keys();
295
296 let agg_col = match &self.selected_agg {
297 Some(selection) => self.df.select_to_vec(selection),
298 None => {
299 let by: Vec<_> = self.selected_keys.iter().map(|s| s.name()).collect();
300 let selection = self
301 .df
302 .columns()
303 .iter()
304 .map(|s| s.name())
305 .filter(|a| !by.contains(a))
306 .cloned()
307 .collect::<Vec<_>>();
308
309 self.df.select_to_vec(selection.as_slice())
310 },
311 }?;
312
313 Ok((keys, agg_col))
314 }
315
316 pub fn count(&self) -> PolarsResult<DataFrame> {
342 let (mut cols, agg_cols) = self.prepare_agg()?;
343
344 for agg_col in agg_cols {
345 let new_name = fmt_group_by_column(
346 agg_col.name().as_str(),
347 GroupByMethod::Count {
348 include_nulls: true,
349 },
350 );
351 let mut ca = self.groups.group_count();
352 ca.rename(new_name);
353 cols.push(ca.into_column());
354 }
355 DataFrame::new_infer_height(cols)
356 }
357
358 pub fn groups(&self) -> PolarsResult<DataFrame> {
384 let mut cols = self.keys();
385 let mut column = self.groups.as_list_chunked();
386 let new_name = fmt_group_by_column("", GroupByMethod::Groups);
387 column.rename(new_name);
388 cols.push(column.into_column());
389 DataFrame::new_infer_height(cols)
390 }
391
392 fn prepare_apply(&self) -> PolarsResult<DataFrame> {
393 if let Some(agg) = &self.selected_agg {
394 if agg.is_empty() {
395 Ok(self.df.clone())
396 } else {
397 let mut new_cols = Vec::with_capacity(self.selected_keys.len() + agg.len());
398 new_cols.extend_from_slice(&self.selected_keys);
399 let cols = self.df.select_to_vec(agg.as_slice())?;
400 new_cols.extend(cols);
401 Ok(unsafe { DataFrame::new_unchecked(self.df.height(), new_cols) })
402 }
403 } else {
404 Ok(self.df.clone())
405 }
406 }
407
408 pub fn apply<F>(&self, f: F) -> PolarsResult<DataFrame>
410 where
411 F: FnMut(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
412 {
413 self.apply_sliced(None, f, None)
414 }
415
416 pub fn apply_sliced<F>(
417 &self,
418 slice: Option<Slice>,
419 mut f: F,
420 schema: Option<&SchemaRef>,
421 ) -> PolarsResult<DataFrame>
422 where
423 F: FnMut(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
424 {
425 if self.df.height() == 0 || self.get_groups().is_empty() {
427 if let Some(schema) = schema {
428 return Ok(DataFrame::empty_with_arc_schema(schema.clone()));
429 }
430
431 polars_bail!(ComputeError: "cannot group_by + apply on empty 'DataFrame'");
432 }
433
434 let df = self.prepare_apply()?;
435 let max_height = slice.and_then(|s| s.end_position());
436 let mut height = 0;
437 let mut dfs = Vec::with_capacity(self.get_groups().len());
438 for g in self.get_groups().iter() {
439 let sub_df = unsafe { take_df(&df, g) };
441 let df = f(sub_df)?;
442 height += df.height();
443 dfs.push(df);
444
445 if max_height.is_some_and(|max_height| height >= max_height) {
448 break;
449 }
450 }
451
452 let mut df = accumulate_dataframes_vertical(dfs)?;
453 if let Some(slice) = slice {
454 df = df.slice(slice);
455 }
456 Ok(df)
457 }
458}
459
460unsafe fn take_df(df: &DataFrame, g: GroupsIndicator) -> DataFrame {
461 match g {
462 GroupsIndicator::Idx(idx) => df.take_slice_unchecked(idx.1),
463 GroupsIndicator::Slice([first, len]) => df.slice((first as usize, len as usize)),
464 }
465}
466
467#[derive(Copy, Clone, Debug)]
468pub enum GroupByMethod {
469 Min,
470 NanMin,
471 Max,
472 NanMax,
473 Median,
474 Mean,
475 First,
476 FirstNonNull,
477 Last,
478 LastNonNull,
479 Item { allow_empty: bool },
480 Sum { null_on_empty: bool },
481 Groups,
482 NUnique,
483 Quantile(f64, QuantileMethod),
484 Count { include_nulls: bool },
485 Implode { maintain_order: bool },
486 Std(u8),
487 Var(u8),
488 ArgMin,
489 ArgMax,
490}
491
492impl Display for GroupByMethod {
493 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
494 use GroupByMethod::*;
495 let s = match self {
496 Min => "min",
497 NanMin => "nan_min",
498 Max => "max",
499 NanMax => "nan_max",
500 Median => "median",
501 Mean => "mean",
502 First => "first",
503 FirstNonNull => "first_non_null",
504 Last => "last",
505 LastNonNull => "last_non_null",
506 Item { .. } => "item",
507 Sum { .. } => "sum",
508 Groups => "groups",
509 NUnique => "n_unique",
510 Quantile(_, _) => "quantile",
511 Count { .. } => "count",
512 Implode { .. } => "implode",
513 Std(_) => "std",
514 Var(_) => "var",
515 ArgMin => "arg_min",
516 ArgMax => "arg_max",
517 };
518 write!(f, "{s}")
519 }
520}
521
522pub fn fmt_group_by_column(name: &str, method: GroupByMethod) -> PlSmallStr {
524 use GroupByMethod::*;
525 match method {
526 Min => format_pl_smallstr!("{name}_min"),
527 Max => format_pl_smallstr!("{name}_max"),
528 NanMin => format_pl_smallstr!("{name}_nan_min"),
529 NanMax => format_pl_smallstr!("{name}_nan_max"),
530 Median => format_pl_smallstr!("{name}_median"),
531 Mean => format_pl_smallstr!("{name}_mean"),
532 First => format_pl_smallstr!("{name}_first"),
533 FirstNonNull => format_pl_smallstr!("{name}_first_non_null"),
534 Last => format_pl_smallstr!("{name}_last"),
535 LastNonNull => format_pl_smallstr!("{name}_last_non_null"),
536 Item { .. } => format_pl_smallstr!("{name}_item"),
537 Sum { .. } => format_pl_smallstr!("{name}_sum"),
538 Groups => PlSmallStr::from_static("groups"),
539 NUnique => format_pl_smallstr!("{name}_n_unique"),
540 Count { .. } => format_pl_smallstr!("{name}_count"),
541 Implode { .. } => format_pl_smallstr!("{name}_agg_list"),
542 Quantile(quantile, _interpol) => format_pl_smallstr!("{name}_quantile_{quantile:.2}"),
543 Std(_) => format_pl_smallstr!("{name}_agg_std"),
544 Var(_) => format_pl_smallstr!("{name}_agg_var"),
545 ArgMin => format_pl_smallstr!("{name}_arg_min"),
546 ArgMax => format_pl_smallstr!("{name}_arg_max"),
547 }
548}