Skip to main content

polars_core/frame/group_by/
mod.rs

1use std::fmt::{Debug, Display, Formatter};
2use std::hash::Hash;
3
4use num_traits::NumCast;
5use polars_compute::rolling::QuantileMethod;
6use polars_utils::broadcast::broadcast_len;
7use polars_utils::format_pl_smallstr;
8use polars_utils::hashing::DirtyHash;
9use polars_utils::slice_enum::Slice;
10use rayon::prelude::*;
11
12use self::hashing::*;
13use crate::prelude::*;
14use crate::runtime::RAYON;
15use crate::utils::{_set_partition_size, accumulate_dataframes_vertical};
16
17pub mod aggregations;
18pub(crate) mod hashing;
19mod into_groups;
20mod position;
21
22pub use into_groups::*;
23pub use position::*;
24
25use crate::chunked_array::ops::row_encode::{
26    encode_rows_with_layout, encode_rows_with_layout_par, get_optimal_row_encoding_layout,
27};
28
29impl DataFrame {
30    pub fn group_by_with_series(
31        &self,
32        mut by: Vec<Column>,
33        multithreaded: bool,
34        sorted: bool,
35    ) -> PolarsResult<GroupBy<'_>> {
36        polars_ensure!(
37            !by.is_empty(),
38            ComputeError: "at least one key is required in a group_by operation"
39        );
40
41        // Ensure all 'by' columns have the same common_height
42        // The condition self.width > 0 ensures we can still call this on a
43        // dummy dataframe where we provide the keys
44        let common_height = if self.width() > 0 {
45            self.height()
46        } else {
47            broadcast_len(by.iter()).context("group_by key")?
48        };
49        for by_key in by.iter_mut() {
50            by_key
51                .broadcast_in_place_to(common_height)
52                .context("group_by keys should have the same length as the DataFrame")?;
53        }
54
55        let groups = if by.len() == 1 {
56            let column = &by[0];
57            column
58                .as_materialized_series()
59                .group_tuples(multithreaded, sorted)
60        } else if by.iter().any(|s| s.dtype().is_object()) {
61            #[cfg(feature = "object")]
62            {
63                let mut df = DataFrame::new(self.height(), by.clone()).unwrap();
64                let n = df.height();
65                let rows = df.to_av_rows();
66                let iter = (0..n).map(|i| rows.get(i));
67                Ok(group_by(iter, sorted))
68            }
69            #[cfg(not(feature = "object"))]
70            {
71                unreachable!()
72            }
73        } else {
74            // Skip null dtype.
75            let by = by
76                .iter()
77                .filter(|s| !s.dtype().is_null())
78                .cloned()
79                .collect::<Vec<_>>();
80            if by.is_empty() {
81                let groups = if self.height() == 0 {
82                    vec![]
83                } else {
84                    vec![[0, self.height() as IdxSize]]
85                };
86
87                Ok(GroupsType::new_slice(groups, false, true))
88            } else {
89                let opts = vec![RowEncodingOptions::new_unsorted(); by.len()];
90                let layout = get_optimal_row_encoding_layout(&by, &[], &opts);
91                let rows = if multithreaded {
92                    encode_rows_with_layout_par(&by, &layout, false)?
93                } else {
94                    encode_rows_with_layout(&by, &layout, false)?
95                };
96                rows.group_tuples(multithreaded, sorted)
97            }
98        };
99        Ok(GroupBy::new(self, by, groups?.into_sliceable(), None))
100    }
101
102    /// Group DataFrame using a Series column.
103    ///
104    /// # Example
105    ///
106    /// ```
107    /// use polars_core::prelude::*;
108    /// fn group_by_example(df: &DataFrame) -> PolarsResult<DataFrame> {
109    ///     df.group_by(["column_name"])?
110    ///     .select(["agg_column_name"])
111    ///     .groups()
112    /// }
113    /// ```
114    pub fn group_by<I, S>(&self, by: I) -> PolarsResult<GroupBy<'_>>
115    where
116        I: IntoIterator<Item = S>,
117        S: AsRef<str>,
118    {
119        let selected_keys = self.select_to_vec(by)?;
120        self.group_by_with_series(selected_keys, true, false)
121    }
122
123    /// Group DataFrame using a Series column.
124    /// The groups are ordered by their smallest row index.
125    pub fn group_by_stable<I, S>(&self, by: I) -> PolarsResult<GroupBy<'_>>
126    where
127        I: IntoIterator<Item = S>,
128        S: AsRef<str>,
129    {
130        let selected_keys = self.select_to_vec(by)?;
131        self.group_by_with_series(selected_keys, true, true)
132    }
133}
134
135/// Returned by a group_by operation on a DataFrame. This struct supports
136/// several aggregations.
137///
138/// Until described otherwise, the examples in this struct are performed on the following DataFrame:
139///
140/// ```ignore
141/// use polars_core::prelude::*;
142///
143/// let dates = &[
144/// "2020-08-21",
145/// "2020-08-21",
146/// "2020-08-22",
147/// "2020-08-23",
148/// "2020-08-22",
149/// ];
150/// // date format
151/// let fmt = "%Y-%m-%d";
152/// // create date series
153/// let s0 = DateChunked::parse_from_str_slice("date", dates, fmt)
154///         .into_series();
155/// // create temperature series
156/// let s1 = Series::new("temp".into(), [20, 10, 7, 9, 1]);
157/// // create rain series
158/// let s2 = Series::new("rain".into(), [0.2, 0.1, 0.3, 0.1, 0.01]);
159/// // create a new DataFrame
160/// let df = DataFrame::new_infer_height(vec![s0, s1, s2]).unwrap();
161/// println!("{:?}", df);
162/// ```
163///
164/// Outputs:
165///
166/// ```text
167/// +------------+------+------+
168/// | date       | temp | rain |
169/// | ---        | ---  | ---  |
170/// | Date       | i32  | f64  |
171/// +============+======+======+
172/// | 2020-08-21 | 20   | 0.2  |
173/// +------------+------+------+
174/// | 2020-08-21 | 10   | 0.1  |
175/// +------------+------+------+
176/// | 2020-08-22 | 7    | 0.3  |
177/// +------------+------+------+
178/// | 2020-08-23 | 9    | 0.1  |
179/// +------------+------+------+
180/// | 2020-08-22 | 1    | 0.01 |
181/// +------------+------+------+
182/// ```
183///
184#[derive(Debug, Clone)]
185pub struct GroupBy<'a> {
186    pub df: &'a DataFrame,
187    pub(crate) selected_keys: Vec<Column>,
188    // [first idx, [other idx]]
189    groups: GroupPositions,
190    // columns selected for aggregation
191    pub(crate) selected_agg: Option<Vec<PlSmallStr>>,
192}
193
194impl<'a> GroupBy<'a> {
195    pub fn new(
196        df: &'a DataFrame,
197        by: Vec<Column>,
198        groups: GroupPositions,
199        selected_agg: Option<Vec<PlSmallStr>>,
200    ) -> Self {
201        GroupBy {
202            df,
203            selected_keys: by,
204            groups,
205            selected_agg,
206        }
207    }
208
209    /// Select the column(s) that should be aggregated.
210    /// You can select a single column or a slice of columns.
211    ///
212    /// Note that making a selection with this method is not required. If you
213    /// skip it all columns (except for the keys) will be selected for aggregation.
214    #[must_use]
215    pub fn select<I: IntoIterator<Item = S>, S: Into<PlSmallStr>>(mut self, selection: I) -> Self {
216        self.selected_agg = Some(selection.into_iter().map(|s| s.into()).collect());
217        self
218    }
219
220    /// Get the internal representation of the GroupBy operation.
221    /// The Vec returned contains:
222    ///     (first_idx, [`Vec<indexes>`])
223    ///     Where second value in the tuple is a vector with all matching indexes.
224    pub fn get_groups(&self) -> &GroupPositions {
225        &self.groups
226    }
227
228    pub fn into_groups(self) -> GroupPositions {
229        self.groups
230    }
231
232    pub fn keys_sliced(&self, slice: Option<Slice>) -> Vec<Column> {
233        #[allow(unused_assignments)]
234        // needed to keep the lifetimes valid for this scope
235        let mut groups_owned = None;
236
237        let groups = if let Some(slice) = slice {
238            groups_owned = Some(self.groups.slice(slice));
239            groups_owned.as_deref().unwrap()
240        } else {
241            &self.groups
242        };
243        RAYON.install(|| {
244            self.selected_keys
245                .par_iter()
246                .map(Column::as_materialized_series)
247                .map(|s| {
248                    match groups {
249                        GroupsType::Idx(groups) => {
250                            // SAFETY: groups are always in bounds.
251                            let mut out = unsafe { s.take_slice_unchecked(groups.first()) };
252                            if groups.groups_sorted_by_first_idx {
253                                out.set_sorted_flag(s.is_sorted_flag());
254                            };
255                            out
256                        },
257                        GroupsType::Slice {
258                            groups,
259                            overlapping,
260                            monotonic: _,
261                        } => {
262                            if *overlapping && !groups.is_empty() {
263                                // Groups can be sliced.
264                                let offset = groups[0][0];
265                                let [upper_offset, upper_len] = groups[groups.len() - 1];
266                                return s.slice((
267                                    offset as usize,
268                                    ((upper_offset + upper_len) - offset) as usize,
269                                ));
270                            }
271
272                            let indices = groups
273                                .iter()
274                                .map(|&[first, _len]| first)
275                                .collect_ca(PlSmallStr::EMPTY);
276                            // SAFETY: groups are always in bounds.
277                            let mut out = unsafe { s.take_unchecked(&indices) };
278                            // Sliced groups are always in order of discovery.
279                            out.set_sorted_flag(s.is_sorted_flag());
280                            out
281                        },
282                    }
283                })
284                .map(Column::from)
285                .collect()
286        })
287    }
288
289    pub fn keys(&self) -> Vec<Column> {
290        self.keys_sliced(None)
291    }
292
293    fn prepare_agg(&self) -> PolarsResult<(Vec<Column>, Vec<Column>)> {
294        let keys = self.keys();
295
296        let agg_col = match &self.selected_agg {
297            Some(selection) => self.df.select_to_vec(selection),
298            None => {
299                let by: Vec<_> = self.selected_keys.iter().map(|s| s.name()).collect();
300                let selection = self
301                    .df
302                    .columns()
303                    .iter()
304                    .map(|s| s.name())
305                    .filter(|a| !by.contains(a))
306                    .cloned()
307                    .collect::<Vec<_>>();
308
309                self.df.select_to_vec(selection.as_slice())
310            },
311        }?;
312
313        Ok((keys, agg_col))
314    }
315
316    /// Aggregate grouped series and compute the number of values per group.
317    ///
318    /// # Example
319    ///
320    /// ```rust
321    /// # use polars_core::prelude::*;
322    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
323    ///     df.group_by(["date"])?.select(["temp"]).count()
324    /// }
325    /// ```
326    /// Returns:
327    ///
328    /// ```text
329    /// +------------+------------+
330    /// | date       | temp_count |
331    /// | ---        | ---        |
332    /// | Date       | u32        |
333    /// +============+============+
334    /// | 2020-08-23 | 1          |
335    /// +------------+------------+
336    /// | 2020-08-22 | 2          |
337    /// +------------+------------+
338    /// | 2020-08-21 | 2          |
339    /// +------------+------------+
340    /// ```
341    pub fn count(&self) -> PolarsResult<DataFrame> {
342        let (mut cols, agg_cols) = self.prepare_agg()?;
343
344        for agg_col in agg_cols {
345            let new_name = fmt_group_by_column(
346                agg_col.name().as_str(),
347                GroupByMethod::Count {
348                    include_nulls: true,
349                },
350            );
351            let mut ca = self.groups.group_count();
352            ca.rename(new_name);
353            cols.push(ca.into_column());
354        }
355        DataFrame::new_infer_height(cols)
356    }
357
358    /// Get the group_by group indexes.
359    ///
360    /// # Example
361    ///
362    /// ```rust
363    /// # use polars_core::prelude::*;
364    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
365    ///     df.group_by(["date"])?.groups()
366    /// }
367    /// ```
368    /// Returns:
369    ///
370    /// ```text
371    /// +--------------+------------+
372    /// | date         | groups     |
373    /// | ---          | ---        |
374    /// | Date(days)   | list [u32] |
375    /// +==============+============+
376    /// | 2020-08-23   | "[3]"      |
377    /// +--------------+------------+
378    /// | 2020-08-22   | "[2, 4]"   |
379    /// +--------------+------------+
380    /// | 2020-08-21   | "[0, 1]"   |
381    /// +--------------+------------+
382    /// ```
383    pub fn groups(&self) -> PolarsResult<DataFrame> {
384        let mut cols = self.keys();
385        let mut column = self.groups.as_list_chunked();
386        let new_name = fmt_group_by_column("", GroupByMethod::Groups);
387        column.rename(new_name);
388        cols.push(column.into_column());
389        DataFrame::new_infer_height(cols)
390    }
391
392    fn prepare_apply(&self) -> PolarsResult<DataFrame> {
393        if let Some(agg) = &self.selected_agg {
394            if agg.is_empty() {
395                Ok(self.df.clone())
396            } else {
397                let mut new_cols = Vec::with_capacity(self.selected_keys.len() + agg.len());
398                new_cols.extend_from_slice(&self.selected_keys);
399                let cols = self.df.select_to_vec(agg.as_slice())?;
400                new_cols.extend(cols);
401                Ok(unsafe { DataFrame::new_unchecked(self.df.height(), new_cols) })
402            }
403        } else {
404            Ok(self.df.clone())
405        }
406    }
407
408    /// Apply a closure over the groups as a new [`DataFrame`].
409    pub fn apply<F>(&self, f: F) -> PolarsResult<DataFrame>
410    where
411        F: FnMut(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
412    {
413        self.apply_sliced(None, f, None)
414    }
415
416    pub fn apply_sliced<F>(
417        &self,
418        slice: Option<Slice>,
419        mut f: F,
420        schema: Option<&SchemaRef>,
421    ) -> PolarsResult<DataFrame>
422    where
423        F: FnMut(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
424    {
425        // Rolling and dynamic group-bys can have rows but no windows.
426        if self.df.height() == 0 || self.get_groups().is_empty() {
427            if let Some(schema) = schema {
428                return Ok(DataFrame::empty_with_arc_schema(schema.clone()));
429            }
430
431            polars_bail!(ComputeError: "cannot group_by + apply on empty 'DataFrame'");
432        }
433
434        let df = self.prepare_apply()?;
435        let max_height = slice.and_then(|s| s.end_position());
436        let mut height = 0;
437        let mut dfs = Vec::with_capacity(self.get_groups().len());
438        for g in self.get_groups().iter() {
439            // SAFETY: groups are in bounds.
440            let sub_df = unsafe { take_df(&df, g) };
441            let df = f(sub_df)?;
442            height += df.height();
443            dfs.push(df);
444
445            // Even if max_height is zero we need at least one df, so check
446            // after first push.
447            if max_height.is_some_and(|max_height| height >= max_height) {
448                break;
449            }
450        }
451
452        let mut df = accumulate_dataframes_vertical(dfs)?;
453        if let Some(slice) = slice {
454            df = df.slice(slice);
455        }
456        Ok(df)
457    }
458}
459
460unsafe fn take_df(df: &DataFrame, g: GroupsIndicator) -> DataFrame {
461    match g {
462        GroupsIndicator::Idx(idx) => df.take_slice_unchecked(idx.1),
463        GroupsIndicator::Slice([first, len]) => df.slice((first as usize, len as usize)),
464    }
465}
466
467#[derive(Copy, Clone, Debug)]
468pub enum GroupByMethod {
469    Min,
470    NanMin,
471    Max,
472    NanMax,
473    Median,
474    Mean,
475    First,
476    FirstNonNull,
477    Last,
478    LastNonNull,
479    Item { allow_empty: bool },
480    Sum { null_on_empty: bool },
481    Groups,
482    NUnique,
483    Quantile(f64, QuantileMethod),
484    Count { include_nulls: bool },
485    Implode { maintain_order: bool },
486    Std(u8),
487    Var(u8),
488    ArgMin,
489    ArgMax,
490}
491
492impl Display for GroupByMethod {
493    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
494        use GroupByMethod::*;
495        let s = match self {
496            Min => "min",
497            NanMin => "nan_min",
498            Max => "max",
499            NanMax => "nan_max",
500            Median => "median",
501            Mean => "mean",
502            First => "first",
503            FirstNonNull => "first_non_null",
504            Last => "last",
505            LastNonNull => "last_non_null",
506            Item { .. } => "item",
507            Sum { .. } => "sum",
508            Groups => "groups",
509            NUnique => "n_unique",
510            Quantile(_, _) => "quantile",
511            Count { .. } => "count",
512            Implode { .. } => "implode",
513            Std(_) => "std",
514            Var(_) => "var",
515            ArgMin => "arg_min",
516            ArgMax => "arg_max",
517        };
518        write!(f, "{s}")
519    }
520}
521
522// Formatting functions used in eager and lazy code for renaming grouped columns
523pub fn fmt_group_by_column(name: &str, method: GroupByMethod) -> PlSmallStr {
524    use GroupByMethod::*;
525    match method {
526        Min => format_pl_smallstr!("{name}_min"),
527        Max => format_pl_smallstr!("{name}_max"),
528        NanMin => format_pl_smallstr!("{name}_nan_min"),
529        NanMax => format_pl_smallstr!("{name}_nan_max"),
530        Median => format_pl_smallstr!("{name}_median"),
531        Mean => format_pl_smallstr!("{name}_mean"),
532        First => format_pl_smallstr!("{name}_first"),
533        FirstNonNull => format_pl_smallstr!("{name}_first_non_null"),
534        Last => format_pl_smallstr!("{name}_last"),
535        LastNonNull => format_pl_smallstr!("{name}_last_non_null"),
536        Item { .. } => format_pl_smallstr!("{name}_item"),
537        Sum { .. } => format_pl_smallstr!("{name}_sum"),
538        Groups => PlSmallStr::from_static("groups"),
539        NUnique => format_pl_smallstr!("{name}_n_unique"),
540        Count { .. } => format_pl_smallstr!("{name}_count"),
541        Implode { .. } => format_pl_smallstr!("{name}_agg_list"),
542        Quantile(quantile, _interpol) => format_pl_smallstr!("{name}_quantile_{quantile:.2}"),
543        Std(_) => format_pl_smallstr!("{name}_agg_std"),
544        Var(_) => format_pl_smallstr!("{name}_agg_var"),
545        ArgMin => format_pl_smallstr!("{name}_arg_min"),
546        ArgMax => format_pl_smallstr!("{name}_arg_max"),
547    }
548}