polars_core/frame/group_by/
mod.rs

1use std::fmt::{Debug, Display, Formatter};
2use std::hash::Hash;
3
4use num_traits::NumCast;
5use polars_compute::rolling::QuantileMethod;
6use polars_utils::format_pl_smallstr;
7use polars_utils::hashing::DirtyHash;
8use rayon::prelude::*;
9
10use self::hashing::*;
11use crate::POOL;
12use crate::prelude::*;
13use crate::utils::{_set_partition_size, accumulate_dataframes_vertical};
14
15pub mod aggregations;
16pub mod expr;
17pub(crate) mod hashing;
18mod into_groups;
19mod position;
20
21pub use into_groups::*;
22pub use position::*;
23
24use crate::chunked_array::ops::row_encode::{
25    encode_rows_unordered, encode_rows_vertical_par_unordered,
26};
27
28impl DataFrame {
29    pub fn group_by_with_series(
30        &self,
31        mut by: Vec<Column>,
32        multithreaded: bool,
33        sorted: bool,
34    ) -> PolarsResult<GroupBy<'_>> {
35        polars_ensure!(
36            !by.is_empty(),
37            ComputeError: "at least one key is required in a group_by operation"
38        );
39
40        // Ensure all 'by' columns have the same common_height
41        // The condition self.width > 0 ensures we can still call this on a
42        // dummy dataframe where we provide the keys
43        let common_height = if self.width() > 0 {
44            self.height()
45        } else {
46            by.iter().map(|s| s.len()).max().expect("at least 1 key")
47        };
48        for by_key in by.iter_mut() {
49            if by_key.len() != common_height {
50                polars_ensure!(
51                    by_key.len() == 1,
52                    ShapeMismatch: "series used as keys should have the same length as the DataFrame"
53                );
54                *by_key = by_key.new_from_index(0, common_height)
55            }
56        }
57
58        let groups = if by.len() == 1 {
59            let column = &by[0];
60            column
61                .as_materialized_series()
62                .group_tuples(multithreaded, sorted)
63        } else if by.iter().any(|s| s.dtype().is_object()) {
64            #[cfg(feature = "object")]
65            {
66                let mut df = DataFrame::new(by.clone()).unwrap();
67                let n = df.height();
68                let rows = df.to_av_rows();
69                let iter = (0..n).map(|i| rows.get(i));
70                Ok(group_by(iter, sorted))
71            }
72            #[cfg(not(feature = "object"))]
73            {
74                unreachable!()
75            }
76        } else {
77            // Skip null dtype.
78            let by = by
79                .iter()
80                .filter(|s| !s.dtype().is_null())
81                .cloned()
82                .collect::<Vec<_>>();
83            if by.is_empty() {
84                let groups = if self.is_empty() {
85                    vec![]
86                } else {
87                    vec![[0, self.height() as IdxSize]]
88                };
89                Ok(GroupsType::Slice {
90                    groups,
91                    rolling: false,
92                })
93            } else {
94                let rows = if multithreaded {
95                    encode_rows_vertical_par_unordered(&by)
96                } else {
97                    encode_rows_unordered(&by)
98                }?
99                .into_series();
100                rows.group_tuples(multithreaded, sorted)
101            }
102        };
103        Ok(GroupBy::new(self, by, groups?.into_sliceable(), None))
104    }
105
106    /// Group DataFrame using a Series column.
107    ///
108    /// # Example
109    ///
110    /// ```
111    /// use polars_core::prelude::*;
112    /// fn group_by_sum(df: &DataFrame) -> PolarsResult<DataFrame> {
113    ///     df.group_by(["column_name"])?
114    ///     .select(["agg_column_name"])
115    ///     .sum()
116    /// }
117    /// ```
118    pub fn group_by<I, S>(&self, by: I) -> PolarsResult<GroupBy<'_>>
119    where
120        I: IntoIterator<Item = S>,
121        S: Into<PlSmallStr>,
122    {
123        let selected_keys = self.select_columns(by)?;
124        self.group_by_with_series(selected_keys, true, false)
125    }
126
127    /// Group DataFrame using a Series column.
128    /// The groups are ordered by their smallest row index.
129    pub fn group_by_stable<I, S>(&self, by: I) -> PolarsResult<GroupBy<'_>>
130    where
131        I: IntoIterator<Item = S>,
132        S: Into<PlSmallStr>,
133    {
134        let selected_keys = self.select_columns(by)?;
135        self.group_by_with_series(selected_keys, true, true)
136    }
137}
138
139/// Returned by a group_by operation on a DataFrame. This struct supports
140/// several aggregations.
141///
142/// Until described otherwise, the examples in this struct are performed on the following DataFrame:
143///
144/// ```ignore
145/// use polars_core::prelude::*;
146///
147/// let dates = &[
148/// "2020-08-21",
149/// "2020-08-21",
150/// "2020-08-22",
151/// "2020-08-23",
152/// "2020-08-22",
153/// ];
154/// // date format
155/// let fmt = "%Y-%m-%d";
156/// // create date series
157/// let s0 = DateChunked::parse_from_str_slice("date", dates, fmt)
158///         .into_series();
159/// // create temperature series
160/// let s1 = Series::new("temp".into(), [20, 10, 7, 9, 1]);
161/// // create rain series
162/// let s2 = Series::new("rain".into(), [0.2, 0.1, 0.3, 0.1, 0.01]);
163/// // create a new DataFrame
164/// let df = DataFrame::new(vec![s0, s1, s2]).unwrap();
165/// println!("{:?}", df);
166/// ```
167///
168/// Outputs:
169///
170/// ```text
171/// +------------+------+------+
172/// | date       | temp | rain |
173/// | ---        | ---  | ---  |
174/// | Date       | i32  | f64  |
175/// +============+======+======+
176/// | 2020-08-21 | 20   | 0.2  |
177/// +------------+------+------+
178/// | 2020-08-21 | 10   | 0.1  |
179/// +------------+------+------+
180/// | 2020-08-22 | 7    | 0.3  |
181/// +------------+------+------+
182/// | 2020-08-23 | 9    | 0.1  |
183/// +------------+------+------+
184/// | 2020-08-22 | 1    | 0.01 |
185/// +------------+------+------+
186/// ```
187///
188#[derive(Debug, Clone)]
189pub struct GroupBy<'a> {
190    pub df: &'a DataFrame,
191    pub(crate) selected_keys: Vec<Column>,
192    // [first idx, [other idx]]
193    groups: GroupPositions,
194    // columns selected for aggregation
195    pub(crate) selected_agg: Option<Vec<PlSmallStr>>,
196}
197
198impl<'a> GroupBy<'a> {
199    pub fn new(
200        df: &'a DataFrame,
201        by: Vec<Column>,
202        groups: GroupPositions,
203        selected_agg: Option<Vec<PlSmallStr>>,
204    ) -> Self {
205        GroupBy {
206            df,
207            selected_keys: by,
208            groups,
209            selected_agg,
210        }
211    }
212
213    /// Select the column(s) that should be aggregated.
214    /// You can select a single column or a slice of columns.
215    ///
216    /// Note that making a selection with this method is not required. If you
217    /// skip it all columns (except for the keys) will be selected for aggregation.
218    #[must_use]
219    pub fn select<I: IntoIterator<Item = S>, S: Into<PlSmallStr>>(mut self, selection: I) -> Self {
220        self.selected_agg = Some(selection.into_iter().map(|s| s.into()).collect());
221        self
222    }
223
224    /// Get the internal representation of the GroupBy operation.
225    /// The Vec returned contains:
226    ///     (first_idx, [`Vec<indexes>`])
227    ///     Where second value in the tuple is a vector with all matching indexes.
228    pub fn get_groups(&self) -> &GroupPositions {
229        &self.groups
230    }
231
232    /// Get the internal representation of the GroupBy operation.
233    /// The Vec returned contains:
234    ///     (first_idx, [`Vec<indexes>`])
235    ///     Where second value in the tuple is a vector with all matching indexes.
236    ///
237    /// # Safety
238    /// Groups should always be in bounds of the `DataFrame` hold by this [`GroupBy`].
239    /// If you mutate it, you must hold that invariant.
240    pub unsafe fn get_groups_mut(&mut self) -> &mut GroupPositions {
241        &mut self.groups
242    }
243
244    pub fn take_groups(self) -> GroupPositions {
245        self.groups
246    }
247
248    pub fn take_groups_mut(&mut self) -> GroupPositions {
249        std::mem::take(&mut self.groups)
250    }
251
252    pub fn keys_sliced(&self, slice: Option<(i64, usize)>) -> Vec<Column> {
253        #[allow(unused_assignments)]
254        // needed to keep the lifetimes valid for this scope
255        let mut groups_owned = None;
256
257        let groups = if let Some((offset, len)) = slice {
258            groups_owned = Some(self.groups.slice(offset, len));
259            groups_owned.as_deref().unwrap()
260        } else {
261            &self.groups
262        };
263        POOL.install(|| {
264            self.selected_keys
265                .par_iter()
266                .map(Column::as_materialized_series)
267                .map(|s| {
268                    match groups {
269                        GroupsType::Idx(groups) => {
270                            // SAFETY: groups are always in bounds.
271                            let mut out = unsafe { s.take_slice_unchecked(groups.first()) };
272                            if groups.sorted {
273                                out.set_sorted_flag(s.is_sorted_flag());
274                            };
275                            out
276                        },
277                        GroupsType::Slice { groups, rolling } => {
278                            if *rolling && !groups.is_empty() {
279                                // Groups can be sliced.
280                                let offset = groups[0][0];
281                                let [upper_offset, upper_len] = groups[groups.len() - 1];
282                                return s.slice(
283                                    offset as i64,
284                                    ((upper_offset + upper_len) - offset) as usize,
285                                );
286                            }
287
288                            let indices = groups
289                                .iter()
290                                .map(|&[first, _len]| first)
291                                .collect_ca(PlSmallStr::EMPTY);
292                            // SAFETY: groups are always in bounds.
293                            let mut out = unsafe { s.take_unchecked(&indices) };
294                            // Sliced groups are always in order of discovery.
295                            out.set_sorted_flag(s.is_sorted_flag());
296                            out
297                        },
298                    }
299                })
300                .map(Column::from)
301                .collect()
302        })
303    }
304
305    pub fn keys(&self) -> Vec<Column> {
306        self.keys_sliced(None)
307    }
308
309    fn prepare_agg(&self) -> PolarsResult<(Vec<Column>, Vec<Column>)> {
310        let keys = self.keys();
311
312        let agg_col = match &self.selected_agg {
313            Some(selection) => self.df.select_columns_impl(selection.as_slice()),
314            None => {
315                let by: Vec<_> = self.selected_keys.iter().map(|s| s.name()).collect();
316                let selection = self
317                    .df
318                    .iter()
319                    .map(|s| s.name())
320                    .filter(|a| !by.contains(a))
321                    .cloned()
322                    .collect::<Vec<_>>();
323
324                self.df.select_columns_impl(selection.as_slice())
325            },
326        }?;
327
328        Ok((keys, agg_col))
329    }
330
331    /// Aggregate grouped series and compute the mean per group.
332    ///
333    /// # Example
334    ///
335    /// ```rust
336    /// # use polars_core::prelude::*;
337    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
338    ///     df.group_by(["date"])?.select(["temp", "rain"]).mean()
339    /// }
340    /// ```
341    /// Returns:
342    ///
343    /// ```text
344    /// +------------+-----------+-----------+
345    /// | date       | temp_mean | rain_mean |
346    /// | ---        | ---       | ---       |
347    /// | Date       | f64       | f64       |
348    /// +============+===========+===========+
349    /// | 2020-08-23 | 9         | 0.1       |
350    /// +------------+-----------+-----------+
351    /// | 2020-08-22 | 4         | 0.155     |
352    /// +------------+-----------+-----------+
353    /// | 2020-08-21 | 15        | 0.15      |
354    /// +------------+-----------+-----------+
355    /// ```
356    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
357    pub fn mean(&self) -> PolarsResult<DataFrame> {
358        let (mut cols, agg_cols) = self.prepare_agg()?;
359
360        for agg_col in agg_cols {
361            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Mean);
362            let mut agg = unsafe { agg_col.agg_mean(&self.groups) };
363            agg.rename(new_name);
364            cols.push(agg);
365        }
366        DataFrame::new(cols)
367    }
368
369    /// Aggregate grouped series and compute the sum per group.
370    ///
371    /// # Example
372    ///
373    /// ```rust
374    /// # use polars_core::prelude::*;
375    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
376    ///     df.group_by(["date"])?.select(["temp"]).sum()
377    /// }
378    /// ```
379    /// Returns:
380    ///
381    /// ```text
382    /// +------------+----------+
383    /// | date       | temp_sum |
384    /// | ---        | ---      |
385    /// | Date       | i32      |
386    /// +============+==========+
387    /// | 2020-08-23 | 9        |
388    /// +------------+----------+
389    /// | 2020-08-22 | 8        |
390    /// +------------+----------+
391    /// | 2020-08-21 | 30       |
392    /// +------------+----------+
393    /// ```
394    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
395    pub fn sum(&self) -> PolarsResult<DataFrame> {
396        let (mut cols, agg_cols) = self.prepare_agg()?;
397
398        for agg_col in agg_cols {
399            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Sum);
400            let mut agg = unsafe { agg_col.agg_sum(&self.groups) };
401            agg.rename(new_name);
402            cols.push(agg);
403        }
404        DataFrame::new(cols)
405    }
406
407    /// Aggregate grouped series and compute the minimal value per group.
408    ///
409    /// # Example
410    ///
411    /// ```rust
412    /// # use polars_core::prelude::*;
413    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
414    ///     df.group_by(["date"])?.select(["temp"]).min()
415    /// }
416    /// ```
417    /// Returns:
418    ///
419    /// ```text
420    /// +------------+----------+
421    /// | date       | temp_min |
422    /// | ---        | ---      |
423    /// | Date       | i32      |
424    /// +============+==========+
425    /// | 2020-08-23 | 9        |
426    /// +------------+----------+
427    /// | 2020-08-22 | 1        |
428    /// +------------+----------+
429    /// | 2020-08-21 | 10       |
430    /// +------------+----------+
431    /// ```
432    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
433    pub fn min(&self) -> PolarsResult<DataFrame> {
434        let (mut cols, agg_cols) = self.prepare_agg()?;
435        for agg_col in agg_cols {
436            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Min);
437            let mut agg = unsafe { agg_col.agg_min(&self.groups) };
438            agg.rename(new_name);
439            cols.push(agg);
440        }
441        DataFrame::new(cols)
442    }
443
444    /// Aggregate grouped series and compute the maximum value per group.
445    ///
446    /// # Example
447    ///
448    /// ```rust
449    /// # use polars_core::prelude::*;
450    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
451    ///     df.group_by(["date"])?.select(["temp"]).max()
452    /// }
453    /// ```
454    /// Returns:
455    ///
456    /// ```text
457    /// +------------+----------+
458    /// | date       | temp_max |
459    /// | ---        | ---      |
460    /// | Date       | i32      |
461    /// +============+==========+
462    /// | 2020-08-23 | 9        |
463    /// +------------+----------+
464    /// | 2020-08-22 | 7        |
465    /// +------------+----------+
466    /// | 2020-08-21 | 20       |
467    /// +------------+----------+
468    /// ```
469    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
470    pub fn max(&self) -> PolarsResult<DataFrame> {
471        let (mut cols, agg_cols) = self.prepare_agg()?;
472        for agg_col in agg_cols {
473            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Max);
474            let mut agg = unsafe { agg_col.agg_max(&self.groups) };
475            agg.rename(new_name);
476            cols.push(agg);
477        }
478        DataFrame::new(cols)
479    }
480
481    /// Aggregate grouped `Series` and find the first value per group.
482    ///
483    /// # Example
484    ///
485    /// ```rust
486    /// # use polars_core::prelude::*;
487    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
488    ///     df.group_by(["date"])?.select(["temp"]).first()
489    /// }
490    /// ```
491    /// Returns:
492    ///
493    /// ```text
494    /// +------------+------------+
495    /// | date       | temp_first |
496    /// | ---        | ---        |
497    /// | Date       | i32        |
498    /// +============+============+
499    /// | 2020-08-23 | 9          |
500    /// +------------+------------+
501    /// | 2020-08-22 | 7          |
502    /// +------------+------------+
503    /// | 2020-08-21 | 20         |
504    /// +------------+------------+
505    /// ```
506    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
507    pub fn first(&self) -> PolarsResult<DataFrame> {
508        let (mut cols, agg_cols) = self.prepare_agg()?;
509        for agg_col in agg_cols {
510            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::First);
511            let mut agg = unsafe { agg_col.agg_first(&self.groups) };
512            agg.rename(new_name);
513            cols.push(agg);
514        }
515        DataFrame::new(cols)
516    }
517
518    /// Aggregate grouped `Series` and return the last value per group.
519    ///
520    /// # Example
521    ///
522    /// ```rust
523    /// # use polars_core::prelude::*;
524    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
525    ///     df.group_by(["date"])?.select(["temp"]).last()
526    /// }
527    /// ```
528    /// Returns:
529    ///
530    /// ```text
531    /// +------------+------------+
532    /// | date       | temp_last |
533    /// | ---        | ---        |
534    /// | Date       | i32        |
535    /// +============+============+
536    /// | 2020-08-23 | 9          |
537    /// +------------+------------+
538    /// | 2020-08-22 | 1          |
539    /// +------------+------------+
540    /// | 2020-08-21 | 10         |
541    /// +------------+------------+
542    /// ```
543    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
544    pub fn last(&self) -> PolarsResult<DataFrame> {
545        let (mut cols, agg_cols) = self.prepare_agg()?;
546        for agg_col in agg_cols {
547            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Last);
548            let mut agg = unsafe { agg_col.agg_last(&self.groups) };
549            agg.rename(new_name);
550            cols.push(agg);
551        }
552        DataFrame::new(cols)
553    }
554
555    /// Aggregate grouped `Series` by counting the number of unique values.
556    ///
557    /// # Example
558    ///
559    /// ```rust
560    /// # use polars_core::prelude::*;
561    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
562    ///     df.group_by(["date"])?.select(["temp"]).n_unique()
563    /// }
564    /// ```
565    /// Returns:
566    ///
567    /// ```text
568    /// +------------+---------------+
569    /// | date       | temp_n_unique |
570    /// | ---        | ---           |
571    /// | Date       | u32           |
572    /// +============+===============+
573    /// | 2020-08-23 | 1             |
574    /// +------------+---------------+
575    /// | 2020-08-22 | 2             |
576    /// +------------+---------------+
577    /// | 2020-08-21 | 2             |
578    /// +------------+---------------+
579    /// ```
580    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
581    pub fn n_unique(&self) -> PolarsResult<DataFrame> {
582        let (mut cols, agg_cols) = self.prepare_agg()?;
583        for agg_col in agg_cols {
584            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::NUnique);
585            let mut agg = unsafe { agg_col.agg_n_unique(&self.groups) };
586            agg.rename(new_name);
587            cols.push(agg);
588        }
589        DataFrame::new(cols)
590    }
591
592    /// Aggregate grouped [`Series`] and determine the quantile per group.
593    ///
594    /// # Example
595    ///
596    /// ```rust
597    /// # use polars_core::prelude::*;
598    ///
599    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
600    ///     df.group_by(["date"])?.select(["temp"]).quantile(0.2, QuantileMethod::default())
601    /// }
602    /// ```
603    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
604    pub fn quantile(&self, quantile: f64, method: QuantileMethod) -> PolarsResult<DataFrame> {
605        polars_ensure!(
606            (0.0..=1.0).contains(&quantile),
607            ComputeError: "`quantile` should be within 0.0 and 1.0"
608        );
609        let (mut cols, agg_cols) = self.prepare_agg()?;
610        for agg_col in agg_cols {
611            let new_name = fmt_group_by_column(
612                agg_col.name().as_str(),
613                GroupByMethod::Quantile(quantile, method),
614            );
615            let mut agg = unsafe { agg_col.agg_quantile(&self.groups, quantile, method) };
616            agg.rename(new_name);
617            cols.push(agg);
618        }
619        DataFrame::new(cols)
620    }
621
622    /// Aggregate grouped [`Series`] and determine the median per group.
623    ///
624    /// # Example
625    ///
626    /// ```rust
627    /// # use polars_core::prelude::*;
628    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
629    ///     df.group_by(["date"])?.select(["temp"]).median()
630    /// }
631    /// ```
632    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
633    pub fn median(&self) -> PolarsResult<DataFrame> {
634        let (mut cols, agg_cols) = self.prepare_agg()?;
635        for agg_col in agg_cols {
636            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Median);
637            let mut agg = unsafe { agg_col.agg_median(&self.groups) };
638            agg.rename(new_name);
639            cols.push(agg);
640        }
641        DataFrame::new(cols)
642    }
643
644    /// Aggregate grouped [`Series`] and determine the variance per group.
645    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
646    pub fn var(&self, ddof: u8) -> PolarsResult<DataFrame> {
647        let (mut cols, agg_cols) = self.prepare_agg()?;
648        for agg_col in agg_cols {
649            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Var(ddof));
650            let mut agg = unsafe { agg_col.agg_var(&self.groups, ddof) };
651            agg.rename(new_name);
652            cols.push(agg);
653        }
654        DataFrame::new(cols)
655    }
656
657    /// Aggregate grouped [`Series`] and determine the standard deviation per group.
658    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
659    pub fn std(&self, ddof: u8) -> PolarsResult<DataFrame> {
660        let (mut cols, agg_cols) = self.prepare_agg()?;
661        for agg_col in agg_cols {
662            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Std(ddof));
663            let mut agg = unsafe { agg_col.agg_std(&self.groups, ddof) };
664            agg.rename(new_name);
665            cols.push(agg);
666        }
667        DataFrame::new(cols)
668    }
669
670    /// Aggregate grouped series and compute the number of values per group.
671    ///
672    /// # Example
673    ///
674    /// ```rust
675    /// # use polars_core::prelude::*;
676    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
677    ///     df.group_by(["date"])?.select(["temp"]).count()
678    /// }
679    /// ```
680    /// Returns:
681    ///
682    /// ```text
683    /// +------------+------------+
684    /// | date       | temp_count |
685    /// | ---        | ---        |
686    /// | Date       | u32        |
687    /// +============+============+
688    /// | 2020-08-23 | 1          |
689    /// +------------+------------+
690    /// | 2020-08-22 | 2          |
691    /// +------------+------------+
692    /// | 2020-08-21 | 2          |
693    /// +------------+------------+
694    /// ```
695    pub fn count(&self) -> PolarsResult<DataFrame> {
696        let (mut cols, agg_cols) = self.prepare_agg()?;
697
698        for agg_col in agg_cols {
699            let new_name = fmt_group_by_column(
700                agg_col.name().as_str(),
701                GroupByMethod::Count {
702                    include_nulls: true,
703                },
704            );
705            let mut ca = self.groups.group_count();
706            ca.rename(new_name);
707            cols.push(ca.into_column());
708        }
709        DataFrame::new(cols)
710    }
711
712    /// Get the group_by group indexes.
713    ///
714    /// # Example
715    ///
716    /// ```rust
717    /// # use polars_core::prelude::*;
718    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
719    ///     df.group_by(["date"])?.groups()
720    /// }
721    /// ```
722    /// Returns:
723    ///
724    /// ```text
725    /// +--------------+------------+
726    /// | date         | groups     |
727    /// | ---          | ---        |
728    /// | Date(days)   | list [u32] |
729    /// +==============+============+
730    /// | 2020-08-23   | "[3]"      |
731    /// +--------------+------------+
732    /// | 2020-08-22   | "[2, 4]"   |
733    /// +--------------+------------+
734    /// | 2020-08-21   | "[0, 1]"   |
735    /// +--------------+------------+
736    /// ```
737    pub fn groups(&self) -> PolarsResult<DataFrame> {
738        let mut cols = self.keys();
739        let mut column = self.groups.as_list_chunked();
740        let new_name = fmt_group_by_column("", GroupByMethod::Groups);
741        column.rename(new_name);
742        cols.push(column.into_column());
743        DataFrame::new(cols)
744    }
745
746    /// Aggregate the groups of the group_by operation into lists.
747    ///
748    /// # Example
749    ///
750    /// ```rust
751    /// # use polars_core::prelude::*;
752    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
753    ///     // GroupBy and aggregate to Lists
754    ///     df.group_by(["date"])?.select(["temp"]).agg_list()
755    /// }
756    /// ```
757    /// Returns:
758    ///
759    /// ```text
760    /// +------------+------------------------+
761    /// | date       | temp_agg_list          |
762    /// | ---        | ---                    |
763    /// | Date       | list [i32]             |
764    /// +============+========================+
765    /// | 2020-08-23 | "[Some(9)]"            |
766    /// +------------+------------------------+
767    /// | 2020-08-22 | "[Some(7), Some(1)]"   |
768    /// +------------+------------------------+
769    /// | 2020-08-21 | "[Some(20), Some(10)]" |
770    /// +------------+------------------------+
771    /// ```
772    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
773    pub fn agg_list(&self) -> PolarsResult<DataFrame> {
774        let (mut cols, agg_cols) = self.prepare_agg()?;
775        for agg_col in agg_cols {
776            let new_name = fmt_group_by_column(agg_col.name().as_str(), GroupByMethod::Implode);
777            let mut agg = unsafe { agg_col.agg_list(&self.groups) };
778            agg.rename(new_name);
779            cols.push(agg);
780        }
781        DataFrame::new(cols)
782    }
783
784    fn prepare_apply(&self) -> PolarsResult<DataFrame> {
785        polars_ensure!(self.df.height() > 0, ComputeError: "cannot group_by + apply on empty 'DataFrame'");
786        if let Some(agg) = &self.selected_agg {
787            if agg.is_empty() {
788                Ok(self.df.clone())
789            } else {
790                let mut new_cols = Vec::with_capacity(self.selected_keys.len() + agg.len());
791                new_cols.extend_from_slice(&self.selected_keys);
792                let cols = self.df.select_columns_impl(agg.as_slice())?;
793                new_cols.extend(cols);
794                Ok(unsafe { DataFrame::new_no_checks(self.df.height(), new_cols) })
795            }
796        } else {
797            Ok(self.df.clone())
798        }
799    }
800
801    /// Apply a closure over the groups as a new [`DataFrame`] in parallel.
802    #[deprecated(since = "0.24.1", note = "use polars.lazy aggregations")]
803    pub fn par_apply<F>(&self, f: F) -> PolarsResult<DataFrame>
804    where
805        F: Fn(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
806    {
807        let df = self.prepare_apply()?;
808        let dfs = self
809            .get_groups()
810            .par_iter()
811            .map(|g| {
812                // SAFETY:
813                // groups are in bounds
814                let sub_df = unsafe { take_df(&df, g) };
815                f(sub_df)
816            })
817            .collect::<PolarsResult<Vec<_>>>()?;
818
819        let mut df = accumulate_dataframes_vertical(dfs)?;
820        df.as_single_chunk_par();
821        Ok(df)
822    }
823
824    /// Apply a closure over the groups as a new [`DataFrame`].
825    pub fn apply<F>(&self, mut f: F) -> PolarsResult<DataFrame>
826    where
827        F: FnMut(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
828    {
829        let df = self.prepare_apply()?;
830        let dfs = self
831            .get_groups()
832            .iter()
833            .map(|g| {
834                // SAFETY:
835                // groups are in bounds
836                let sub_df = unsafe { take_df(&df, g) };
837                f(sub_df)
838            })
839            .collect::<PolarsResult<Vec<_>>>()?;
840
841        let mut df = accumulate_dataframes_vertical(dfs)?;
842        df.as_single_chunk_par();
843        Ok(df)
844    }
845
846    pub fn sliced(mut self, slice: Option<(i64, usize)>) -> Self {
847        match slice {
848            None => self,
849            Some((offset, length)) => {
850                self.groups = self.groups.slice(offset, length);
851                self.selected_keys = self.keys_sliced(slice);
852                self
853            },
854        }
855    }
856}
857
858unsafe fn take_df(df: &DataFrame, g: GroupsIndicator) -> DataFrame {
859    match g {
860        GroupsIndicator::Idx(idx) => df.take_slice_unchecked(idx.1),
861        GroupsIndicator::Slice([first, len]) => df.slice(first as i64, len as usize),
862    }
863}
864
865#[derive(Copy, Clone, Debug)]
866pub enum GroupByMethod {
867    Min,
868    NanMin,
869    Max,
870    NanMax,
871    Median,
872    Mean,
873    First,
874    Last,
875    Sum,
876    Groups,
877    NUnique,
878    Quantile(f64, QuantileMethod),
879    Count { include_nulls: bool },
880    Implode,
881    Std(u8),
882    Var(u8),
883}
884
885impl Display for GroupByMethod {
886    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
887        use GroupByMethod::*;
888        let s = match self {
889            Min => "min",
890            NanMin => "nan_min",
891            Max => "max",
892            NanMax => "nan_max",
893            Median => "median",
894            Mean => "mean",
895            First => "first",
896            Last => "last",
897            Sum => "sum",
898            Groups => "groups",
899            NUnique => "n_unique",
900            Quantile(_, _) => "quantile",
901            Count { .. } => "count",
902            Implode => "list",
903            Std(_) => "std",
904            Var(_) => "var",
905        };
906        write!(f, "{s}")
907    }
908}
909
910// Formatting functions used in eager and lazy code for renaming grouped columns
911pub fn fmt_group_by_column(name: &str, method: GroupByMethod) -> PlSmallStr {
912    use GroupByMethod::*;
913    match method {
914        Min => format_pl_smallstr!("{name}_min"),
915        Max => format_pl_smallstr!("{name}_max"),
916        NanMin => format_pl_smallstr!("{name}_nan_min"),
917        NanMax => format_pl_smallstr!("{name}_nan_max"),
918        Median => format_pl_smallstr!("{name}_median"),
919        Mean => format_pl_smallstr!("{name}_mean"),
920        First => format_pl_smallstr!("{name}_first"),
921        Last => format_pl_smallstr!("{name}_last"),
922        Sum => format_pl_smallstr!("{name}_sum"),
923        Groups => PlSmallStr::from_static("groups"),
924        NUnique => format_pl_smallstr!("{name}_n_unique"),
925        Count { .. } => format_pl_smallstr!("{name}_count"),
926        Implode => format_pl_smallstr!("{name}_agg_list"),
927        Quantile(quantile, _interpol) => format_pl_smallstr!("{name}_quantile_{quantile:.2}"),
928        Std(_) => format_pl_smallstr!("{name}_agg_std"),
929        Var(_) => format_pl_smallstr!("{name}_agg_var"),
930    }
931}
932
933#[cfg(test)]
934mod test {
935    use num_traits::FloatConst;
936
937    use crate::prelude::*;
938
939    #[test]
940    #[cfg(feature = "dtype-date")]
941    #[cfg_attr(miri, ignore)]
942    fn test_group_by() -> PolarsResult<()> {
943        let s0 = Column::new(
944            PlSmallStr::from_static("date"),
945            &[
946                "2020-08-21",
947                "2020-08-21",
948                "2020-08-22",
949                "2020-08-23",
950                "2020-08-22",
951            ],
952        );
953        let s1 = Column::new(PlSmallStr::from_static("temp"), [20, 10, 7, 9, 1]);
954        let s2 = Column::new(PlSmallStr::from_static("rain"), [0.2, 0.1, 0.3, 0.1, 0.01]);
955        let df = DataFrame::new(vec![s0, s1, s2]).unwrap();
956
957        let out = df.group_by_stable(["date"])?.select(["temp"]).count()?;
958        assert_eq!(
959            out.column("temp_count")?,
960            &Column::new(PlSmallStr::from_static("temp_count"), [2 as IdxSize, 2, 1])
961        );
962
963        // Use of deprecated mean() for testing purposes
964        #[allow(deprecated)]
965        // Select multiple
966        let out = df
967            .group_by_stable(["date"])?
968            .select(["temp", "rain"])
969            .mean()?;
970        assert_eq!(
971            out.column("temp_mean")?,
972            &Column::new(PlSmallStr::from_static("temp_mean"), [15.0f64, 4.0, 9.0])
973        );
974
975        // Use of deprecated `mean()` for testing purposes
976        #[allow(deprecated)]
977        // Group by multiple
978        let out = df
979            .group_by_stable(["date", "temp"])?
980            .select(["rain"])
981            .mean()?;
982        assert!(out.column("rain_mean").is_ok());
983
984        // Use of deprecated `sum()` for testing purposes
985        #[allow(deprecated)]
986        let out = df.group_by_stable(["date"])?.select(["temp"]).sum()?;
987        assert_eq!(
988            out.column("temp_sum")?,
989            &Column::new(PlSmallStr::from_static("temp_sum"), [30, 8, 9])
990        );
991
992        // Use of deprecated `n_unique()` for testing purposes
993        #[allow(deprecated)]
994        // implicit select all and only aggregate on methods that support that aggregation
995        let gb = df.group_by(["date"]).unwrap().n_unique().unwrap();
996        // check the group by column is filtered out.
997        assert_eq!(gb.width(), 3);
998        Ok(())
999    }
1000
1001    #[test]
1002    #[cfg_attr(miri, ignore)]
1003    fn test_static_group_by_by_12_columns() {
1004        // Build GroupBy DataFrame.
1005        let s0 = Column::new("G1".into(), ["A", "A", "B", "B", "C"].as_ref());
1006        let s1 = Column::new("N".into(), [1, 2, 2, 4, 2].as_ref());
1007        let s2 = Column::new("G2".into(), ["k", "l", "m", "m", "l"].as_ref());
1008        let s3 = Column::new("G3".into(), ["a", "b", "c", "c", "d"].as_ref());
1009        let s4 = Column::new("G4".into(), ["1", "2", "3", "3", "4"].as_ref());
1010        let s5 = Column::new("G5".into(), ["X", "Y", "Z", "Z", "W"].as_ref());
1011        let s6 = Column::new("G6".into(), [false, true, true, true, false].as_ref());
1012        let s7 = Column::new("G7".into(), ["r", "x", "q", "q", "o"].as_ref());
1013        let s8 = Column::new("G8".into(), ["R", "X", "Q", "Q", "O"].as_ref());
1014        let s9 = Column::new("G9".into(), [1, 2, 3, 3, 4].as_ref());
1015        let s10 = Column::new("G10".into(), [".", "!", "?", "?", "/"].as_ref());
1016        let s11 = Column::new("G11".into(), ["(", ")", "@", "@", "$"].as_ref());
1017        let s12 = Column::new("G12".into(), ["-", "_", ";", ";", ","].as_ref());
1018
1019        let df =
1020            DataFrame::new(vec![s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12]).unwrap();
1021
1022        // Use of deprecated `sum()` for testing purposes
1023        #[allow(deprecated)]
1024        let adf = df
1025            .group_by([
1026                "G1", "G2", "G3", "G4", "G5", "G6", "G7", "G8", "G9", "G10", "G11", "G12",
1027            ])
1028            .unwrap()
1029            .select(["N"])
1030            .sum()
1031            .unwrap();
1032
1033        assert_eq!(
1034            Vec::from(&adf.column("N_sum").unwrap().i32().unwrap().sort(false)),
1035            &[Some(1), Some(2), Some(2), Some(6)]
1036        );
1037    }
1038
1039    #[test]
1040    #[cfg_attr(miri, ignore)]
1041    fn test_dynamic_group_by_by_13_columns() {
1042        // The content for every group_by series.
1043        let series_content = ["A", "A", "B", "B", "C"];
1044
1045        // The name of every group_by series.
1046        let series_names = [
1047            "G1", "G2", "G3", "G4", "G5", "G6", "G7", "G8", "G9", "G10", "G11", "G12", "G13",
1048        ];
1049
1050        // Vector to contain every series.
1051        let mut columns = Vec::with_capacity(14);
1052
1053        // Create a series for every group name.
1054        for series_name in series_names {
1055            let group_columns = Column::new(series_name.into(), series_content.as_ref());
1056            columns.push(group_columns);
1057        }
1058
1059        // Create a series for the aggregation column.
1060        let agg_series = Column::new("N".into(), [1, 2, 3, 3, 4].as_ref());
1061        columns.push(agg_series);
1062
1063        // Create the dataframe with the computed series.
1064        let df = DataFrame::new(columns).unwrap();
1065
1066        // Use of deprecated `sum()` for testing purposes
1067        #[allow(deprecated)]
1068        // Compute the aggregated DataFrame by the 13 columns defined in `series_names`.
1069        let adf = df
1070            .group_by(series_names)
1071            .unwrap()
1072            .select(["N"])
1073            .sum()
1074            .unwrap();
1075
1076        // Check that the results of the group-by are correct. The content of every column
1077        // is equal, then, the grouped columns shall be equal and in the same order.
1078        for series_name in &series_names {
1079            assert_eq!(
1080                Vec::from(&adf.column(series_name).unwrap().str().unwrap().sort(false)),
1081                &[Some("A"), Some("B"), Some("C")]
1082            );
1083        }
1084
1085        // Check the aggregated column is the expected one.
1086        assert_eq!(
1087            Vec::from(&adf.column("N_sum").unwrap().i32().unwrap().sort(false)),
1088            &[Some(3), Some(4), Some(6)]
1089        );
1090    }
1091
1092    #[test]
1093    #[cfg_attr(miri, ignore)]
1094    fn test_group_by_floats() {
1095        let df = df! {"flt" => [1., 1., 2., 2., 3.],
1096                    "val" => [1, 1, 1, 1, 1]
1097        }
1098        .unwrap();
1099        // Use of deprecated `sum()` for testing purposes
1100        #[allow(deprecated)]
1101        let res = df.group_by(["flt"]).unwrap().sum().unwrap();
1102        let res = res.sort(["flt"], SortMultipleOptions::default()).unwrap();
1103        assert_eq!(
1104            Vec::from(res.column("val_sum").unwrap().i32().unwrap()),
1105            &[Some(2), Some(2), Some(1)]
1106        );
1107    }
1108
1109    #[test]
1110    #[cfg_attr(miri, ignore)]
1111    #[cfg(feature = "dtype-categorical")]
1112    fn test_group_by_categorical() {
1113        let mut df = df! {"foo" => ["a", "a", "b", "b", "c"],
1114                    "ham" => ["a", "a", "b", "b", "c"],
1115                    "bar" => [1, 1, 1, 1, 1]
1116        }
1117        .unwrap();
1118
1119        df.apply("foo", |s| {
1120            s.cast(&DataType::from_categories(Categories::global()))
1121                .unwrap()
1122        })
1123        .unwrap();
1124
1125        // Use of deprecated `sum()` for testing purposes
1126        #[allow(deprecated)]
1127        // check multiple keys and categorical
1128        let res = df
1129            .group_by_stable(["foo", "ham"])
1130            .unwrap()
1131            .select(["bar"])
1132            .sum()
1133            .unwrap();
1134
1135        assert_eq!(
1136            Vec::from(
1137                res.column("bar_sum")
1138                    .unwrap()
1139                    .as_materialized_series()
1140                    .i32()
1141                    .unwrap()
1142            ),
1143            &[Some(2), Some(2), Some(1)]
1144        );
1145    }
1146
1147    #[test]
1148    #[cfg_attr(miri, ignore)]
1149    fn test_group_by_null_handling() -> PolarsResult<()> {
1150        let df = df!(
1151            "a" => ["a", "a", "a", "b", "b"],
1152            "b" => [Some(1), Some(2), None, None, Some(1)]
1153        )?;
1154        // Use of deprecated `mean()` for testing purposes
1155        #[allow(deprecated)]
1156        let out = df.group_by_stable(["a"])?.mean()?;
1157
1158        assert_eq!(
1159            Vec::from(out.column("b_mean")?.as_materialized_series().f64()?),
1160            &[Some(1.5), Some(1.0)]
1161        );
1162        Ok(())
1163    }
1164
1165    #[test]
1166    #[cfg_attr(miri, ignore)]
1167    fn test_group_by_var() -> PolarsResult<()> {
1168        // check variance and proper coercion to f64
1169        let df = df![
1170            "g" => ["foo", "foo", "bar"],
1171            "flt" => [1.0, 2.0, 3.0],
1172            "int" => [1, 2, 3]
1173        ]?;
1174
1175        // Use of deprecated `sum()` for testing purposes
1176        #[allow(deprecated)]
1177        let out = df.group_by_stable(["g"])?.select(["int"]).var(1)?;
1178
1179        assert_eq!(out.column("int_agg_var")?.f64()?.get(0), Some(0.5));
1180        // Use of deprecated `std()` for testing purposes
1181        #[allow(deprecated)]
1182        let out = df.group_by_stable(["g"])?.select(["int"]).std(1)?;
1183        let val = out.column("int_agg_std")?.f64()?.get(0).unwrap();
1184        let expected = f64::FRAC_1_SQRT_2();
1185        assert!((val - expected).abs() < 0.000001);
1186        Ok(())
1187    }
1188
1189    #[test]
1190    #[cfg_attr(miri, ignore)]
1191    #[cfg(feature = "dtype-categorical")]
1192    fn test_group_by_null_group() -> PolarsResult<()> {
1193        // check if null is own group
1194        let mut df = df![
1195            "g" => [Some("foo"), Some("foo"), Some("bar"), None, None],
1196            "flt" => [1.0, 2.0, 3.0, 1.0, 1.0],
1197            "int" => [1, 2, 3, 1, 1]
1198        ]?;
1199
1200        df.try_apply("g", |s| {
1201            s.cast(&DataType::from_categories(Categories::global()))
1202        })?;
1203
1204        // Use of deprecated `sum()` for testing purposes
1205        #[allow(deprecated)]
1206        let _ = df.group_by(["g"])?.sum()?;
1207        Ok(())
1208    }
1209}