Skip to main content

polars_core/frame/
mod.rs

1#![allow(unsafe_op_in_unsafe_fn)]
2//! DataFrame module.
3use std::borrow::Cow;
4
5use either::Either;
6use polars_arrow::datatypes::ArrowSchemaRef;
7use polars_row::ArrayRef;
8use polars_utils::UnitVec;
9use polars_utils::itertools::Itertools;
10use rayon::prelude::*;
11
12use crate::chunked_array::flags::StatisticsFlags;
13#[cfg(feature = "algorithm_group_by")]
14use crate::chunked_array::ops::unique::is_unique_helper;
15use crate::prelude::gather::check_bounds_ca;
16use crate::prelude::*;
17#[cfg(feature = "row_hash")]
18use crate::utils::split_df;
19use crate::utils::{Container, NoNull, try_get_supertype};
20use crate::{HEAD_DEFAULT_LENGTH, TAIL_DEFAULT_LENGTH};
21
22#[cfg(feature = "dataframe_arithmetic")]
23mod arithmetic;
24pub mod builder;
25mod chunks;
26pub use chunks::chunk_df_for_writing;
27pub mod column;
28mod dataframe;
29mod filter;
30mod projection;
31pub use dataframe::DataFrame;
32use filter::filter_zero_width;
33use projection::{AmortizedColumnSelector, LINEAR_SEARCH_LIMIT};
34
35pub mod explode;
36mod from;
37#[cfg(feature = "algorithm_group_by")]
38pub mod group_by;
39pub(crate) mod horizontal;
40#[cfg(any(feature = "rows", feature = "object"))]
41pub mod row;
42mod top_k;
43mod upstream_traits;
44pub(crate) mod validation;
45
46use polars_arrow::record_batch::{RecordBatch, RecordBatchT};
47use polars_utils::pl_str::PlSmallStr;
48use polars_utils::slice_enum::Slice;
49#[cfg(feature = "serde")]
50use serde::{Deserialize, Serialize};
51use strum_macros::IntoStaticStr;
52
53#[cfg(feature = "row_hash")]
54use crate::hashing::_df_rows_to_hashes_threaded_vertical;
55use crate::prelude::sort::arg_sort;
56use crate::runtime::RAYON;
57use crate::series::IsSorted;
58use crate::series::arrow_export::{arrow_export_cut, contains_arrow_map, map_spans_fit};
59
60#[derive(Copy, Clone, Debug, PartialEq, Eq, Default, Hash, IntoStaticStr)]
61#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
62#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
63#[strum(serialize_all = "snake_case")]
64pub enum UniqueKeepStrategy {
65    /// Keep the first unique row.
66    First,
67    /// Keep the last unique row.
68    Last,
69    /// Keep None of the unique rows.
70    None,
71    /// Keep any of the unique rows
72    /// This allows more optimizations
73    #[default]
74    Any,
75}
76
77#[derive(Copy, Clone, Debug, PartialEq, Eq, Default, Hash, IntoStaticStr)]
78#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
79#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
80#[strum(serialize_all = "snake_case")]
81/// Naming strategy for the results of a pivot.
82pub enum PivotColumnNaming {
83    /// Always combine the values and on-column names.
84    Combine,
85    /// Prefix the values column name only if there is more than one values
86    /// column.
87    #[default]
88    Auto,
89}
90
91/// The chunk lengths of a column that has chunks of its own: a series, or the codes of a
92/// dictionary column (which materializes with the same chunks). Scalar columns have none.
93fn column_chunk_lengths(c: &Column) -> Option<Box<dyn Iterator<Item = usize> + '_>> {
94    match c {
95        Column::Series(_) => c.as_series().map(|s| Box::new(s.chunk_lengths()) as _),
96        Column::Dictionary(d) => Some(Box::new(d.codes().chunk_lengths())),
97        Column::Scalar(_) => None,
98    }
99}
100
101impl DataFrame {
102    /// The chunk lengths the frame is laid out along: those of the first series column, as the
103    /// chunks of series columns are selected by index; only without one, those of the first
104    /// dictionary column's codes (which are sliced by offset).
105    fn chunk_lengths_of_leading_column(&self) -> Option<Box<dyn Iterator<Item = usize> + '_>> {
106        self.columns()
107            .iter()
108            .find(|c| matches!(c, Column::Series(_)))
109            .and_then(column_chunk_lengths)
110            .or_else(|| self.columns().iter().find_map(column_chunk_lengths))
111    }
112
113    pub fn materialized_column_iter(&self) -> impl ExactSizeIterator<Item = &Series> {
114        self.columns().iter().map(Column::as_materialized_series)
115    }
116
117    /// Returns an estimation of the total (heap) allocated size of the `DataFrame` in bytes.
118    ///
119    /// # Implementation
120    /// This estimation is the sum of the size of its buffers, validity, including nested arrays.
121    /// Multiple arrays may share buffers and bitmaps. Therefore, the size of 2 arrays is not the
122    /// sum of the sizes computed from this function. In particular, [`StructArray`]'s size is an upper bound.
123    ///
124    /// When an array is sliced, its allocated size remains constant because the buffer unchanged.
125    /// However, this function will yield a smaller number. This is because this function returns
126    /// the visible size of the buffer, not its total capacity.
127    ///
128    /// FFI buffers are included in this estimation.
129    ///
130    /// If `expanded` is true, scalar columns are counted as if they were materialized. Otherwise,
131    /// they are counted by the memory they currently use.
132    pub fn estimated_size(&self, expanded: bool) -> usize {
133        self.columns()
134            .iter()
135            .map(|c| c.estimated_size(expanded))
136            .sum()
137    }
138
139    pub fn try_apply_columns(
140        &self,
141        func: impl Fn(&Column) -> PolarsResult<Column> + Send + Sync,
142    ) -> PolarsResult<Vec<Column>> {
143        return inner(self, &func);
144
145        fn inner(
146            slf: &DataFrame,
147            func: &(dyn Fn(&Column) -> PolarsResult<Column> + Send + Sync),
148        ) -> PolarsResult<Vec<Column>> {
149            slf.columns().iter().map(func).collect()
150        }
151    }
152
153    pub fn apply_columns(&self, func: impl Fn(&Column) -> Column + Send + Sync) -> Vec<Column> {
154        return inner(self, &func);
155
156        fn inner(slf: &DataFrame, func: &(dyn Fn(&Column) -> Column + Send + Sync)) -> Vec<Column> {
157            slf.columns().iter().map(func).collect()
158        }
159    }
160
161    pub fn try_apply_columns_par(
162        &self,
163        func: impl Fn(&Column) -> PolarsResult<Column> + Send + Sync,
164    ) -> PolarsResult<Vec<Column>> {
165        return inner(self, &func);
166
167        fn inner(
168            slf: &DataFrame,
169            func: &(dyn Fn(&Column) -> PolarsResult<Column> + Send + Sync),
170        ) -> PolarsResult<Vec<Column>> {
171            RAYON.install(|| slf.columns().par_iter().map(func).collect())
172        }
173    }
174
175    pub fn apply_columns_par(&self, func: impl Fn(&Column) -> Column + Send + Sync) -> Vec<Column> {
176        return inner(self, &func);
177
178        fn inner(slf: &DataFrame, func: &(dyn Fn(&Column) -> Column + Send + Sync)) -> Vec<Column> {
179            RAYON.install(|| slf.columns().par_iter().map(func).collect())
180        }
181    }
182
183    /// Turns dictionary-coded columns into [`Series`] columns.
184    pub fn materialize_dictionary_columns(&mut self) {
185        if !self
186            .columns()
187            .iter()
188            .any(|c| c.as_dictionary_column().is_some())
189        {
190            return;
191        }
192        // SAFETY: a materialized column has the same name, dtype and length.
193        for c in unsafe { self.columns_mut_retain_schema() } {
194            if c.as_dictionary_column().is_some() {
195                c.into_materialized_series();
196            }
197        }
198    }
199
200    /// Reserve additional slots into the chunks of the series.
201    pub(crate) fn reserve_chunks(&mut self, additional: usize) {
202        for s in unsafe { self.columns_mut_retain_schema() } {
203            if let Column::Series(s) = s {
204                // SAFETY:
205                // do not modify the data, simply resize.
206                unsafe { s.chunks_mut().reserve(additional) }
207            }
208        }
209    }
210    pub fn new_from_index(&self, index: usize, height: usize) -> Self {
211        let new_cols = self.apply_columns(|c| c.new_from_index(index, height));
212
213        unsafe { Self::_new_unchecked_impl(height, new_cols).with_schema_from(self) }
214    }
215
216    /// Create a new `DataFrame` with the given schema, only containing nulls.
217    pub fn full_null(schema: &Schema, height: usize) -> Self {
218        let columns = schema
219            .iter_fields()
220            .map(|f| Column::full_null(f.name().clone(), height, f.dtype()))
221            .collect();
222
223        unsafe { DataFrame::_new_unchecked_impl(height, columns) }
224    }
225
226    /// Ensure this DataFrame matches the given schema. Casts null columns to
227    /// the expected schema if necessary (but nothing else).
228    pub fn ensure_matches_schema(&mut self, schema: &Schema) -> PolarsResult<()> {
229        let mut did_cast = false;
230        let cached_schema = self.cached_schema().cloned();
231
232        for (col, (name, dt)) in unsafe { self.columns_mut() }.iter_mut().zip(schema.iter()) {
233            polars_ensure!(
234                col.name() == name,
235                SchemaMismatch: "column name mismatch: expected {:?}, found {:?}",
236                name,
237                col.name()
238            );
239
240            let needs_cast = col.dtype().matches_schema_type(dt)?;
241
242            if needs_cast {
243                *col = col.cast(dt)?;
244                did_cast = true;
245            }
246        }
247
248        if !did_cast {
249            unsafe { self.set_opt_schema(cached_schema) };
250        }
251
252        Ok(())
253    }
254
255    /// Add a new column at index 0 that counts the rows.
256    ///
257    /// # Example
258    ///
259    /// ```
260    /// # use polars_core::prelude::*;
261    /// let df1: DataFrame = df!("Name" => ["James", "Mary", "John", "Patricia"])?;
262    /// assert_eq!(df1.shape(), (4, 1));
263    ///
264    /// let df2: DataFrame = df1.with_row_index("Id".into(), None)?;
265    /// assert_eq!(df2.shape(), (4, 2));
266    /// println!("{}", df2);
267    ///
268    /// # Ok::<(), PolarsError>(())
269    /// ```
270    ///
271    /// Output:
272    ///
273    /// ```text
274    ///  shape: (4, 2)
275    ///  +-----+----------+
276    ///  | Id  | Name     |
277    ///  | --- | ---      |
278    ///  | u32 | str      |
279    ///  +=====+==========+
280    ///  | 0   | James    |
281    ///  +-----+----------+
282    ///  | 1   | Mary     |
283    ///  +-----+----------+
284    ///  | 2   | John     |
285    ///  +-----+----------+
286    ///  | 3   | Patricia |
287    ///  +-----+----------+
288    /// ```
289    pub fn with_row_index(&self, name: PlSmallStr, offset: Option<IdxSize>) -> PolarsResult<Self> {
290        let mut new_columns = Vec::with_capacity(self.width() + 1);
291        let offset = offset.unwrap_or(0);
292
293        if self.get_column_index(&name).is_some() {
294            polars_bail!(duplicate = name)
295        }
296
297        let col = Column::new_row_index(name, offset, self.height())?;
298        new_columns.push(col);
299        new_columns.extend_from_slice(self.columns());
300
301        Ok(unsafe { DataFrame::new_unchecked(self.height(), new_columns) })
302    }
303
304    /// Add a row index column in place.
305    ///
306    /// # Safety
307    /// The caller should ensure the DataFrame does not already contain a column with the given name.
308    ///
309    /// # Panics
310    /// Panics if the resulting column would reach or overflow IdxSize::MAX.
311    pub unsafe fn with_row_index_mut(
312        &mut self,
313        name: PlSmallStr,
314        offset: Option<IdxSize>,
315    ) -> &mut Self {
316        debug_assert!(
317            self.get_column_index(&name).is_none(),
318            "with_row_index_mut(): column with name {} already exists",
319            name
320        );
321
322        let offset = offset.unwrap_or(0);
323        let col = Column::new_row_index(name, offset, self.height()).unwrap();
324
325        unsafe { self.columns_mut() }.insert(0, col);
326        self
327    }
328
329    /// Shrink the capacity of this DataFrame to fit its length.
330    pub fn shrink_to_fit(&mut self) {
331        // Don't parallelize this. Memory overhead
332        for s in unsafe { self.columns_mut_retain_schema() } {
333            s.shrink_to_fit();
334        }
335    }
336
337    /// Aggregate all the chunks in the DataFrame to a single chunk in parallel.
338    /// This may lead to more peak memory consumption.
339    pub fn rechunk_mut_par(&mut self) -> &mut Self {
340        if self.columns().iter().any(|c| c.n_chunks() > 1) {
341            RAYON.install(|| {
342                unsafe { self.columns_mut_retain_schema() }
343                    .par_iter_mut()
344                    .for_each(|c| *c = c.rechunk());
345            })
346        }
347
348        self
349    }
350
351    /// Rechunks all columns to only have a single chunk.
352    pub fn rechunk_mut(&mut self) -> &mut Self {
353        // SAFETY: We never adjust the length or names of the columns.
354        let columns = unsafe { self.columns_mut() };
355
356        for col in columns.iter_mut().filter(|c| c.n_chunks() > 1) {
357            *col = col.rechunk();
358        }
359
360        self
361    }
362
363    /// Returns true if the chunks of the columns do not align and re-chunking should be done
364    pub fn should_rechunk(&self) -> bool {
365        // Scalar columns have no chunks of their own and align with any layout, even when they
366        // are materialized.
367        //
368        // Fast check. It is also needed for correctness, as code below doesn't check if the number
369        // of chunks is equal.
370        if !self
371            .columns()
372            .iter()
373            .filter_map(column_chunk_lengths)
374            .map(|cl| cl.size_hint().0)
375            .all_equal()
376        {
377            return true;
378        }
379
380        let mut chunk_lengths = self.columns().iter().filter_map(column_chunk_lengths);
381        match chunk_lengths.next() {
382            None => false,
383            Some(first_column_chunk_lengths) => {
384                // Fast Path for single Chunk Series
385                if first_column_chunk_lengths.size_hint().0 == 1 {
386                    return chunk_lengths.any(|cl| cl.size_hint().0 != 1);
387                }
388                // Always rechunk if we have more chunks than rows.
389                // except when we have an empty df containing a single chunk
390                let height = self.height();
391                let n_chunks = first_column_chunk_lengths.size_hint().0;
392                if n_chunks > height && !(height == 0 && n_chunks == 1) {
393                    return true;
394                }
395                // Slow Path for multi Chunk series
396                let v: Vec<_> = first_column_chunk_lengths.collect();
397                for cl in chunk_lengths {
398                    if cl.enumerate().any(|(idx, el)| Some(&el) != v.get(idx)) {
399                        return true;
400                    }
401                }
402                false
403            },
404        }
405    }
406
407    /// Ensure all the chunks in the [`DataFrame`] are aligned.
408    pub fn align_chunks_par(&mut self) -> &mut Self {
409        if self.should_rechunk() {
410            self.rechunk_mut_par()
411        } else {
412            self
413        }
414    }
415
416    /// Ensure all the chunks in the [`DataFrame`] are aligned.
417    pub fn align_chunks(&mut self) -> &mut Self {
418        if self.should_rechunk() {
419            self.rechunk_mut()
420        } else {
421            self
422        }
423    }
424
425    /// # Example
426    ///
427    /// ```rust
428    /// # use polars_core::prelude::*;
429    /// let df: DataFrame = df!("Language" => ["Rust", "Python"],
430    ///                         "Designer" => ["Graydon Hoare", "Guido van Rossum"])?;
431    ///
432    /// assert_eq!(df.get_column_names(), &["Language", "Designer"]);
433    /// # Ok::<(), PolarsError>(())
434    /// ```
435    pub fn get_column_names(&self) -> Vec<&PlSmallStr> {
436        self.columns().iter().map(|s| s.name()).collect()
437    }
438
439    /// Get the [`Vec<PlSmallStr>`] representing the column names.
440    pub fn get_column_names_owned(&self) -> Vec<PlSmallStr> {
441        self.columns().iter().map(|s| s.name().clone()).collect()
442    }
443
444    /// Set the column names.
445    /// # Example
446    ///
447    /// ```rust
448    /// # use polars_core::prelude::*;
449    /// let mut df: DataFrame = df!("Mathematical set" => ["ℕ", "ℤ", "𝔻", "ℚ", "ℝ", "ℂ"])?;
450    /// df.set_column_names(&["Set"])?;
451    ///
452    /// assert_eq!(df.get_column_names(), &["Set"]);
453    /// # Ok::<(), PolarsError>(())
454    /// ```
455    pub fn set_column_names<T>(&mut self, new_names: &[T]) -> PolarsResult<()>
456    where
457        T: AsRef<str>,
458    {
459        polars_ensure!(
460            new_names.len() == self.width(),
461            ShapeMismatch: "{} column names provided for a DataFrame of width {}",
462            new_names.len(), self.width()
463        );
464
465        validation::ensure_names_unique(new_names)?;
466
467        *unsafe { self.columns_mut() } = std::mem::take(unsafe { self.columns_mut() })
468            .into_iter()
469            .zip(new_names)
470            .map(|(c, name)| c.with_name(PlSmallStr::from_str(name.as_ref())))
471            .collect();
472
473        Ok(())
474    }
475
476    /// Get the data types of the columns in the [`DataFrame`].
477    ///
478    /// # Example
479    ///
480    /// ```rust
481    /// # use polars_core::prelude::*;
482    /// let venus_air: DataFrame = df!("Element" => ["Carbon dioxide", "Nitrogen"],
483    ///                                "Fraction" => [0.965, 0.035])?;
484    ///
485    /// assert_eq!(venus_air.dtypes(), &[DataType::String, DataType::Float64]);
486    /// # Ok::<(), PolarsError>(())
487    /// ```
488    pub fn dtypes(&self) -> Vec<DataType> {
489        self.columns().iter().map(|s| s.dtype().clone()).collect()
490    }
491
492    /// The number of chunks for the first column.
493    pub fn first_col_n_chunks(&self) -> usize {
494        match self.chunk_lengths_of_leading_column() {
495            None if self.width() == 0 => 0,
496            None => 1,
497            Some(cl) => cl.size_hint().0,
498        }
499    }
500
501    /// The chunk lengths of the first column that has chunks, see [`Self::first_col_n_chunks`].
502    pub(crate) fn first_col_chunk_lengths(&self) -> impl Iterator<Item = usize> + '_ {
503        match self.chunk_lengths_of_leading_column() {
504            None => Either::Right((self.width() > 0).then_some(self.height()).into_iter()),
505            Some(cl) => Either::Left(cl),
506        }
507    }
508
509    /// The highest number of chunks for any column.
510    pub fn max_n_chunks(&self) -> usize {
511        self.columns()
512            .iter()
513            .map(|c| column_chunk_lengths(c).map_or(1, |cl| cl.size_hint().0))
514            .max()
515            .unwrap_or(0)
516    }
517
518    /// Generate the schema fields of the [`DataFrame`].
519    ///
520    /// # Example
521    ///
522    /// ```rust
523    /// # use polars_core::prelude::*;
524    /// let earth: DataFrame = df!("Surface type" => ["Water", "Land"],
525    ///                            "Fraction" => [0.708, 0.292])?;
526    ///
527    /// let f1: Field = Field::new("Surface type".into(), DataType::String);
528    /// let f2: Field = Field::new("Fraction".into(), DataType::Float64);
529    ///
530    /// assert_eq!(earth.fields(), &[f1, f2]);
531    /// # Ok::<(), PolarsError>(())
532    /// ```
533    pub fn fields(&self) -> Vec<Field> {
534        self.columns()
535            .iter()
536            .map(|s| s.field().into_owned())
537            .collect()
538    }
539
540    /// Add multiple [`Series`] to a [`DataFrame`].
541    /// The added `Series` are required to have the same length.
542    ///
543    /// # Example
544    ///
545    /// ```rust
546    /// # use polars_core::prelude::*;
547    /// let df1: DataFrame = df!("Element" => ["Copper", "Silver", "Gold"])?;
548    /// let s1 = Column::new("Proton".into(), [29, 47, 79]);
549    /// let s2 = Column::new("Electron".into(), [29, 47, 79]);
550    ///
551    /// let df2: DataFrame = df1.hstack(&[s1, s2])?;
552    /// assert_eq!(df2.shape(), (3, 3));
553    /// println!("{}", df2);
554    /// # Ok::<(), PolarsError>(())
555    /// ```
556    ///
557    /// Output:
558    ///
559    /// ```text
560    /// shape: (3, 3)
561    /// +---------+--------+----------+
562    /// | Element | Proton | Electron |
563    /// | ---     | ---    | ---      |
564    /// | str     | i32    | i32      |
565    /// +=========+========+==========+
566    /// | Copper  | 29     | 29       |
567    /// +---------+--------+----------+
568    /// | Silver  | 47     | 47       |
569    /// +---------+--------+----------+
570    /// | Gold    | 79     | 79       |
571    /// +---------+--------+----------+
572    /// ```
573    pub fn hstack(&self, columns: &[Column]) -> PolarsResult<Self> {
574        let mut new_cols = Vec::with_capacity(self.width() + columns.len());
575
576        new_cols.extend(self.columns().iter().cloned());
577        new_cols.extend_from_slice(columns);
578
579        DataFrame::new(self.height(), new_cols)
580    }
581    /// Concatenate a [`DataFrame`] to this [`DataFrame`] and return as newly allocated [`DataFrame`].
582    ///
583    /// If many `vstack` operations are done, it is recommended to call [`DataFrame::align_chunks_par`].
584    ///
585    /// # Example
586    ///
587    /// ```rust
588    /// # use polars_core::prelude::*;
589    /// let df1: DataFrame = df!("Element" => ["Copper", "Silver", "Gold"],
590    ///                          "Melting Point (K)" => [1357.77, 1234.93, 1337.33])?;
591    /// let df2: DataFrame = df!("Element" => ["Platinum", "Palladium"],
592    ///                          "Melting Point (K)" => [2041.4, 1828.05])?;
593    ///
594    /// let df3: DataFrame = df1.vstack(&df2)?;
595    ///
596    /// assert_eq!(df3.shape(), (5, 2));
597    /// println!("{}", df3);
598    /// # Ok::<(), PolarsError>(())
599    /// ```
600    ///
601    /// Output:
602    ///
603    /// ```text
604    /// shape: (5, 2)
605    /// +-----------+-------------------+
606    /// | Element   | Melting Point (K) |
607    /// | ---       | ---               |
608    /// | str       | f64               |
609    /// +===========+===================+
610    /// | Copper    | 1357.77           |
611    /// +-----------+-------------------+
612    /// | Silver    | 1234.93           |
613    /// +-----------+-------------------+
614    /// | Gold      | 1337.33           |
615    /// +-----------+-------------------+
616    /// | Platinum  | 2041.4            |
617    /// +-----------+-------------------+
618    /// | Palladium | 1828.05           |
619    /// +-----------+-------------------+
620    /// ```
621    pub fn vstack(&self, other: &DataFrame) -> PolarsResult<Self> {
622        let mut df = self.clone();
623        df.vstack_mut(other)?;
624        Ok(df)
625    }
626
627    /// Concatenate a [`DataFrame`] to this [`DataFrame`]
628    ///
629    /// If many `vstack` operations are done, it is recommended to call [`DataFrame::align_chunks_par`].
630    ///
631    /// # Example
632    ///
633    /// ```rust
634    /// # use polars_core::prelude::*;
635    /// let mut df1: DataFrame = df!("Element" => ["Copper", "Silver", "Gold"],
636    ///                          "Melting Point (K)" => [1357.77, 1234.93, 1337.33])?;
637    /// let df2: DataFrame = df!("Element" => ["Platinum", "Palladium"],
638    ///                          "Melting Point (K)" => [2041.4, 1828.05])?;
639    ///
640    /// df1.vstack_mut(&df2)?;
641    ///
642    /// assert_eq!(df1.shape(), (5, 2));
643    /// println!("{}", df1);
644    /// # Ok::<(), PolarsError>(())
645    /// ```
646    ///
647    /// Output:
648    ///
649    /// ```text
650    /// shape: (5, 2)
651    /// +-----------+-------------------+
652    /// | Element   | Melting Point (K) |
653    /// | ---       | ---               |
654    /// | str       | f64               |
655    /// +===========+===================+
656    /// | Copper    | 1357.77           |
657    /// +-----------+-------------------+
658    /// | Silver    | 1234.93           |
659    /// +-----------+-------------------+
660    /// | Gold      | 1337.33           |
661    /// +-----------+-------------------+
662    /// | Platinum  | 2041.4            |
663    /// +-----------+-------------------+
664    /// | Palladium | 1828.05           |
665    /// +-----------+-------------------+
666    /// ```
667    pub fn vstack_mut(&mut self, other: &DataFrame) -> PolarsResult<&mut Self> {
668        if self.width() != other.width() {
669            polars_ensure!(
670                self.shape() == (0, 0),
671                ShapeMismatch:
672                "unable to append to a DataFrame of shape {:?} with a DataFrame of width {}",
673                self.shape(), other.width(),
674            );
675
676            self.clone_from(other);
677
678            return Ok(self);
679        }
680
681        let new_height = usize::checked_add(self.height(), other.height()).unwrap();
682
683        unsafe { self.columns_mut_retain_schema() }
684            .iter_mut()
685            .zip(other.columns())
686            .try_for_each::<_, PolarsResult<_>>(|(left, right)| {
687                ensure_can_extend(&*left, right)?;
688                left.append(right)
689                    .with_context(|| format!("failed to vstack column '{}'", right.name()))?;
690                Ok(())
691            })?;
692
693        unsafe { self.set_height(new_height) };
694
695        Ok(self)
696    }
697
698    pub fn vstack_mut_owned(&mut self, other: DataFrame) -> PolarsResult<&mut Self> {
699        if self.width() != other.width() {
700            polars_ensure!(
701                self.shape() == (0, 0),
702                ShapeMismatch:
703                "unable to append to a DataFrame of width {} with a DataFrame of width {}",
704                self.width(), other.width(),
705            );
706
707            *self = other;
708
709            return Ok(self);
710        }
711
712        let new_height = usize::checked_add(self.height(), other.height()).unwrap();
713
714        unsafe { self.columns_mut_retain_schema() }
715            .iter_mut()
716            .zip(other.into_columns())
717            .try_for_each::<_, PolarsResult<_>>(|(left, right)| {
718                ensure_can_extend(&*left, &right)?;
719                let right_name = right.name().clone();
720                left.append_owned(right)
721                    .with_context(|| format!("failed to vstack column '{right_name}'"))?;
722                Ok(())
723            })?;
724
725        unsafe { self.set_height(new_height) };
726
727        Ok(self)
728    }
729
730    /// Concatenate a [`DataFrame`] to this [`DataFrame`]
731    ///
732    /// If many `vstack` operations are done, it is recommended to call [`DataFrame::align_chunks_par`].
733    ///
734    /// # Panics
735    /// Panics if the schema's don't match.
736    pub fn vstack_mut_unchecked(&mut self, other: &DataFrame) -> &mut Self {
737        let new_height = usize::checked_add(self.height(), other.height()).unwrap();
738
739        unsafe { self.columns_mut_retain_schema() }
740            .iter_mut()
741            .zip(other.columns())
742            .for_each(|(left, right)| {
743                left.append(right)
744                    .with_context(|| format!("failed to vstack column '{}'", right.name()))
745                    .expect("should not fail");
746            });
747
748        unsafe { self.set_height(new_height) };
749
750        self
751    }
752
753    /// Concatenate a [`DataFrame`] to this [`DataFrame`]
754    ///
755    /// If many `vstack` operations are done, it is recommended to call [`DataFrame::align_chunks_par`].
756    ///
757    /// # Panics
758    /// Panics if the schema's don't match.
759    pub fn vstack_mut_owned_unchecked(&mut self, other: DataFrame) -> &mut Self {
760        let new_height = usize::checked_add(self.height(), other.height()).unwrap();
761
762        unsafe { self.columns_mut_retain_schema() }
763            .iter_mut()
764            .zip(other.into_columns())
765            .for_each(|(left, right)| {
766                left.append_owned(right).expect("should not fail");
767            });
768
769        unsafe { self.set_height(new_height) };
770
771        self
772    }
773
774    /// Extend the memory backed by this [`DataFrame`] with the values from `other`.
775    ///
776    /// Different from [`vstack`](Self::vstack) which adds the chunks from `other` to the chunks of this [`DataFrame`]
777    /// `extend` appends the data from `other` to the underlying memory locations and thus may cause a reallocation.
778    ///
779    /// If this does not cause a reallocation, the resulting data structure will not have any extra chunks
780    /// and thus will yield faster queries.
781    ///
782    /// Prefer `extend` over `vstack` when you want to do a query after a single append. For instance during
783    /// online operations where you add `n` rows and rerun a query.
784    ///
785    /// Prefer `vstack` over `extend` when you want to append many times before doing a query. For instance
786    /// when you read in multiple files and want to store them in a single `DataFrame`. In the latter case, finish the sequence
787    /// of `append` operations with a [`rechunk`](Self::align_chunks_par).
788    pub fn extend(&mut self, other: &DataFrame) -> PolarsResult<()> {
789        polars_ensure!(
790            self.width() == other.width(),
791            ShapeMismatch:
792            "unable to extend a DataFrame of width {} with a DataFrame of width {}",
793            self.width(), other.width(),
794        );
795
796        let new_height = usize::checked_add(self.height(), other.height()).unwrap();
797
798        unsafe { self.columns_mut_retain_schema() }
799            .iter_mut()
800            .zip(other.columns())
801            .try_for_each::<_, PolarsResult<_>>(|(left, right)| {
802                ensure_can_extend(&*left, right)?;
803                left.extend(right)
804                    .with_context(|| format!("failed to extend column '{}'", right.name()))?;
805                Ok(())
806            })?;
807
808        unsafe { self.set_height(new_height) };
809
810        Ok(())
811    }
812
813    /// Remove a column by name and return the column removed.
814    ///
815    /// # Example
816    ///
817    /// ```rust
818    /// # use polars_core::prelude::*;
819    /// let mut df: DataFrame = df!("Animal" => ["Tiger", "Lion", "Great auk"],
820    ///                             "IUCN" => ["Endangered", "Vulnerable", "Extinct"])?;
821    ///
822    /// let s1: PolarsResult<Column> = df.drop_in_place("Average weight");
823    /// assert!(s1.is_err());
824    ///
825    /// let s2: Column = df.drop_in_place("Animal")?;
826    /// assert_eq!(s2, Column::new("Animal".into(), &["Tiger", "Lion", "Great auk"]));
827    /// # Ok::<(), PolarsError>(())
828    /// ```
829    pub fn drop_in_place(&mut self, name: &str) -> PolarsResult<Column> {
830        let idx = self.try_get_column_index(name)?;
831        Ok(unsafe { self.columns_mut() }.remove(idx))
832    }
833
834    /// Return a new [`DataFrame`] where all null values are dropped.
835    ///
836    /// # Example
837    ///
838    /// ```no_run
839    /// # use polars_core::prelude::*;
840    /// let df1: DataFrame = df!("Country" => ["Malta", "Liechtenstein", "North Korea"],
841    ///                         "Tax revenue (% GDP)" => [Some(32.7), None, None])?;
842    /// assert_eq!(df1.shape(), (3, 2));
843    ///
844    /// let df2: DataFrame = df1.drop_nulls::<String>(None)?;
845    /// assert_eq!(df2.shape(), (1, 2));
846    /// println!("{}", df2);
847    /// # Ok::<(), PolarsError>(())
848    /// ```
849    ///
850    /// Output:
851    ///
852    /// ```text
853    /// shape: (1, 2)
854    /// +---------+---------------------+
855    /// | Country | Tax revenue (% GDP) |
856    /// | ---     | ---                 |
857    /// | str     | f64                 |
858    /// +=========+=====================+
859    /// | Malta   | 32.7                |
860    /// +---------+---------------------+
861    /// ```
862    pub fn drop_nulls<S>(&self, subset: Option<&[S]>) -> PolarsResult<Self>
863    where
864        for<'a> &'a S: AsRef<str>,
865    {
866        if let Some(v) = subset {
867            let v = self.select_to_vec(v)?;
868            self._drop_nulls_impl(v.as_slice())
869        } else {
870            self._drop_nulls_impl(self.columns())
871        }
872    }
873
874    fn _drop_nulls_impl(&self, subset: &[Column]) -> PolarsResult<Self> {
875        // fast path for no nulls in df
876        if subset.iter().all(|s| !s.has_nulls()) {
877            return Ok(self.clone());
878        }
879
880        let mut iter = subset.iter();
881
882        let mask = iter
883            .next()
884            .ok_or_else(|| polars_err!(NoData: "no data to drop nulls from"))?;
885        let mut mask = mask.is_not_null();
886
887        for c in iter {
888            mask = mask & c.is_not_null();
889        }
890        self.filter(&mask)
891    }
892
893    /// Drop a column by name.
894    /// This is a pure method and will return a new [`DataFrame`] instead of modifying
895    /// the current one in place.
896    ///
897    /// # Example
898    ///
899    /// ```rust
900    /// # use polars_core::prelude::*;
901    /// let df1: DataFrame = df!("Ray type" => ["α", "β", "X", "γ"])?;
902    /// let df2: DataFrame = df1.drop("Ray type")?;
903    ///
904    /// assert_eq!(df2.width(), 0);
905    /// # Ok::<(), PolarsError>(())
906    /// ```
907    pub fn drop(&self, name: &str) -> PolarsResult<Self> {
908        let idx = self.try_get_column_index(name)?;
909        let mut new_cols = Vec::with_capacity(self.width() - 1);
910
911        self.columns().iter().enumerate().for_each(|(i, s)| {
912            if i != idx {
913                new_cols.push(s.clone())
914            }
915        });
916
917        Ok(unsafe { DataFrame::_new_unchecked_impl(self.height(), new_cols) })
918    }
919
920    /// Drop columns that are in `names`.
921    pub fn drop_many<I, S>(&self, names: I) -> Self
922    where
923        I: IntoIterator<Item = S>,
924        S: Into<PlSmallStr>,
925    {
926        let names: PlHashSet<PlSmallStr> = names.into_iter().map(|s| s.into()).collect();
927        self.drop_many_amortized(&names)
928    }
929
930    /// Drop columns that are in `names` without allocating a [`HashSet`](std::collections::HashSet).
931    pub fn drop_many_amortized(&self, names: &PlHashSet<PlSmallStr>) -> DataFrame {
932        if names.is_empty() {
933            return self.clone();
934        }
935        let mut new_cols = Vec::with_capacity(self.width().saturating_sub(names.len()));
936        self.columns().iter().for_each(|s| {
937            if !names.contains(s.name()) {
938                new_cols.push(s.clone())
939            }
940        });
941
942        unsafe { DataFrame::new_unchecked(self.height(), new_cols) }
943    }
944
945    /// Insert a new column at a given index without checking for duplicates.
946    /// This can leave the [`DataFrame`] at an invalid state
947    fn insert_column_no_namecheck(
948        &mut self,
949        index: usize,
950        column: Column,
951    ) -> PolarsResult<&mut Self> {
952        polars_ensure!(
953            column.len() == self.height(),
954            ShapeMismatch:
955            "unable to add a column of length {} to a DataFrame of height {}",
956            column.len(), self.height(),
957        );
958
959        unsafe { self.columns_mut() }.insert(index, column);
960        Ok(self)
961    }
962
963    /// Insert a new column at a given index.
964    pub fn insert_column(&mut self, index: usize, column: Column) -> PolarsResult<&mut Self> {
965        let name = column.name();
966
967        polars_ensure!(
968            self.get_column_index(name).is_none(),
969            Duplicate:
970            "column with name {:?} is already present in the DataFrame", name
971        );
972
973        self.insert_column_no_namecheck(index, column)
974    }
975
976    /// Add a new column to this [`DataFrame`] or replace an existing one. Broadcasts unit-length
977    /// columns.
978    pub fn with_column(&mut self, mut column: Column) -> PolarsResult<&mut Self> {
979        column.broadcast_in_place_to(self.height())?;
980
981        if let Some(i) = self.get_column_index(column.name()) {
982            *unsafe { self.columns_mut() }.get_mut(i).unwrap() = column
983        } else {
984            unsafe { self.columns_mut() }.push(column)
985        };
986
987        Ok(self)
988    }
989
990    /// Adds a column to the [`DataFrame`] without doing any checks
991    /// on length or duplicates.
992    ///
993    /// # Safety
994    /// The caller must ensure `column.len() == self.height()` .
995    pub unsafe fn push_column_unchecked(&mut self, column: Column) -> &mut Self {
996        unsafe { self.columns_mut() }.push(column);
997        self
998    }
999
1000    /// Add or replace columns to this [`DataFrame`] or replace an existing one.
1001    /// Broadcasts unit-length columns, and uses an existing schema to amortize lookups.
1002    pub fn with_columns_mut(
1003        &mut self,
1004        columns: impl IntoIterator<Item = Column>,
1005        output_schema: &Schema,
1006    ) -> PolarsResult<()> {
1007        let columns = columns.into_iter();
1008
1009        unsafe {
1010            self.columns_mut_retain_schema()
1011                .reserve(columns.size_hint().0)
1012        }
1013
1014        for c in columns {
1015            self.with_column_and_schema_mut(c, output_schema)?;
1016        }
1017
1018        Ok(())
1019    }
1020
1021    fn with_column_and_schema_mut(
1022        &mut self,
1023        mut column: Column,
1024        output_schema: &Schema,
1025    ) -> PolarsResult<&mut Self> {
1026        column.broadcast_in_place_to(self.height())?;
1027
1028        let i = output_schema
1029            .index_of(column.name())
1030            .or_else(|| self.get_column_index(column.name()))
1031            .unwrap_or(self.width());
1032
1033        if i < self.width() {
1034            *unsafe { self.columns_mut() }.get_mut(i).unwrap() = column
1035        } else if i == self.width() {
1036            unsafe { self.columns_mut() }.push(column)
1037        } else {
1038            // Unordered column insertion is not handled.
1039            panic!("{:?}, {}", output_schema, column.name());
1040        }
1041
1042        Ok(self)
1043    }
1044
1045    /// Get a row in the [`DataFrame`]. Beware this is slow.
1046    ///
1047    /// # Example
1048    ///
1049    /// ```
1050    /// # use polars_core::prelude::*;
1051    /// fn example(df: &mut DataFrame, idx: usize) -> Option<Vec<AnyValue>> {
1052    ///     df.get(idx)
1053    /// }
1054    /// ```
1055    pub fn get(&self, idx: usize) -> Option<Vec<AnyValue<'_>>> {
1056        (idx < self.height()).then(|| self.columns().iter().map(|c| c.get(idx).unwrap()).collect())
1057    }
1058
1059    /// Select a [`Series`] by index.
1060    ///
1061    /// # Example
1062    ///
1063    /// ```rust
1064    /// # use polars_core::prelude::*;
1065    /// let df: DataFrame = df!("Star" => ["Sun", "Betelgeuse", "Sirius A", "Sirius B"],
1066    ///                         "Absolute magnitude" => [4.83, -5.85, 1.42, 11.18])?;
1067    ///
1068    /// let s1: Option<&Column> = df.select_at_idx(0);
1069    /// let s2 = Column::new("Star".into(), ["Sun", "Betelgeuse", "Sirius A", "Sirius B"]);
1070    ///
1071    /// assert_eq!(s1, Some(&s2));
1072    /// # Ok::<(), PolarsError>(())
1073    /// ```
1074    pub fn select_at_idx(&self, idx: usize) -> Option<&Column> {
1075        self.columns().get(idx)
1076    }
1077
1078    /// Get column index of a [`Series`] by name.
1079    /// # Example
1080    ///
1081    /// ```rust
1082    /// # use polars_core::prelude::*;
1083    /// let df: DataFrame = df!("Name" => ["Player 1", "Player 2", "Player 3"],
1084    ///                         "Health" => [100, 200, 500],
1085    ///                         "Mana" => [250, 100, 0],
1086    ///                         "Strength" => [30, 150, 300])?;
1087    ///
1088    /// assert_eq!(df.get_column_index("Name"), Some(0));
1089    /// assert_eq!(df.get_column_index("Health"), Some(1));
1090    /// assert_eq!(df.get_column_index("Mana"), Some(2));
1091    /// assert_eq!(df.get_column_index("Strength"), Some(3));
1092    /// assert_eq!(df.get_column_index("Haste"), None);
1093    /// # Ok::<(), PolarsError>(())
1094    /// ```
1095    pub fn get_column_index(&self, name: &str) -> Option<usize> {
1096        if let Some(schema) = self.cached_schema() {
1097            schema.index_of(name)
1098        } else if self.width() <= LINEAR_SEARCH_LIMIT {
1099            self.columns().iter().position(|s| s.name() == name)
1100        } else {
1101            self.schema().index_of(name)
1102        }
1103    }
1104
1105    /// Get column index of a [`Series`] by name.
1106    pub fn try_get_column_index(&self, name: &str) -> PolarsResult<usize> {
1107        self.get_column_index(name)
1108            .ok_or_else(|| polars_err!(col_not_found = name))
1109    }
1110
1111    /// Select a single column by name.
1112    ///
1113    /// # Example
1114    ///
1115    /// ```rust
1116    /// # use polars_core::prelude::*;
1117    /// let s1 = Column::new("Password".into(), ["123456", "[]B$u$g$s$B#u#n#n#y[]{}"]);
1118    /// let s2 = Column::new("Robustness".into(), ["Weak", "Strong"]);
1119    /// let df: DataFrame = DataFrame::new_infer_height(vec![s1.clone(), s2])?;
1120    ///
1121    /// assert_eq!(df.column("Password")?, &s1);
1122    /// # Ok::<(), PolarsError>(())
1123    /// ```
1124    pub fn column(&self, name: &str) -> PolarsResult<&Column> {
1125        let idx = self.try_get_column_index(name)?;
1126        Ok(self.select_at_idx(idx).unwrap())
1127    }
1128
1129    /// Select column(s) from this [`DataFrame`] and return a new [`DataFrame`].
1130    ///
1131    /// # Examples
1132    ///
1133    /// ```
1134    /// # use polars_core::prelude::*;
1135    /// fn example(df: &DataFrame) -> PolarsResult<DataFrame> {
1136    ///     df.select(["foo", "bar"])
1137    /// }
1138    /// ```
1139    pub fn select<I, S>(&self, names: I) -> PolarsResult<Self>
1140    where
1141        I: IntoIterator<Item = S>,
1142        S: AsRef<str>,
1143    {
1144        DataFrame::new(self.height(), self.select_to_vec(names)?)
1145    }
1146
1147    /// Does not check for duplicates.
1148    ///
1149    /// # Safety
1150    /// `names` must not contain duplicates.
1151    pub unsafe fn select_unchecked<I, S>(&self, names: I) -> PolarsResult<Self>
1152    where
1153        I: IntoIterator<Item = S>,
1154        S: AsRef<str>,
1155    {
1156        Ok(unsafe { DataFrame::new_unchecked(self.height(), self.select_to_vec(names)?) })
1157    }
1158
1159    /// Select column(s) from this [`DataFrame`] and return them into a [`Vec`].
1160    ///
1161    /// This does not error on duplicate selections.
1162    ///
1163    /// # Example
1164    ///
1165    /// ```rust
1166    /// # use polars_core::prelude::*;
1167    /// let df: DataFrame = df!("Name" => ["Methane", "Ethane", "Propane"],
1168    ///                         "Carbon" => [1, 2, 3],
1169    ///                         "Hydrogen" => [4, 6, 8])?;
1170    /// let sv: Vec<Column> = df.select_to_vec(["Carbon", "Hydrogen"])?;
1171    ///
1172    /// assert_eq!(df["Carbon"], sv[0]);
1173    /// assert_eq!(df["Hydrogen"], sv[1]);
1174    /// # Ok::<(), PolarsError>(())
1175    /// ```
1176    pub fn select_to_vec(
1177        &self,
1178        selection: impl IntoIterator<Item = impl AsRef<str>>,
1179    ) -> PolarsResult<Vec<Column>> {
1180        AmortizedColumnSelector::new(self).select_multiple(selection)
1181    }
1182
1183    /// Take the [`DataFrame`] rows by a boolean mask.
1184    ///
1185    /// # Example
1186    ///
1187    /// ```
1188    /// # use polars_core::prelude::*;
1189    /// fn example(df: &DataFrame) -> PolarsResult<DataFrame> {
1190    ///     let mask = df.column("sepal_width")?.is_not_null();
1191    ///     df.filter(&mask)
1192    /// }
1193    /// ```
1194    pub fn filter(&self, mask: &BooleanChunked) -> PolarsResult<Self> {
1195        if self.width() == 0 {
1196            filter_zero_width(self.height(), mask)
1197        } else if mask.len() == 1 && self.len() >= 1 {
1198            if mask.all() && mask.null_count() == 0 {
1199                Ok(self.clone())
1200            } else {
1201                Ok(self.clear())
1202            }
1203        } else {
1204            // Rechunk when not all chunks are aligned. This avoid O(n*m) overhead,
1205            // where n = number of chunks, and m = number of columns.
1206            let all_chunks_aligned =
1207                !self.should_rechunk() && self.first_col_chunk_lengths().eq(mask.chunk_lengths());
1208
1209            let mask = if all_chunks_aligned {
1210                Cow::Borrowed(mask)
1211            } else {
1212                mask.rechunk()
1213            };
1214
1215            let new_columns: Vec<Column> =
1216                self.try_apply_columns_par(|s| s.filter(mask.as_ref()))?;
1217            let out = unsafe {
1218                DataFrame::new_unchecked(new_columns[0].len(), new_columns).with_schema_from(self)
1219            };
1220
1221            Ok(out)
1222        }
1223    }
1224
1225    /// Same as `filter` but does not parallelize.
1226    pub fn filter_seq(&self, mask: &BooleanChunked) -> PolarsResult<Self> {
1227        if self.width() == 0 {
1228            filter_zero_width(self.height(), mask)
1229        } else if mask.len() == 1 && mask.null_count() == 0 && self.len() >= 1 {
1230            if mask.all() && mask.null_count() == 0 {
1231                Ok(self.clone())
1232            } else {
1233                Ok(self.clear())
1234            }
1235        } else {
1236            let all_chunks_aligned =
1237                !self.should_rechunk() && self.first_col_chunk_lengths().eq(mask.chunk_lengths());
1238
1239            let mask = if all_chunks_aligned {
1240                Cow::Borrowed(mask)
1241            } else {
1242                mask.rechunk()
1243            };
1244
1245            let new_columns: Vec<Column> = self.try_apply_columns(|s| s.filter(mask.as_ref()))?;
1246            let out = unsafe {
1247                DataFrame::new_unchecked(new_columns[0].len(), new_columns).with_schema_from(self)
1248            };
1249
1250            Ok(out)
1251        }
1252    }
1253
1254    /// Gather [`DataFrame`] rows by index values.
1255    ///
1256    /// # Example
1257    ///
1258    /// ```
1259    /// # use polars_core::prelude::*;
1260    /// fn example(df: &DataFrame) -> PolarsResult<DataFrame> {
1261    ///     let idx = IdxCa::new("idx".into(), [0, 1, 9]);
1262    ///     df.take(&idx)
1263    /// }
1264    /// ```
1265    pub fn take(&self, indices: &IdxCa) -> PolarsResult<Self> {
1266        check_bounds_ca(indices, self.height().try_into().unwrap_or(IdxSize::MAX))?;
1267
1268        let new_cols = self.apply_columns_par(|c| {
1269            assert_eq!(c.len(), self.height());
1270            unsafe { c.take_unchecked(indices) }
1271        });
1272
1273        Ok(unsafe { DataFrame::new_unchecked(indices.len(), new_cols).with_schema_from(self) })
1274    }
1275
1276    /// # Safety
1277    /// The indices must be in-bounds.
1278    pub unsafe fn take_unchecked(&self, idx: &IdxCa) -> Self {
1279        self.take_unchecked_impl(idx, true)
1280    }
1281
1282    /// # Safety
1283    /// The indices must be in-bounds.
1284    #[cfg(feature = "algorithm_group_by")]
1285    pub unsafe fn gather_group_unchecked(&self, group: &GroupsIndicator) -> Self {
1286        match group {
1287            GroupsIndicator::Idx((_, indices)) => unsafe {
1288                self.take_slice_unchecked_impl(indices.as_slice(), false)
1289            },
1290            GroupsIndicator::Slice([offset, len]) => self.slice((*offset as usize, *len as usize)),
1291        }
1292    }
1293
1294    /// # Safety
1295    /// The indices must be in-bounds.
1296    pub unsafe fn take_unchecked_impl(&self, idx: &IdxCa, allow_threads: bool) -> Self {
1297        let cols = if allow_threads && RAYON.current_num_threads() > 1 {
1298            RAYON.install(|| {
1299                if RAYON.current_num_threads() > self.width() {
1300                    let stride = usize::max(idx.len().div_ceil(RAYON.current_num_threads()), 256);
1301                    if self.height() / stride >= 2 {
1302                        self.apply_columns_par(|c| {
1303                            // Nested types initiate a rechunk in their take_unchecked implementation.
1304                            // If we do not rechunk, it will result in rechunk storms downstream.
1305                            let c = if c.dtype().is_nested() {
1306                                &c.rechunk()
1307                            } else {
1308                                c
1309                            };
1310
1311                            (0..idx.len().div_ceil(stride))
1312                                .into_par_iter()
1313                                .map(|i| c.take_unchecked(&idx.slice((i * stride, stride))))
1314                                .reduce(
1315                                    || Column::new_empty(c.name().clone(), c.dtype()),
1316                                    |mut a, b| {
1317                                        a.append_owned(b).unwrap();
1318                                        a
1319                                    },
1320                                )
1321                        })
1322                    } else {
1323                        self.apply_columns_par(|c| c.take_unchecked(idx))
1324                    }
1325                } else {
1326                    self.apply_columns_par(|c| c.take_unchecked(idx))
1327                }
1328            })
1329        } else {
1330            self.apply_columns(|s| s.take_unchecked(idx))
1331        };
1332
1333        unsafe { DataFrame::new_unchecked(idx.len(), cols).with_schema_from(self) }
1334    }
1335
1336    /// # Safety
1337    /// The indices must be in-bounds.
1338    pub unsafe fn take_slice_unchecked(&self, idx: &[IdxSize]) -> Self {
1339        self.take_slice_unchecked_impl(idx, true)
1340    }
1341
1342    /// # Safety
1343    /// The indices must be in-bounds.
1344    pub unsafe fn take_slice_unchecked_impl(&self, idx: &[IdxSize], allow_threads: bool) -> Self {
1345        let cols = if allow_threads && RAYON.current_num_threads() > 1 {
1346            RAYON.install(|| {
1347                if RAYON.current_num_threads() > self.width() {
1348                    let stride = usize::max(idx.len().div_ceil(RAYON.current_num_threads()), 256);
1349                    if self.height() / stride >= 2 {
1350                        self.apply_columns_par(|c| {
1351                            // Nested types initiate a rechunk in their take_unchecked implementation.
1352                            // If we do not rechunk, it will result in rechunk storms downstream.
1353                            let c = if c.dtype().is_nested() {
1354                                &c.rechunk()
1355                            } else {
1356                                c
1357                            };
1358
1359                            (0..idx.len().div_ceil(stride))
1360                                .into_par_iter()
1361                                .map(|i| {
1362                                    let idx = &idx[i * stride..];
1363                                    let idx = &idx[..idx.len().min(stride)];
1364                                    c.take_slice_unchecked(idx)
1365                                })
1366                                .reduce(
1367                                    || Column::new_empty(c.name().clone(), c.dtype()),
1368                                    |mut a, b| {
1369                                        a.append_owned(b).unwrap();
1370                                        a
1371                                    },
1372                                )
1373                        })
1374                    } else {
1375                        self.apply_columns_par(|s| s.take_slice_unchecked(idx))
1376                    }
1377                } else {
1378                    self.apply_columns_par(|s| s.take_slice_unchecked(idx))
1379                }
1380            })
1381        } else {
1382            self.apply_columns(|s| s.take_slice_unchecked(idx))
1383        };
1384        unsafe { DataFrame::new_unchecked(idx.len(), cols).with_schema_from(self) }
1385    }
1386
1387    /// Rename a column in the [`DataFrame`].
1388    ///
1389    /// Should not be called in a loop as that can lead to quadratic behavior.
1390    ///
1391    /// # Example
1392    ///
1393    /// ```
1394    /// # use polars_core::prelude::*;
1395    /// fn example(df: &mut DataFrame) -> PolarsResult<&mut DataFrame> {
1396    ///     let original_name = "foo";
1397    ///     let new_name = "bar";
1398    ///     df.rename(original_name, new_name.into())
1399    /// }
1400    /// ```
1401    pub fn rename(&mut self, column: &str, name: PlSmallStr) -> PolarsResult<&mut Self> {
1402        if column == name.as_str() {
1403            return Ok(self);
1404        }
1405        polars_ensure!(
1406            !self.schema().contains(&name),
1407            Duplicate: "column rename attempted with already existing name \"{name}\""
1408        );
1409
1410        self.get_column_index(column)
1411            .and_then(|idx| unsafe { self.columns_mut() }.get_mut(idx))
1412            .ok_or_else(|| polars_err!(col_not_found = column))
1413            .map(|c| c.rename(name))?;
1414
1415        Ok(self)
1416    }
1417
1418    pub fn rename_many<'a>(
1419        mut self,
1420        renames: impl Iterator<Item = (&'a str, PlSmallStr)>,
1421    ) -> PolarsResult<Self> {
1422        let schema = self.schema().clone();
1423
1424        for (from, to) in renames {
1425            if from == to.as_str() {
1426                continue;
1427            }
1428
1429            let idx = schema
1430                .index_of(from)
1431                .ok_or_else(|| polars_err!(col_not_found = from))?;
1432
1433            unsafe { self.columns_mut() }
1434                .get_mut(idx)
1435                .unwrap()
1436                .rename(to);
1437        }
1438
1439        // Check for duplicates.
1440        let schema = Schema::from_iter_check_duplicates(
1441            self.columns()
1442                .iter()
1443                .map(|c| c.name().clone())
1444                .zip_eq(schema.iter_values().cloned()),
1445        )?;
1446
1447        unsafe { self.set_schema(Arc::new(schema)) };
1448
1449        Ok(self)
1450    }
1451
1452    /// Sort [`DataFrame`] in place.
1453    ///
1454    /// See [`DataFrame::sort`] for more instruction.
1455    pub fn sort_in_place(
1456        &mut self,
1457        by: impl IntoIterator<Item = impl AsRef<str>>,
1458        sort_options: SortMultipleOptions,
1459    ) -> PolarsResult<&mut Self> {
1460        let by: Vec<_> = by.into_iter().collect();
1461        // Several keys are sorted through a row encoding of single chunks; one
1462        // key may skip the sort by its sorted flag.
1463        if by.len() > 1 {
1464            self.rechunk_mut_par();
1465        }
1466        let by_column = self.select_to_vec(by)?;
1467
1468        let mut out = self.clone().sort_impl(by_column, sort_options, None)?;
1469        unsafe { out.set_schema_from(self) };
1470
1471        *self = out;
1472
1473        Ok(self)
1474    }
1475
1476    #[doc(hidden)]
1477    /// This is the dispatch of Self::sort, and exists to reduce compile bloat by monomorphization.
1478    pub fn sort_impl(
1479        self,
1480        mut by_column: Vec<Column>,
1481        sort_options: SortMultipleOptions,
1482        slice: Option<Slice>,
1483    ) -> PolarsResult<Self> {
1484        if by_column.is_empty() {
1485            // If no columns selected, any order (including original order) is correct.
1486            return Ok(match slice {
1487                Some(slice) => self.slice(slice),
1488                None => self,
1489            });
1490        }
1491
1492        for column in &by_column {
1493            if column.dtype().is_object() {
1494                polars_bail!(
1495                    InvalidOperation: "column '{}' has a dtype of '{}', which does not support sorting", column.name(), column.dtype()
1496                )
1497            }
1498        }
1499
1500        // note that the by_column argument also contains evaluated expression from
1501        // polars-lazy that may not even be present in this dataframe. therefore
1502        // when we try to set the first columns as sorted, we ignore the error as
1503        // expressions are not present (they are renamed to _POLARS_SORT_COLUMN_i.
1504        let first_descending = sort_options.descending[0];
1505        let first_by_column = by_column[0].name().to_string();
1506
1507        let set_sorted = |df: &mut DataFrame| {
1508            // Mark the first sort column as sorted; if the column does not exist it
1509            // is ok, because we sorted by an expression not present in the dataframe
1510            let _ = df.apply(&first_by_column, |s| {
1511                let mut s = s.clone();
1512                if first_descending {
1513                    s.set_sorted_flag(IsSorted::Descending)
1514                } else {
1515                    s.set_sorted_flag(IsSorted::Ascending)
1516                }
1517                s
1518            });
1519        };
1520
1521        if self.shape_has_zero() {
1522            let mut out = self;
1523            set_sorted(&mut out);
1524            return Ok(out);
1525        }
1526
1527        if let Some(Slice::Positive { offset: 0, len: k }) = slice {
1528            if k < self.height() {
1529                return self.bottom_k_impl(k, by_column, sort_options);
1530            }
1531        }
1532        // Check if the required column is already sorted; if so we can exit early
1533        // We can do so when there is only one column to sort by, for multiple columns
1534        // it will be complicated to do so
1535        #[cfg(feature = "dtype-categorical")]
1536        let is_not_categorical_enum =
1537            !(matches!(by_column[0].dtype(), DataType::Categorical(_, _))
1538                || matches!(by_column[0].dtype(), DataType::Enum(_, _)));
1539
1540        #[cfg(not(feature = "dtype-categorical"))]
1541        #[allow(non_upper_case_globals)]
1542        const is_not_categorical_enum: bool = true;
1543
1544        if by_column.len() == 1 && is_not_categorical_enum {
1545            let required_sorting = if sort_options.descending[0] {
1546                IsSorted::Descending
1547            } else {
1548                IsSorted::Ascending
1549            };
1550            // If null count is 0 then nulls_last doesn't matter
1551            // Safe to get value at last position since the dataframe is not empty (taken care above)
1552            let no_sorting_required = (by_column[0].is_sorted_flag() == required_sorting)
1553                && ((by_column[0].null_count() == 0)
1554                    || by_column[0].get(by_column[0].len() - 1).unwrap().is_null()
1555                        == sort_options.nulls_last[0]);
1556
1557            if no_sorting_required {
1558                return Ok(match slice {
1559                    Some(slice) => self.slice(slice),
1560                    None => self,
1561                });
1562            }
1563        }
1564
1565        let has_nested = by_column.iter().any(|s| s.dtype().is_nested());
1566        let allow_threads = sort_options.multithreaded;
1567
1568        // a lot of indirection in both sorting and take; rechunking the owned
1569        // frame frees the old chunks
1570        let mut df = self;
1571        if df.columns().iter().any(|c| c.n_chunks() > 1) {
1572            // Keys that are columns of this frame take the rechunked columns,
1573            // so that their old chunks are freed too.
1574            let frame_key_idxs: Vec<Option<usize>> = by_column
1575                .iter()
1576                .map(|c| {
1577                    let s = c.as_series()?;
1578                    let idx = df.get_column_index(s.name())?;
1579                    let ds = df.columns()[idx].as_series()?;
1580                    Arc::ptr_eq(&ds.0, &s.0).then_some(idx)
1581                })
1582                .collect();
1583            df.rechunk_mut_par();
1584            for (c, idx) in by_column.iter_mut().zip(frame_key_idxs) {
1585                if let Some(idx) = idx {
1586                    *c = df.columns()[idx].clone();
1587                }
1588            }
1589        }
1590        let df = &df;
1591        let mut take = match (by_column.len(), has_nested) {
1592            (1, false) => {
1593                let s = &by_column[0];
1594                let options = SortOptions {
1595                    descending: sort_options.descending[0],
1596                    nulls_last: sort_options.nulls_last[0],
1597                    multithreaded: sort_options.multithreaded,
1598                    maintain_order: sort_options.maintain_order,
1599                    limit: sort_options.limit,
1600                };
1601                // fast path for a frame with a single series
1602                // no need to compute the sort indices and then take by these indices
1603                // simply sort and return as frame
1604                if df.width() == 1 && df.try_get_column_index(s.name().as_str()).is_ok() {
1605                    let mut out = s.sort_with(options)?;
1606                    if let Some(slice) = slice {
1607                        out = out.slice(slice);
1608                    }
1609                    return Ok(out.into_frame());
1610                }
1611                s.arg_sort(options)
1612            },
1613            _ => arg_sort(&by_column, sort_options)?,
1614        };
1615
1616        if let Some(slice) = slice {
1617            take = take.slice(slice);
1618        }
1619
1620        // SAFETY:
1621        // the created indices are in bounds
1622        let mut df = unsafe { df.take_unchecked_impl(&take, allow_threads) };
1623        set_sorted(&mut df);
1624        Ok(df)
1625    }
1626
1627    /// Create a `DataFrame` that has fields for all the known runtime metadata for each column.
1628    ///
1629    /// This dataframe does not necessarily have a specified schema and may be changed at any
1630    /// point. It is primarily used for debugging.
1631    pub fn _to_metadata(&self) -> DataFrame {
1632        let num_columns = self.width();
1633
1634        let mut column_names =
1635            StringChunkedBuilder::new(PlSmallStr::from_static("column_name"), num_columns);
1636        let mut repr_ca = StringChunkedBuilder::new(PlSmallStr::from_static("repr"), num_columns);
1637        let mut sorted_asc_ca =
1638            BooleanChunkedBuilder::new(PlSmallStr::from_static("sorted_asc"), num_columns);
1639        let mut sorted_dsc_ca =
1640            BooleanChunkedBuilder::new(PlSmallStr::from_static("sorted_dsc"), num_columns);
1641        let mut fast_explode_list_ca =
1642            BooleanChunkedBuilder::new(PlSmallStr::from_static("fast_explode_list"), num_columns);
1643        let mut materialized_at_ca =
1644            StringChunkedBuilder::new(PlSmallStr::from_static("materialized_at"), num_columns);
1645
1646        for col in self.columns() {
1647            let flags = col.get_flags();
1648
1649            let (repr, materialized_at) = match col {
1650                Column::Series(s) => ("series", s.materialized_at()),
1651                Column::Scalar(_) => ("scalar", None),
1652                Column::Dictionary(_) => ("dictionary", None),
1653            };
1654            let sorted_asc = flags.contains(StatisticsFlags::IS_SORTED_ASC);
1655            let sorted_dsc = flags.contains(StatisticsFlags::IS_SORTED_DSC);
1656            let fast_explode_list = flags.contains(StatisticsFlags::CAN_FAST_EXPLODE_LIST);
1657
1658            column_names.append_value(col.name().clone());
1659            repr_ca.append_value(repr);
1660            sorted_asc_ca.append_value(sorted_asc);
1661            sorted_dsc_ca.append_value(sorted_dsc);
1662            fast_explode_list_ca.append_value(fast_explode_list);
1663            materialized_at_ca.append_option(materialized_at.map(|v| format!("{v:#?}")));
1664        }
1665
1666        unsafe {
1667            DataFrame::new_unchecked(
1668                self.width(),
1669                vec![
1670                    column_names.finish().into_column(),
1671                    repr_ca.finish().into_column(),
1672                    sorted_asc_ca.finish().into_column(),
1673                    sorted_dsc_ca.finish().into_column(),
1674                    fast_explode_list_ca.finish().into_column(),
1675                    materialized_at_ca.finish().into_column(),
1676                ],
1677            )
1678        }
1679    }
1680    /// Return a sorted clone of this [`DataFrame`].
1681    ///
1682    /// In many cases the output chunks will be continuous in memory but this is not guaranteed
1683    /// # Example
1684    ///
1685    /// Sort by a single column with default options:
1686    /// ```
1687    /// # use polars_core::prelude::*;
1688    /// fn sort_by_sepal_width(df: &DataFrame) -> PolarsResult<DataFrame> {
1689    ///     df.sort(["sepal_width"], Default::default())
1690    /// }
1691    /// ```
1692    /// Sort by a single column with specific order:
1693    /// ```
1694    /// # use polars_core::prelude::*;
1695    /// fn sort_with_specific_order(df: &DataFrame, descending: bool) -> PolarsResult<DataFrame> {
1696    ///     df.sort(
1697    ///         ["sepal_width"],
1698    ///         SortMultipleOptions::new()
1699    ///             .with_order_descending(descending)
1700    ///     )
1701    /// }
1702    /// ```
1703    /// Sort by multiple columns with specifying order for each column:
1704    /// ```
1705    /// # use polars_core::prelude::*;
1706    /// fn sort_by_multiple_columns_with_specific_order(df: &DataFrame) -> PolarsResult<DataFrame> {
1707    ///     df.sort(
1708    ///         ["sepal_width", "sepal_length"],
1709    ///         SortMultipleOptions::new()
1710    ///             .with_order_descending_multi([false, true])
1711    ///     )
1712    /// }
1713    /// ```
1714    /// See [`SortMultipleOptions`] for more options.
1715    ///
1716    /// Also see [`DataFrame::sort_in_place`].
1717    pub fn sort(
1718        &self,
1719        by: impl IntoIterator<Item = impl AsRef<str>>,
1720        sort_options: SortMultipleOptions,
1721    ) -> PolarsResult<Self> {
1722        let mut df = self.clone();
1723        df.sort_in_place(by, sort_options)?;
1724        Ok(df)
1725    }
1726
1727    /// Replace a column with a [`Column`].
1728    ///
1729    /// # Example
1730    ///
1731    /// ```rust
1732    /// # use polars_core::prelude::*;
1733    /// let mut df: DataFrame = df!("Country" => ["United States", "China"],
1734    ///                         "Area (km²)" => [9_833_520, 9_596_961])?;
1735    /// let s: Column = Column::new("Country".into(), ["USA", "PRC"]);
1736    ///
1737    /// assert!(df.replace("Nation", s.clone()).is_err());
1738    /// assert!(df.replace("Country", s).is_ok());
1739    /// # Ok::<(), PolarsError>(())
1740    /// ```
1741    pub fn replace(&mut self, column: &str, new_col: Column) -> PolarsResult<&mut Self> {
1742        self.apply(column, |_| new_col)
1743    }
1744
1745    /// Replace column at index `idx` with a [`Series`].
1746    ///
1747    /// # Example
1748    ///
1749    /// ```ignored
1750    /// # use polars_core::prelude::*;
1751    /// let s0 = Series::new("foo".into(), ["ham", "spam", "egg"]);
1752    /// let s1 = Series::new("ascii".into(), [70, 79, 79]);
1753    /// let mut df = DataFrame::new_infer_height(vec![s0, s1])?;
1754    ///
1755    /// // Add 32 to get lowercase ascii values
1756    /// df.replace_column(1, df.select_at_idx(1).unwrap() + 32);
1757    /// # Ok::<(), PolarsError>(())
1758    /// ```
1759    pub fn replace_column(&mut self, index: usize, new_column: Column) -> PolarsResult<&mut Self> {
1760        polars_ensure!(
1761            index < self.width(),
1762            ShapeMismatch:
1763            "unable to replace at index {}, the DataFrame has only {} columns",
1764            index, self.width(),
1765        );
1766
1767        polars_ensure!(
1768            new_column.len() == self.height(),
1769            ShapeMismatch:
1770            "unable to replace a column, series length {} doesn't match the DataFrame height {}",
1771            new_column.len(), self.height(),
1772        );
1773
1774        unsafe { *self.columns_mut().get_mut(index).unwrap() = new_column };
1775
1776        Ok(self)
1777    }
1778
1779    /// Apply a closure to a column. This is the recommended way to do in place modification.
1780    ///
1781    /// # Example
1782    ///
1783    /// ```rust
1784    /// # use polars_core::prelude::*;
1785    /// let s0 = Column::new("foo".into(), ["ham", "spam", "egg"]);
1786    /// let s1 = Column::new("names".into(), ["Jean", "Claude", "van"]);
1787    /// let mut df = DataFrame::new_infer_height(vec![s0, s1])?;
1788    ///
1789    /// fn str_to_len(str_val: &Column) -> Column {
1790    ///     str_val.str()
1791    ///         .unwrap()
1792    ///         .iter()
1793    ///         .map(|opt_name: Option<&str>| {
1794    ///             opt_name.map(|name: &str| name.len() as u32)
1795    ///          })
1796    ///         .collect::<UInt32Chunked>()
1797    ///         .into_column()
1798    /// }
1799    ///
1800    /// // Replace the names column by the length of the names.
1801    /// df.apply("names", str_to_len);
1802    /// # Ok::<(), PolarsError>(())
1803    /// ```
1804    /// Results in:
1805    ///
1806    /// ```text
1807    /// +--------+-------+
1808    /// | foo    |       |
1809    /// | ---    | names |
1810    /// | str    | u32   |
1811    /// +========+=======+
1812    /// | "ham"  | 4     |
1813    /// +--------+-------+
1814    /// | "spam" | 6     |
1815    /// +--------+-------+
1816    /// | "egg"  | 3     |
1817    /// +--------+-------+
1818    /// ```
1819    pub fn apply<F, C>(&mut self, name: &str, f: F) -> PolarsResult<&mut Self>
1820    where
1821        F: FnOnce(&Column) -> C,
1822        C: IntoColumn,
1823    {
1824        let idx = self.try_get_column_index(name)?;
1825        self.apply_at_idx(idx, f)?;
1826        Ok(self)
1827    }
1828
1829    /// Apply a closure to a column at index `idx`. This is the recommended way to do in place
1830    /// modification.
1831    ///
1832    /// # Example
1833    ///
1834    /// ```rust
1835    /// # use polars_core::prelude::*;
1836    /// let s0 = Column::new("foo".into(), ["ham", "spam", "egg"]);
1837    /// let s1 = Column::new("ascii".into(), [70, 79, 79]);
1838    /// let mut df = DataFrame::new_infer_height(vec![s0, s1])?;
1839    ///
1840    /// // Add 32 to get lowercase ascii values
1841    /// df.apply_at_idx(1, |s| s + 32);
1842    /// # Ok::<(), PolarsError>(())
1843    /// ```
1844    /// Results in:
1845    ///
1846    /// ```text
1847    /// +--------+-------+
1848    /// | foo    | ascii |
1849    /// | ---    | ---   |
1850    /// | str    | i32   |
1851    /// +========+=======+
1852    /// | "ham"  | 102   |
1853    /// +--------+-------+
1854    /// | "spam" | 111   |
1855    /// +--------+-------+
1856    /// | "egg"  | 111   |
1857    /// +--------+-------+
1858    /// ```
1859    pub fn apply_at_idx<F, C>(&mut self, idx: usize, f: F) -> PolarsResult<&mut Self>
1860    where
1861        F: FnOnce(&Column) -> C,
1862        C: IntoColumn,
1863    {
1864        let df_height = self.height();
1865        let width = self.width();
1866
1867        let cached_schema = self.cached_schema().cloned();
1868
1869        let col = unsafe { self.columns_mut() }.get_mut(idx).ok_or_else(|| {
1870            polars_err!(
1871                ComputeError: "invalid column index: {} for a DataFrame with {} columns",
1872                idx, width
1873            )
1874        })?;
1875
1876        let new_col = f(col)
1877            .into_column()
1878            .with_name(col.name().clone())
1879            .broadcast_owned_to(df_height)?;
1880        let col_before = std::mem::replace(col, new_col);
1881
1882        if col.dtype() == col_before.dtype() {
1883            unsafe { self.set_opt_schema(cached_schema) };
1884        }
1885
1886        Ok(self)
1887    }
1888
1889    /// Apply a closure that may fail to a column at index `idx`. This is the recommended way to do in place
1890    /// modification.
1891    ///
1892    /// # Example
1893    ///
1894    /// This is the idiomatic way to replace some values a column of a `DataFrame` given range of indexes.
1895    ///
1896    /// ```rust
1897    /// # use polars_core::prelude::*;
1898    /// let s0 = Column::new("foo".into(), ["ham", "spam", "egg", "bacon", "quack"]);
1899    /// let s1 = Column::new("values".into(), [1, 2, 3, 4, 5]);
1900    /// let mut df = DataFrame::new_infer_height(vec![s0, s1])?;
1901    ///
1902    /// let idx = vec![0, 1, 4];
1903    ///
1904    /// df.try_apply("foo", |c| {
1905    ///     c.str()?
1906    ///     .scatter_with(idx, |opt_val| opt_val.map(|string| format!("{}-is-modified", string)))
1907    /// });
1908    /// # Ok::<(), PolarsError>(())
1909    /// ```
1910    /// Results in:
1911    ///
1912    /// ```text
1913    /// +---------------------+--------+
1914    /// | foo                 | values |
1915    /// | ---                 | ---    |
1916    /// | str                 | i32    |
1917    /// +=====================+========+
1918    /// | "ham-is-modified"   | 1      |
1919    /// +---------------------+--------+
1920    /// | "spam-is-modified"  | 2      |
1921    /// +---------------------+--------+
1922    /// | "egg"               | 3      |
1923    /// +---------------------+--------+
1924    /// | "bacon"             | 4      |
1925    /// +---------------------+--------+
1926    /// | "quack-is-modified" | 5      |
1927    /// +---------------------+--------+
1928    /// ```
1929    pub fn try_apply_at_idx<F, C>(&mut self, idx: usize, f: F) -> PolarsResult<&mut Self>
1930    where
1931        F: FnOnce(&Column) -> PolarsResult<C>,
1932        C: IntoColumn,
1933    {
1934        let df_height = self.height();
1935        let width = self.width();
1936
1937        let cached_schema = self.cached_schema().cloned();
1938
1939        let col = unsafe { self.columns_mut() }.get_mut(idx).ok_or_else(|| {
1940            polars_err!(
1941                ComputeError: "invalid column index: {} for a DataFrame with {} columns",
1942                idx, width
1943            )
1944        })?;
1945
1946        let mut new_col = f(col).map(|c| c.into_column())?;
1947
1948        polars_ensure!(
1949            new_col.len() == df_height,
1950            ShapeMismatch:
1951            "try_apply_at_idx: resulting Series has length {} while the DataFrame has height {}",
1952            new_col.len(), df_height
1953        );
1954
1955        // make sure the name remains the same after applying the closure
1956        new_col = new_col.with_name(col.name().clone());
1957        let col_before = std::mem::replace(col, new_col);
1958
1959        if col.dtype() == col_before.dtype() {
1960            unsafe { self.set_opt_schema(cached_schema) };
1961        }
1962
1963        Ok(self)
1964    }
1965
1966    /// Apply a closure that may fail to a column. This is the recommended way to do in place
1967    /// modification.
1968    ///
1969    /// # Example
1970    ///
1971    /// This is the idiomatic way to replace some values a column of a `DataFrame` given a boolean mask.
1972    ///
1973    /// ```rust
1974    /// # use polars_core::prelude::*;
1975    /// let s0 = Column::new("foo".into(), ["ham", "spam", "egg", "bacon", "quack"]);
1976    /// let s1 = Column::new("values".into(), [1, 2, 3, 4, 5]);
1977    /// let mut df = DataFrame::new_infer_height(vec![s0, s1])?;
1978    ///
1979    /// // create a mask
1980    /// let values = df.column("values")?.as_materialized_series();
1981    /// let mask = values.lt_eq(1)? | values.gt_eq(5_i32)?;
1982    ///
1983    /// df.try_apply("foo", |c| {
1984    ///     c.str()?
1985    ///     .set(&mask, Some("not_within_bounds"))
1986    /// });
1987    /// # Ok::<(), PolarsError>(())
1988    /// ```
1989    /// Results in:
1990    ///
1991    /// ```text
1992    /// +---------------------+--------+
1993    /// | foo                 | values |
1994    /// | ---                 | ---    |
1995    /// | str                 | i32    |
1996    /// +=====================+========+
1997    /// | "not_within_bounds" | 1      |
1998    /// +---------------------+--------+
1999    /// | "spam"              | 2      |
2000    /// +---------------------+--------+
2001    /// | "egg"               | 3      |
2002    /// +---------------------+--------+
2003    /// | "bacon"             | 4      |
2004    /// +---------------------+--------+
2005    /// | "not_within_bounds" | 5      |
2006    /// +---------------------+--------+
2007    /// ```
2008    pub fn try_apply<F, C>(&mut self, column: &str, f: F) -> PolarsResult<&mut Self>
2009    where
2010        F: FnOnce(&Series) -> PolarsResult<C>,
2011        C: IntoColumn,
2012    {
2013        let idx = self.try_get_column_index(column)?;
2014        self.try_apply_at_idx(idx, |c| f(c.as_materialized_series()))
2015    }
2016
2017    /// Slice the [`DataFrame`] along the rows.
2018    ///
2019    /// # Example
2020    ///
2021    /// ```rust
2022    /// # use polars_core::prelude::*;
2023    /// let df: DataFrame = df!("Fruit" => ["Apple", "Grape", "Grape", "Fig", "Fig"],
2024    ///                         "Color" => ["Green", "Red", "White", "White", "Red"])?;
2025    /// let sl: DataFrame = df.slice((2i64, 3));
2026    ///
2027    /// assert_eq!(sl.shape(), (3, 2));
2028    /// println!("{}", sl);
2029    /// # Ok::<(), PolarsError>(())
2030    /// ```
2031    /// Output:
2032    /// ```text
2033    /// shape: (3, 2)
2034    /// +-------+-------+
2035    /// | Fruit | Color |
2036    /// | ---   | ---   |
2037    /// | str   | str   |
2038    /// +=======+=======+
2039    /// | Grape | White |
2040    /// +-------+-------+
2041    /// | Fig   | White |
2042    /// +-------+-------+
2043    /// | Fig   | Red   |
2044    /// +-------+-------+
2045    /// ```
2046    #[must_use]
2047    pub fn slice(&self, slice: impl Into<Slice>) -> Self {
2048        let range = slice.into().resolve(self.height());
2049
2050        if range.len() == self.height() {
2051            return self.clone();
2052        }
2053
2054        if range.is_empty() {
2055            return self.clear();
2056        }
2057
2058        let cols = self.apply_columns(|s| s.slice(range.clone()));
2059
2060        unsafe { DataFrame::_new_unchecked_impl(range.len(), cols).with_schema_from(self) }
2061    }
2062
2063    /// Split [`DataFrame`] at the given `offset`.
2064    pub fn split_at(&self, offset: i64) -> (Self, Self) {
2065        let (a, b) = self.columns().iter().map(|s| s.split_at(offset)).unzip();
2066
2067        let idx = Slice::resolve_offset(offset, self.height());
2068
2069        let a = unsafe { DataFrame::new_unchecked(idx, a).with_schema_from(self) };
2070        let b = unsafe { DataFrame::new_unchecked(self.height() - idx, b).with_schema_from(self) };
2071        (a, b)
2072    }
2073
2074    #[must_use]
2075    pub fn clear(&self) -> Self {
2076        let cols = self.columns().iter().map(|s| s.clear()).collect::<Vec<_>>();
2077        unsafe { DataFrame::_new_unchecked_impl(0, cols).with_schema_from(self) }
2078    }
2079
2080    /// Get the head of the [`DataFrame`].
2081    ///
2082    /// # Example
2083    ///
2084    /// ```rust
2085    /// # use polars_core::prelude::*;
2086    /// let countries: DataFrame =
2087    ///     df!("Rank by GDP (2021)" => [1, 2, 3, 4, 5],
2088    ///         "Continent" => ["North America", "Asia", "Asia", "Europe", "Europe"],
2089    ///         "Country" => ["United States", "China", "Japan", "Germany", "United Kingdom"],
2090    ///         "Capital" => ["Washington", "Beijing", "Tokyo", "Berlin", "London"])?;
2091    /// assert_eq!(countries.shape(), (5, 4));
2092    ///
2093    /// println!("{}", countries.head(Some(3)));
2094    /// # Ok::<(), PolarsError>(())
2095    /// ```
2096    ///
2097    /// Output:
2098    ///
2099    /// ```text
2100    /// shape: (3, 4)
2101    /// +--------------------+---------------+---------------+------------+
2102    /// | Rank by GDP (2021) | Continent     | Country       | Capital    |
2103    /// | ---                | ---           | ---           | ---        |
2104    /// | i32                | str           | str           | str        |
2105    /// +====================+===============+===============+============+
2106    /// | 1                  | North America | United States | Washington |
2107    /// +--------------------+---------------+---------------+------------+
2108    /// | 2                  | Asia          | China         | Beijing    |
2109    /// +--------------------+---------------+---------------+------------+
2110    /// | 3                  | Asia          | Japan         | Tokyo      |
2111    /// +--------------------+---------------+---------------+------------+
2112    /// ```
2113    #[must_use]
2114    pub fn head(&self, length: Option<usize>) -> Self {
2115        let new_height = usize::min(self.height(), length.unwrap_or(HEAD_DEFAULT_LENGTH));
2116        let new_cols = self.apply_columns(|c| c.head(Some(new_height)));
2117
2118        unsafe { DataFrame::new_unchecked(new_height, new_cols).with_schema_from(self) }
2119    }
2120
2121    /// Get the tail of the [`DataFrame`].
2122    ///
2123    /// # Example
2124    ///
2125    /// ```rust
2126    /// # use polars_core::prelude::*;
2127    /// let countries: DataFrame =
2128    ///     df!("Rank (2021)" => [105, 106, 107, 108, 109],
2129    ///         "Apple Price (€/kg)" => [0.75, 0.70, 0.70, 0.65, 0.52],
2130    ///         "Country" => ["Kosovo", "Moldova", "North Macedonia", "Syria", "Turkey"])?;
2131    /// assert_eq!(countries.shape(), (5, 3));
2132    ///
2133    /// println!("{}", countries.tail(Some(2)));
2134    /// # Ok::<(), PolarsError>(())
2135    /// ```
2136    ///
2137    /// Output:
2138    ///
2139    /// ```text
2140    /// shape: (2, 3)
2141    /// +-------------+--------------------+---------+
2142    /// | Rank (2021) | Apple Price (€/kg) | Country |
2143    /// | ---         | ---                | ---     |
2144    /// | i32         | f64                | str     |
2145    /// +=============+====================+=========+
2146    /// | 108         | 0.65               | Syria   |
2147    /// +-------------+--------------------+---------+
2148    /// | 109         | 0.52               | Turkey  |
2149    /// +-------------+--------------------+---------+
2150    /// ```
2151    #[must_use]
2152    pub fn tail(&self, length: Option<usize>) -> Self {
2153        let new_height = usize::min(self.height(), length.unwrap_or(TAIL_DEFAULT_LENGTH));
2154        let new_cols = self.apply_columns(|c| c.tail(Some(new_height)));
2155
2156        unsafe { DataFrame::new_unchecked(new_height, new_cols).with_schema_from(self) }
2157    }
2158
2159    /// Iterator over the rows in this [`DataFrame`] as Arrow RecordBatches.
2160    ///
2161    /// A chunk is split into several batches where its arrow MAP arrays would overflow
2162    /// their `i32` offsets.
2163    ///
2164    /// The columns' chunks must be aligned ([`DataFrame::should_rechunk`] is false); only debug
2165    /// builds check this. Aligning is left to the caller, which can reuse the aligned frame.
2166    ///
2167    /// # Panics
2168    ///
2169    /// Panics where [`DataFrame::try_iter_chunks`] yields an error.
2170    pub fn iter_chunks(
2171        &self,
2172        compat_level: CompatLevel,
2173        parallel: bool,
2174    ) -> impl Iterator<Item = RecordBatch> + '_ {
2175        self.try_iter_chunks(compat_level, parallel)
2176            .map(|batch| batch.unwrap())
2177    }
2178
2179    /// Like [`DataFrame::iter_chunks`], but yields export errors, such as a row with more map
2180    /// entries than the `i32` offsets of an arrow MAP can hold.
2181    ///
2182    /// The columns' chunks must be aligned, as for [`DataFrame::iter_chunks`].
2183    pub fn try_iter_chunks(
2184        &self,
2185        compat_level: CompatLevel,
2186        parallel: bool,
2187    ) -> impl Iterator<Item = PolarsResult<RecordBatch>> + Send + '_ {
2188        // If any of the columns is binview and we don't convert `compat_level` we allow parallelism
2189        // as we must allocate arrow strings/binaries.
2190        let parallel = parallel
2191            && compat_level.0 == 0
2192            && self
2193                .columns()
2194                .iter()
2195                .any(|s| matches!(s.dtype(), DataType::String | DataType::Binary));
2196        RecordBatchIterWrap::new(Cow::Borrowed(self), compat_level, parallel)
2197    }
2198
2199    /// Like [`DataFrame::try_iter_chunks`], but the iterator owns the frame, for consumers that
2200    /// outlive it, such as an arrow C stream.
2201    ///
2202    /// With `parallel`, the columns of each batch are exported on the thread pool, whatever their
2203    /// dtypes.
2204    pub fn into_try_iter_chunks(
2205        self,
2206        compat_level: CompatLevel,
2207        parallel: bool,
2208    ) -> impl Iterator<Item = PolarsResult<RecordBatch>> + Send + 'static {
2209        RecordBatchIterWrap::new(Cow::Owned(self), compat_level, parallel)
2210    }
2211
2212    /// Iterator over the rows in this [`DataFrame`] as Arrow RecordBatches as physical values.
2213    ///
2214    /// # Panics
2215    ///
2216    /// Panics if the [`DataFrame`] that is passed is not rechunked.
2217    ///
2218    /// This responsibility is left to the caller as we don't want to take mutable references here,
2219    /// but we also don't want to rechunk here, as this operation is costly and would benefit the caller
2220    /// as well.
2221    pub fn iter_chunks_physical(&self) -> impl Iterator<Item = RecordBatch> + '_ {
2222        debug_assert!(!self.should_rechunk());
2223
2224        let iter = if self.width() == 0 {
2225            RecordBatchIterWrap::new_zero_width(self.height())
2226        } else {
2227            RecordBatchIterWrap::PhysicalBatches(PhysRecordBatchIter {
2228                df: self,
2229                schema: Arc::new(
2230                    self.columns()
2231                        .iter()
2232                        .map(|c| c.field().to_arrow(CompatLevel::newest()))
2233                        .collect(),
2234                ),
2235                cursor: ChunkCursor::new(self),
2236                offset: 0,
2237            })
2238        };
2239        iter.map(|batch| batch.unwrap())
2240    }
2241
2242    /// Get a [`DataFrame`] with all the columns in reversed order.
2243    #[must_use]
2244    pub fn reverse(&self) -> Self {
2245        let new_cols = self.apply_columns(Column::reverse);
2246        unsafe { DataFrame::new_unchecked(self.height(), new_cols).with_schema_from(self) }
2247    }
2248
2249    /// Shift the values by a given period and fill the parts that will be empty due to this operation
2250    /// with `Nones`.
2251    ///
2252    /// See the method on [Series](crate::series::SeriesTrait::shift) for more info on the `shift` operation.
2253    #[must_use]
2254    pub fn shift(&self, periods: i64) -> Self {
2255        let col = self.apply_columns_par(|s| s.shift(periods));
2256        unsafe { DataFrame::new_unchecked(self.height(), col).with_schema_from(self) }
2257    }
2258
2259    /// Replace None values with one of the following strategies:
2260    /// * Forward fill (replace None with the previous value)
2261    /// * Backward fill (replace None with the next value)
2262    /// * Mean fill (replace None with the mean of the whole array)
2263    /// * Min fill (replace None with the minimum of the whole array)
2264    /// * Max fill (replace None with the maximum of the whole array)
2265    ///
2266    /// See the method on [Series](crate::series::Series::fill_null) for more info on the `fill_null` operation.
2267    pub fn fill_null(&self, strategy: FillNullStrategy) -> PolarsResult<Self> {
2268        let col = self.try_apply_columns_par(|s| s.fill_null(strategy))?;
2269
2270        Ok(unsafe { DataFrame::new_unchecked(self.height(), col) })
2271    }
2272
2273    /// Drop duplicate rows from a [`DataFrame`].
2274    /// *This fails when there is a column of type List in DataFrame*
2275    ///
2276    /// Stable means that the order is maintained. This has a higher cost than an unstable distinct.
2277    ///
2278    /// # Example
2279    ///
2280    /// ```no_run
2281    /// # use polars_core::prelude::*;
2282    /// let df = df! {
2283    ///               "flt" => [1., 1., 2., 2., 3., 3.],
2284    ///               "int" => [1, 1, 2, 2, 3, 3, ],
2285    ///               "str" => ["a", "a", "b", "b", "c", "c"]
2286    ///           }?;
2287    ///
2288    /// println!("{}", df.unique_stable(None, UniqueKeepStrategy::First, None)?);
2289    /// # Ok::<(), PolarsError>(())
2290    /// ```
2291    /// Returns
2292    ///
2293    /// ```text
2294    /// +-----+-----+-----+
2295    /// | flt | int | str |
2296    /// | --- | --- | --- |
2297    /// | f64 | i32 | str |
2298    /// +=====+=====+=====+
2299    /// | 1   | 1   | "a" |
2300    /// +-----+-----+-----+
2301    /// | 2   | 2   | "b" |
2302    /// +-----+-----+-----+
2303    /// | 3   | 3   | "c" |
2304    /// +-----+-----+-----+
2305    /// ```
2306    #[cfg(feature = "algorithm_group_by")]
2307    pub fn unique_stable(
2308        &self,
2309        subset: Option<&[String]>,
2310        keep: UniqueKeepStrategy,
2311        slice: Option<Slice>,
2312    ) -> PolarsResult<DataFrame> {
2313        self.unique_impl(
2314            true,
2315            subset.map(|v| v.iter().map(|x| PlSmallStr::from_str(x.as_str())).collect()),
2316            keep,
2317            slice,
2318        )
2319    }
2320
2321    /// Unstable distinct. See [`DataFrame::unique_stable`].
2322    #[cfg(feature = "algorithm_group_by")]
2323    pub fn unique<I, S>(
2324        &self,
2325        subset: Option<&[String]>,
2326        keep: UniqueKeepStrategy,
2327        slice: Option<Slice>,
2328    ) -> PolarsResult<DataFrame> {
2329        self.unique_impl(
2330            false,
2331            subset.map(|v| v.iter().map(|x| PlSmallStr::from_str(x.as_str())).collect()),
2332            keep,
2333            slice,
2334        )
2335    }
2336
2337    #[cfg(feature = "algorithm_group_by")]
2338    pub fn unique_impl(
2339        &self,
2340        maintain_order: bool,
2341        subset: Option<Vec<PlSmallStr>>,
2342        keep: UniqueKeepStrategy,
2343        slice: Option<Slice>,
2344    ) -> PolarsResult<Self> {
2345        if self.width() == 0 {
2346            let height = usize::min(self.height(), 1);
2347            return Ok(DataFrame::empty_with_height(height));
2348        }
2349
2350        let names = subset.unwrap_or_else(|| self.get_column_names_owned());
2351        let mut df = self.clone();
2352        // take on multiple chunks is terrible
2353        df.rechunk_mut_par();
2354
2355        fn optionally_slice(
2356            groups: &'_ GroupPositions,
2357            slice: Option<Slice>,
2358        ) -> Cow<'_, GroupPositions> {
2359            match slice {
2360                Some(s) => Cow::Owned(groups.slice(s)),
2361                None => Cow::Borrowed(groups),
2362            }
2363        }
2364
2365        let columns = match (keep, maintain_order) {
2366            (UniqueKeepStrategy::First | UniqueKeepStrategy::Any, true) => {
2367                let gb = df.group_by_stable(names)?;
2368                let groups = gb.get_groups();
2369                let groups = optionally_slice(groups, slice);
2370                df.apply_columns_par(|s| unsafe { s.agg_first(&groups) })
2371            },
2372            (UniqueKeepStrategy::Last, true) => {
2373                // maintain order by last values, so the sorted groups are not correct as they
2374                // are sorted by the first value
2375                let gb = df.group_by_stable(names)?;
2376                let groups = gb.get_groups();
2377
2378                let last_idx: NoNull<IdxCa> = groups
2379                    .iter()
2380                    .map(|g| match g {
2381                        GroupsIndicator::Idx((_first, idx)) => idx[idx.len() - 1],
2382                        GroupsIndicator::Slice([first, len]) => first + len - 1,
2383                    })
2384                    .collect();
2385
2386                let mut last_idx = last_idx.into_inner().sort(false);
2387
2388                if let Some(slice) = slice {
2389                    last_idx = last_idx.slice(slice);
2390                }
2391
2392                let last_idx = NoNull::new(last_idx);
2393                let out = unsafe { df.take_unchecked(&last_idx) };
2394                return Ok(out);
2395            },
2396            (UniqueKeepStrategy::First | UniqueKeepStrategy::Any, false) => {
2397                let gb = df.group_by(names)?;
2398                let groups = gb.get_groups();
2399                let groups = optionally_slice(groups, slice);
2400                df.apply_columns_par(|s| unsafe { s.agg_first(&groups) })
2401            },
2402            (UniqueKeepStrategy::Last, false) => {
2403                let gb = df.group_by(names)?;
2404                let groups = gb.get_groups();
2405                let groups = optionally_slice(groups, slice);
2406                df.apply_columns_par(|s| unsafe { s.agg_last(&groups) })
2407            },
2408            (UniqueKeepStrategy::None, _) => {
2409                let df_part = df.select(names)?;
2410                let mask = df_part.is_unique()?;
2411                let mut filtered = df.filter(&mask)?;
2412
2413                if let Some(slice) = slice {
2414                    filtered = filtered.slice(slice);
2415                }
2416                return Ok(filtered);
2417            },
2418        };
2419        Ok(unsafe { DataFrame::new_unchecked_infer_height(columns).with_schema_from(self) })
2420    }
2421
2422    /// Get a mask of all the unique rows in the [`DataFrame`].
2423    ///
2424    /// # Example
2425    ///
2426    /// ```no_run
2427    /// # use polars_core::prelude::*;
2428    /// let df: DataFrame = df!("Company" => ["Apple", "Microsoft"],
2429    ///                         "ISIN" => ["US0378331005", "US5949181045"])?;
2430    /// let ca: ChunkedArray<BooleanType> = df.is_unique()?;
2431    ///
2432    /// assert!(ca.all());
2433    /// # Ok::<(), PolarsError>(())
2434    /// ```
2435    #[cfg(feature = "algorithm_group_by")]
2436    pub fn is_unique(&self) -> PolarsResult<BooleanChunked> {
2437        let gb = self.group_by(self.get_column_names_owned())?;
2438        let groups = gb.get_groups();
2439        Ok(is_unique_helper(
2440            groups,
2441            self.height() as IdxSize,
2442            true,
2443            false,
2444        ))
2445    }
2446
2447    /// Get a mask of all the duplicated rows in the [`DataFrame`].
2448    ///
2449    /// # Example
2450    ///
2451    /// ```no_run
2452    /// # use polars_core::prelude::*;
2453    /// let df: DataFrame = df!("Company" => ["Alphabet", "Alphabet"],
2454    ///                         "ISIN" => ["US02079K3059", "US02079K1079"])?;
2455    /// let ca: ChunkedArray<BooleanType> = df.is_duplicated()?;
2456    ///
2457    /// assert!(!ca.all());
2458    /// # Ok::<(), PolarsError>(())
2459    /// ```
2460    #[cfg(feature = "algorithm_group_by")]
2461    pub fn is_duplicated(&self) -> PolarsResult<BooleanChunked> {
2462        let gb = self.group_by(self.get_column_names_owned())?;
2463        let groups = gb.get_groups();
2464        Ok(is_unique_helper(
2465            groups,
2466            self.height() as IdxSize,
2467            false,
2468            true,
2469        ))
2470    }
2471
2472    /// Create a new [`DataFrame`] that shows the null counts per column.
2473    #[must_use]
2474    pub fn null_count(&self) -> Self {
2475        let cols =
2476            self.apply_columns(|c| Column::new(c.name().clone(), [c.null_count() as IdxSize]));
2477        unsafe { Self::new_unchecked(1, cols) }
2478    }
2479
2480    /// Hash and combine the row values
2481    #[cfg(feature = "row_hash")]
2482    pub fn hash_rows(
2483        &mut self,
2484        hasher_builder: Option<PlSeedableRandomStateQuality>,
2485    ) -> PolarsResult<UInt64Chunked> {
2486        let dfs = split_df(self, RAYON.current_num_threads(), false);
2487        let (cas, _) = _df_rows_to_hashes_threaded_vertical(&dfs, hasher_builder)?;
2488
2489        let mut iter = cas.into_iter();
2490        let mut acc_ca = iter.next().unwrap();
2491        for ca in iter {
2492            acc_ca.append(&ca)?;
2493        }
2494        Ok(acc_ca.rechunk().into_owned())
2495    }
2496
2497    /// Get the supertype of the columns in this DataFrame
2498    pub fn get_supertype(&self) -> Option<PolarsResult<DataType>> {
2499        self.columns()
2500            .iter()
2501            .map(|s| Ok(s.dtype().clone()))
2502            .reduce(|acc, b| try_get_supertype(&acc?, &b.unwrap()))
2503    }
2504
2505    /// Take by index values given by the slice `idx`.
2506    /// # Warning
2507    /// Be careful with allowing threads when calling this in a large hot loop
2508    /// every thread split may be on rayon stack and lead to SO
2509    #[doc(hidden)]
2510    pub unsafe fn _take_unchecked_slice(&self, idx: &[IdxSize], allow_threads: bool) -> Self {
2511        self._take_unchecked_slice_sorted(idx, allow_threads, IsSorted::Not)
2512    }
2513
2514    /// Take by index values given by the slice `idx`. Use this over `_take_unchecked_slice`
2515    /// if the index value in `idx` are sorted. This will maintain sorted flags.
2516    ///
2517    /// # Warning
2518    /// Be careful with allowing threads when calling this in a large hot loop
2519    /// every thread split may be on rayon stack and lead to SO
2520    #[doc(hidden)]
2521    pub unsafe fn _take_unchecked_slice_sorted(
2522        &self,
2523        idx: &[IdxSize],
2524        allow_threads: bool,
2525        sorted: IsSorted,
2526    ) -> Self {
2527        #[cfg(debug_assertions)]
2528        {
2529            if idx.len() > 2 {
2530                use crate::series::IsSorted;
2531
2532                match sorted {
2533                    IsSorted::Ascending => {
2534                        assert!(idx[0] <= idx[idx.len() - 1]);
2535                    },
2536                    IsSorted::Descending => {
2537                        assert!(idx[0] >= idx[idx.len() - 1]);
2538                    },
2539                    _ => {},
2540                }
2541            }
2542        }
2543        let mut ca = IdxCa::mmap_slice(PlSmallStr::EMPTY, idx);
2544        ca.set_sorted_flag(sorted);
2545        self.take_unchecked_impl(&ca, allow_threads)
2546    }
2547    #[cfg(all(feature = "partition_by", feature = "algorithm_group_by"))]
2548    #[doc(hidden)]
2549    pub fn _partition_by_impl(
2550        &self,
2551        cols: &[PlSmallStr],
2552        stable: bool,
2553        include_key: bool,
2554        parallel: bool,
2555    ) -> PolarsResult<Vec<DataFrame>> {
2556        let selected_keys = self.select_to_vec(cols.iter().cloned())?;
2557        let groups = self.group_by_with_series(selected_keys, parallel, stable)?;
2558        let groups = groups.into_groups();
2559
2560        // drop key columns prior to calculation if requested
2561        let df = if include_key {
2562            self.clone()
2563        } else {
2564            self.drop_many(cols.iter().cloned())
2565        };
2566
2567        if parallel {
2568            // don't parallelize this
2569            // there is a lot of parallelization in take and this may easily SO
2570            RAYON.install(|| {
2571                match groups.as_ref() {
2572                    GroupsType::Idx(idx) => {
2573                        // Rechunk as the gather may rechunk for every group #17562.
2574                        let mut df = df.clone();
2575                        df.rechunk_mut_par();
2576                        Ok(idx
2577                            .into_par_iter()
2578                            .map(|(_, group)| {
2579                                // groups are in bounds
2580                                unsafe {
2581                                    df._take_unchecked_slice_sorted(
2582                                        group,
2583                                        false,
2584                                        IsSorted::Ascending,
2585                                    )
2586                                }
2587                            })
2588                            .collect())
2589                    },
2590                    GroupsType::Slice { groups, .. } => Ok(groups
2591                        .into_par_iter()
2592                        .map(|[first, len]| df.slice((*first as usize, *len as usize)))
2593                        .collect()),
2594                }
2595            })
2596        } else {
2597            match groups.as_ref() {
2598                GroupsType::Idx(idx) => {
2599                    // Rechunk as the gather may rechunk for every group #17562.
2600                    let mut df = df;
2601                    df.rechunk_mut();
2602                    Ok(idx
2603                        .into_iter()
2604                        .map(|(_, group)| {
2605                            // groups are in bounds
2606                            unsafe {
2607                                df._take_unchecked_slice_sorted(group, false, IsSorted::Ascending)
2608                            }
2609                        })
2610                        .collect())
2611                },
2612                GroupsType::Slice { groups, .. } => Ok(groups
2613                    .iter()
2614                    .map(|[first, len]| df.slice((*first as usize, *len as usize)))
2615                    .collect()),
2616            }
2617        }
2618    }
2619
2620    /// Split into multiple DataFrames partitioned by groups
2621    #[cfg(feature = "partition_by")]
2622    pub fn partition_by<I, S>(&self, cols: I, include_key: bool) -> PolarsResult<Vec<DataFrame>>
2623    where
2624        I: IntoIterator<Item = S>,
2625        S: Into<PlSmallStr>,
2626    {
2627        let cols: UnitVec<PlSmallStr> = cols.into_iter().map(Into::into).collect();
2628        self._partition_by_impl(cols.as_slice(), false, include_key, true)
2629    }
2630
2631    /// Split into multiple DataFrames partitioned by groups
2632    /// Order of the groups are maintained.
2633    #[cfg(feature = "partition_by")]
2634    pub fn partition_by_stable<I, S>(
2635        &self,
2636        cols: I,
2637        include_key: bool,
2638    ) -> PolarsResult<Vec<DataFrame>>
2639    where
2640        I: IntoIterator<Item = S>,
2641        S: Into<PlSmallStr>,
2642    {
2643        let cols: UnitVec<PlSmallStr> = cols.into_iter().map(Into::into).collect();
2644        self._partition_by_impl(cols.as_slice(), true, include_key, true)
2645    }
2646
2647    /// Unnest the given `Struct` columns. This means that the fields of the `Struct` type will be
2648    /// inserted as columns.
2649    #[cfg(feature = "dtype-struct")]
2650    pub fn unnest(
2651        &self,
2652        cols: impl IntoIterator<Item = impl Into<PlSmallStr>>,
2653        separator: Option<&str>,
2654    ) -> PolarsResult<DataFrame> {
2655        self.unnest_impl(cols.into_iter().map(Into::into).collect(), separator)
2656    }
2657
2658    #[cfg(feature = "dtype-struct")]
2659    fn unnest_impl(
2660        &self,
2661        cols: PlHashSet<PlSmallStr>,
2662        separator: Option<&str>,
2663    ) -> PolarsResult<DataFrame> {
2664        let mut new_cols = Vec::with_capacity(std::cmp::min(self.width() * 2, self.width() + 128));
2665        let mut count = 0;
2666        for s in self.columns() {
2667            if cols.contains(s.name()) {
2668                let ca = s.struct_()?.clone();
2669                new_cols.extend(ca.fields_as_series().into_iter().map(|mut f| {
2670                    if let Some(separator) = &separator {
2671                        f.rename(polars_utils::format_pl_smallstr!(
2672                            "{}{}{}",
2673                            s.name(),
2674                            separator,
2675                            f.name()
2676                        ));
2677                    }
2678                    Column::from(f)
2679                }));
2680                count += 1;
2681            } else {
2682                new_cols.push(s.clone())
2683            }
2684        }
2685        if count != cols.len() {
2686            // one or more columns not found
2687            // the code below will return an error with the missing name
2688            let schema = self.schema();
2689            for col in cols {
2690                let _ = schema
2691                    .get(col.as_str())
2692                    .ok_or_else(|| polars_err!(col_not_found = col))?;
2693            }
2694        }
2695
2696        DataFrame::new(self.height(), new_cols)
2697    }
2698
2699    pub fn append_record_batch(&mut self, rb: RecordBatchT<ArrayRef>) -> PolarsResult<()> {
2700        // @Optimize: this does a lot of unnecessary allocations. We should probably have a
2701        // append_chunk or something like this. It is just quite difficult to make that safe.
2702        let df = DataFrame::from(rb);
2703        polars_ensure!(
2704            self.schema() == df.schema(),
2705            SchemaMismatch: "cannot append record batch with different schema\n\n
2706        Got {:?}\nexpected: {:?}", df.schema(), self.schema(),
2707        );
2708        self.vstack_mut_owned_unchecked(df);
2709        Ok(())
2710    }
2711}
2712
2713/// Cursor over the chunks a [`DataFrame`] is laid out along.
2714struct ChunkCursor {
2715    /// The chunk lengths of the first column that has chunks (a series, or a dictionary
2716    /// column's codes), as `first_col_chunk_lengths`; empty when no column has.
2717    chunks: Vec<usize>,
2718    height: usize,
2719    idx: usize,
2720}
2721
2722impl ChunkCursor {
2723    fn new(df: &DataFrame) -> Self {
2724        Self {
2725            chunks: df.first_col_chunk_lengths().collect(),
2726            height: df.height(),
2727            idx: 0,
2728        }
2729    }
2730
2731    fn n_chunks(&self) -> usize {
2732        usize::max(1, self.chunks.len())
2733    }
2734
2735    /// The index and number of rows of the next chunk.
2736    fn next(&mut self) -> Option<(usize, usize)> {
2737        let idx = self.idx;
2738        if idx >= self.n_chunks() {
2739            return None;
2740        }
2741
2742        self.idx += 1;
2743        Some((idx, self.chunks.get(idx).copied().unwrap_or(self.height)))
2744    }
2745
2746    fn size_hint(&self) -> (usize, Option<usize>) {
2747        let n = self.n_chunks() - self.idx;
2748        (n, Some(n))
2749    }
2750}
2751
2752pub struct RecordBatchIter<'a> {
2753    df: Cow<'a, DataFrame>,
2754    schema: ArrowSchemaRef,
2755    cursor: ChunkCursor,
2756    compat_level: CompatLevel,
2757    parallel: bool,
2758    /// The columns that export to an arrow MAP, whose `i32` offsets may make a chunk need several
2759    /// batches; sorted by index.
2760    map_columns: Vec<MapColumn>,
2761    split: Option<SplitChunk>,
2762    /// The first row of the next chunk, to slice columns without chunks of their own.
2763    offset: usize,
2764}
2765
2766/// A column that exports to an arrow MAP.
2767struct MapColumn {
2768    idx: usize,
2769    /// The one row a Scalar column repeats, so it is planned without materializing the column;
2770    /// `None` for a Series column.
2771    row: Option<ArrayRef>,
2772}
2773
2774/// A chunk exported as one batch per piece, cut as the iterator reaches it.
2775struct SplitChunk {
2776    chunk: DataFrame,
2777    /// The columns without an arrow MAP, converted once for the whole chunk so that their pieces
2778    /// share one dictionary; `None` for the columns converted per piece.
2779    converted: Vec<Option<ArrayRef>>,
2780    /// The first row of the next piece.
2781    start: usize,
2782}
2783
2784/// `rows` of `c`, a column without chunks of its own, as a series of one chunk.
2785fn materialize_rows(c: &Column, rows: std::ops::Range<usize>) -> Series {
2786    // A column exported whole reuses the series it caches.
2787    let s = if rows.len() == c.len() {
2788        c.as_materialized_series().clone()
2789    } else {
2790        c.slice(rows).take_materialized_series()
2791    };
2792    if s.n_chunks() > 1 { s.rechunk() } else { s }
2793}
2794
2795/// Export `rows` of `c`, which its chunk `idx` holds if it has chunks of its own.
2796fn column_to_arrow(
2797    c: &Column,
2798    idx: usize,
2799    rows: std::ops::Range<usize>,
2800    compat_level: CompatLevel,
2801) -> PolarsResult<ArrayRef> {
2802    match c.as_series() {
2803        Some(s) => s.try_to_arrow(idx, compat_level),
2804        // A dictionary column's codes need not have the chunks of the other columns.
2805        None => materialize_rows(c, rows).try_to_arrow(0, compat_level),
2806    }
2807}
2808
2809impl<'a> RecordBatchIter<'a> {
2810    /// See [`DataFrame::try_iter_chunks`], for a `df` with at least one column.
2811    pub(crate) fn new(df: Cow<'a, DataFrame>, compat_level: CompatLevel, parallel: bool) -> Self {
2812        let parallel = parallel && df.width() > 1;
2813
2814        let schema: ArrowSchemaRef = Arc::new(
2815            df.columns()
2816                .iter()
2817                .map(|c| c.field().to_arrow(compat_level))
2818                .collect(),
2819        );
2820        let map_columns = df
2821            .columns()
2822            .iter()
2823            .zip(schema.iter_values())
2824            .enumerate()
2825            .filter(|(_, (_, f))| contains_arrow_map(f.dtype()))
2826            .filter_map(|(idx, (c, _))| {
2827                let Some(sc) = c.as_scalar_column() else {
2828                    return Some(MapColumn { idx, row: None });
2829                };
2830                let row = sc.as_single_value_series().rechunk();
2831                // An empty Scalar column has no row, and the frame then has none either.
2832                let row = row.chunks().first().filter(|row| row.len() == 1)?.clone();
2833                Some(MapColumn {
2834                    idx,
2835                    row: Some(row),
2836                })
2837            })
2838            .collect();
2839
2840        Self {
2841            cursor: ChunkCursor::new(&df),
2842            df,
2843            schema,
2844            compat_level,
2845            parallel,
2846            map_columns,
2847            split: None,
2848            offset: 0,
2849        }
2850    }
2851
2852    /// `f` of each column index below `width`, in parallel if this export is.
2853    fn per_column<T: Send>(
2854        &self,
2855        width: usize,
2856        f: impl Fn(usize) -> PolarsResult<T> + Send + Sync,
2857    ) -> PolarsResult<Vec<T>> {
2858        if self.parallel {
2859            // Collecting into a `Result` would lose the exact length, so the error is kept aside.
2860            let error = std::sync::Mutex::new(None);
2861            let out: Vec<Option<T>> = RAYON.install(|| {
2862                (0..width)
2863                    .into_par_iter()
2864                    .map(|i| f(i).map_err(|err| *error.lock().unwrap() = Some(err)).ok())
2865                    .collect()
2866            });
2867            if let Some(err) = error.into_inner().unwrap() {
2868                return Err(err);
2869            }
2870            Ok(out.into_iter().map(Option::unwrap).collect())
2871        } else {
2872            // Collecting into a `Result` can't reserve the length, so the `Vec` would grow as it
2873            // fills.
2874            let mut out = Vec::with_capacity(width);
2875            for i in 0..width {
2876                out.push(f(i)?);
2877            }
2878            Ok(out)
2879        }
2880    }
2881
2882    /// Export chunk `idx` of `df`, which holds `rows`, as one batch.
2883    fn export(
2884        &self,
2885        df: &DataFrame,
2886        idx: usize,
2887        rows: std::ops::Range<usize>,
2888    ) -> PolarsResult<RecordBatch> {
2889        let length = rows.len();
2890        let arrays = self.per_column(df.width(), |i| {
2891            column_to_arrow(&df.columns()[i], idx, rows.clone(), self.compat_level)
2892        })?;
2893        Ok(RecordBatch::new(length, self.schema.clone(), arrays))
2894    }
2895
2896    /// Export `range` of a split chunk as one batch.
2897    fn export_piece(
2898        &self,
2899        split: &SplitChunk,
2900        range: std::ops::Range<usize>,
2901    ) -> PolarsResult<RecordBatch> {
2902        let length = range.len();
2903        let arrays = self.per_column(split.chunk.width(), |i| match &split.converted[i] {
2904            Some(arr) => Ok(arr.sliced(range.start, length)),
2905            None => {
2906                let c = &split.chunk.columns()[i];
2907                let piece = match c.as_series() {
2908                    // `Column::slice` would turn a one-row Series into a Scalar, which reads the row.
2909                    Some(s) => s.slice(range.clone()),
2910                    None => materialize_rows(c, range.clone()),
2911                };
2912                piece.try_piece_to_arrow(
2913                    self.schema.get_at_index(i).unwrap().1.dtype(),
2914                    self.compat_level,
2915                )
2916            },
2917        })?;
2918        Ok(RecordBatch::new(length, self.schema.clone(), arrays))
2919    }
2920
2921    /// Whether `rows` of chunk `idx` of `df` export without overflowing the `i32` offsets of an
2922    /// arrow MAP; a chunk too short for `rows` doesn't, which leaves it to [`Self::split`] to report.
2923    pub(crate) fn fits(&self, df: &DataFrame, idx: usize, rows: std::ops::Range<usize>) -> bool {
2924        self.map_columns.iter().all(|m| {
2925            let dtype = self.schema.get_at_index(m.idx).unwrap().1.dtype();
2926            match (&m.row, df.columns()[m.idx].as_series()) {
2927                (Some(row), _) => map_spans_fit(row.as_ref(), dtype, 0..1, rows.len()),
2928                (None, Some(s)) => s.chunks().get(idx).is_some_and(|c| {
2929                    rows.end <= c.len() && map_spans_fit(c.as_ref(), dtype, rows.clone(), 1)
2930                }),
2931                (None, None) => true,
2932            }
2933        })
2934    }
2935
2936    /// Prepare chunk `idx`, which holds `rows`, to be exported in pieces.
2937    fn split(&self, idx: usize, rows: std::ops::Range<usize>) -> PolarsResult<SplitChunk> {
2938        // The pieces slice every column to their rows, so `RecordBatch` would not catch a
2939        // misaligned chunk.
2940        for s in self.df.columns().iter().filter_map(Column::as_series) {
2941            polars_ensure!(
2942                s.chunks().get(idx).is_some_and(|c| c.len() == rows.len()),
2943                ShapeMismatch: "cannot export to arrow: the chunks of column '{}' are not \
2944                aligned with the other columns", s.name(),
2945            );
2946        }
2947        let chunk = self.df.chunk_frame(idx, rows);
2948        let length = chunk.height();
2949        let converted = self.per_column(chunk.width(), |i| {
2950            if self.map_columns.binary_search_by_key(&i, |m| m.idx).is_ok() {
2951                return Ok(None);
2952            }
2953            column_to_arrow(&chunk.columns()[i], 0, 0..length, self.compat_level).map(Some)
2954        })?;
2955        Ok(SplitChunk {
2956            chunk,
2957            converted,
2958            start: 0,
2959        })
2960    }
2961}
2962
2963impl Iterator for RecordBatchIter<'_> {
2964    type Item = PolarsResult<RecordBatch>;
2965
2966    fn next(&mut self) -> Option<Self::Item> {
2967        if let Some(split) = &self.split {
2968            let height = split.chunk.height();
2969            if split.start < height {
2970                let end = arrow_export_cut(split.start..height, |r| self.fits(&split.chunk, 0, r));
2971                let batch = self.export_piece(split, split.start..end);
2972                self.split.as_mut().unwrap().start = end;
2973                return Some(batch);
2974            }
2975            self.split = None;
2976        }
2977
2978        let (idx, length) = self.cursor.next()?;
2979        let rows = self.offset..self.offset + length;
2980        self.offset += length;
2981        if !self.fits(&self.df, idx, 0..length) {
2982            match self.split(idx, rows) {
2983                Ok(split) => {
2984                    self.split = Some(split);
2985                    return self.next();
2986                },
2987                Err(err) => return Some(Err(err)),
2988            }
2989        }
2990        Some(self.export(&self.df, idx, rows))
2991    }
2992
2993    fn size_hint(&self) -> (usize, Option<usize>) {
2994        let (chunks, _) = self.cursor.size_hint();
2995        let piece = self
2996            .split
2997            .as_ref()
2998            .is_some_and(|split| split.start < split.chunk.height());
2999        let lower = chunks + piece as usize;
3000        (lower, self.map_columns.is_empty().then_some(lower))
3001    }
3002}
3003
3004pub struct PhysRecordBatchIter<'a> {
3005    df: &'a DataFrame,
3006    schema: ArrowSchemaRef,
3007    cursor: ChunkCursor,
3008    /// The first row of the next chunk, to slice columns without chunks of their own.
3009    offset: usize,
3010}
3011
3012impl Iterator for PhysRecordBatchIter<'_> {
3013    type Item = RecordBatch;
3014
3015    fn next(&mut self) -> Option<Self::Item> {
3016        let (idx, length) = self.cursor.next()?;
3017        let offset = self.offset;
3018        self.offset += length;
3019
3020        let arrs = self
3021            .df
3022            .columns()
3023            .iter()
3024            .map(|c| match c.as_series() {
3025                Some(s) => s.chunks()[idx].clone(),
3026                None => c
3027                    .slice((offset, length))
3028                    .take_materialized_series()
3029                    .rechunk()
3030                    .chunks()[0]
3031                    .clone(),
3032            })
3033            .collect::<Vec<_>>();
3034
3035        Some(RecordBatch::new(length, self.schema.clone(), arrs))
3036    }
3037
3038    fn size_hint(&self) -> (usize, Option<usize>) {
3039        self.cursor.size_hint()
3040    }
3041}
3042
3043pub enum RecordBatchIterWrap<'a> {
3044    ZeroWidth {
3045        remaining_height: usize,
3046        chunk_size: usize,
3047    },
3048    Batches(RecordBatchIter<'a>),
3049    PhysicalBatches(PhysRecordBatchIter<'a>),
3050}
3051
3052impl<'a> RecordBatchIterWrap<'a> {
3053    fn new(df: Cow<'a, DataFrame>, compat_level: CompatLevel, parallel: bool) -> Self {
3054        debug_assert!(!df.should_rechunk(), "expected equal chunks");
3055
3056        if df.width() == 0 {
3057            return Self::new_zero_width(df.height());
3058        }
3059
3060        Self::Batches(RecordBatchIter::new(df, compat_level, parallel))
3061    }
3062
3063    fn new_zero_width(height: usize) -> Self {
3064        Self::ZeroWidth {
3065            remaining_height: height,
3066            chunk_size: polars_config::config().ideal_morsel_size() as usize,
3067        }
3068    }
3069}
3070
3071impl Iterator for RecordBatchIterWrap<'_> {
3072    type Item = PolarsResult<RecordBatch>;
3073
3074    fn next(&mut self) -> Option<Self::Item> {
3075        match self {
3076            Self::ZeroWidth {
3077                remaining_height,
3078                chunk_size,
3079            } => {
3080                let n = usize::min(*remaining_height, *chunk_size);
3081                *remaining_height -= n;
3082
3083                (n > 0).then(|| Ok(RecordBatch::new(n, ArrowSchemaRef::default(), vec![])))
3084            },
3085            Self::Batches(v) => v.next(),
3086            Self::PhysicalBatches(v) => v.next().map(Ok),
3087        }
3088    }
3089
3090    fn size_hint(&self) -> (usize, Option<usize>) {
3091        match self {
3092            Self::ZeroWidth {
3093                remaining_height,
3094                chunk_size,
3095            } => {
3096                let n = remaining_height.div_ceil(*chunk_size);
3097                (n, Some(n))
3098            },
3099            Self::Batches(v) => v.size_hint(),
3100            Self::PhysicalBatches(v) => v.size_hint(),
3101        }
3102    }
3103}
3104
3105// utility to test if we can vstack/extend the columns
3106fn ensure_can_extend(left: &Column, right: &Column) -> PolarsResult<()> {
3107    polars_ensure!(
3108        left.name() == right.name(),
3109        ShapeMismatch: "unable to vstack, column names don't match: {:?} and {:?}",
3110        left.name(), right.name(),
3111    );
3112    Ok(())
3113}
3114
3115#[cfg(test)]
3116mod test {
3117    use super::*;
3118
3119    fn create_frame() -> DataFrame {
3120        let s0 = Column::new("days".into(), [0, 1, 2].as_ref());
3121        let s1 = Column::new("temp".into(), [22.1, 19.9, 7.].as_ref());
3122        DataFrame::new_infer_height(vec![s0, s1]).unwrap()
3123    }
3124
3125    #[test]
3126    fn sort_in_place_keeps_the_chunks_of_a_frame_sorted_by_one_key() {
3127        let mut df = df!("a" => [1, 2], "b" => [1, 1]).unwrap();
3128        df.vstack_mut(&df!("a" => [3, 4], "b" => [1, 1]).unwrap())
3129            .unwrap();
3130        df.apply("a", |c| {
3131            let mut c = c.clone();
3132            c.set_sorted_flag(IsSorted::Ascending);
3133            c
3134        })
3135        .unwrap();
3136        assert_eq!(df.first_col_n_chunks(), 2);
3137
3138        df.sort_in_place(["a"], SortMultipleOptions::default())
3139            .unwrap();
3140        assert_eq!(df.first_col_n_chunks(), 2);
3141
3142        df.sort_in_place(["a", "b"], SortMultipleOptions::default())
3143            .unwrap();
3144        assert_eq!(df.first_col_n_chunks(), 1);
3145        let a = df.column("a").unwrap().as_materialized_series();
3146        assert_eq!(
3147            a.i32().unwrap().to_vec(),
3148            [Some(1), Some(2), Some(3), Some(4)]
3149        );
3150    }
3151
3152    #[test]
3153    #[cfg_attr(miri, ignore)]
3154    fn test_recordbatch_iterator() {
3155        let df = df!(
3156            "foo" => [1, 2, 3, 4, 5]
3157        )
3158        .unwrap();
3159        let mut iter = df.iter_chunks(CompatLevel::newest(), false);
3160        assert_eq!(5, iter.next().unwrap().len());
3161        assert!(iter.next().is_none());
3162    }
3163
3164    #[test]
3165    #[cfg_attr(miri, ignore)]
3166    fn test_select() {
3167        let df = create_frame();
3168        assert_eq!(
3169            df.column("days")
3170                .unwrap()
3171                .as_series()
3172                .unwrap()
3173                .equal(1)
3174                .unwrap()
3175                .sum(),
3176            Some(1)
3177        );
3178    }
3179
3180    #[test]
3181    #[cfg_attr(miri, ignore)]
3182    fn test_filter_broadcast_on_string_col() {
3183        let col_name = "some_col";
3184        let v = vec!["test".to_string()];
3185        let s0 = Column::new(PlSmallStr::from_str(col_name), v);
3186        let mut df = DataFrame::new_infer_height(vec![s0]).unwrap();
3187
3188        df = df
3189            .filter(
3190                &df.column(col_name)
3191                    .unwrap()
3192                    .as_materialized_series()
3193                    .equal("")
3194                    .unwrap(),
3195            )
3196            .unwrap();
3197        assert_eq!(
3198            df.column(col_name)
3199                .unwrap()
3200                .as_materialized_series()
3201                .n_chunks(),
3202            1
3203        );
3204    }
3205
3206    #[test]
3207    #[cfg_attr(miri, ignore)]
3208    fn test_filter_broadcast_on_list_col() {
3209        let s1 = Series::new(PlSmallStr::EMPTY, [true, false, true]);
3210        let ll: ListChunked = [&s1].iter().copied().collect();
3211
3212        let mask = BooleanChunked::from_slice(PlSmallStr::EMPTY, &[false]);
3213        let new = ll.filter(&mask).unwrap();
3214
3215        assert_eq!(new.chunks.len(), 1);
3216        assert_eq!(new.len(), 0);
3217    }
3218
3219    #[test]
3220    fn slice() {
3221        let df = create_frame();
3222        let sliced_df = df.slice(Slice::head(2));
3223        assert_eq!(sliced_df.shape(), (2, 2));
3224    }
3225
3226    #[test]
3227    fn rechunk_false() {
3228        let df = create_frame();
3229        assert!(!df.should_rechunk())
3230    }
3231
3232    #[test]
3233    fn rechunk_true() -> PolarsResult<()> {
3234        let mut base = df!(
3235            "a" => [1, 2, 3],
3236            "b" => [1, 2, 3]
3237        )?;
3238
3239        // Create a series with multiple chunks
3240        let mut s = Series::new("foo".into(), 0..2);
3241        let s2 = Series::new("bar".into(), 0..1);
3242        s.append(&s2)?;
3243
3244        // Append series to frame
3245        let out = base.with_column(s.into_column())?;
3246
3247        // Now we should rechunk
3248        assert!(out.should_rechunk());
3249        Ok(())
3250    }
3251
3252    #[test]
3253    fn test_duplicate_column() {
3254        let mut df = df! {
3255            "foo" => [1, 2, 3]
3256        }
3257        .unwrap();
3258        // check if column is replaced
3259        assert!(
3260            df.with_column(Column::new("foo".into(), &[1, 2, 3]))
3261                .is_ok()
3262        );
3263        assert!(
3264            df.with_column(Column::new("bar".into(), &[1, 2, 3]))
3265                .is_ok()
3266        );
3267        assert!(df.column("bar").is_ok())
3268    }
3269
3270    #[test]
3271    #[cfg_attr(miri, ignore)]
3272    fn distinct() {
3273        let df = df! {
3274            "flt" => [1., 1., 2., 2., 3., 3.],
3275            "int" => [1, 1, 2, 2, 3, 3, ],
3276            "str" => ["a", "a", "b", "b", "c", "c"]
3277        }
3278        .unwrap();
3279        let df = df
3280            .unique_stable(None, UniqueKeepStrategy::First, None)
3281            .unwrap()
3282            .sort(["flt"], SortMultipleOptions::default())
3283            .unwrap();
3284        let valid = df! {
3285            "flt" => [1., 2., 3.],
3286            "int" => [1, 2, 3],
3287            "str" => ["a", "b", "c"]
3288        }
3289        .unwrap();
3290        assert!(df.equals(&valid));
3291    }
3292
3293    #[test]
3294    fn test_vstack() {
3295        // check that it does not accidentally rechunks
3296        let mut df = df! {
3297            "flt" => [1., 1., 2., 2., 3., 3.],
3298            "int" => [1, 1, 2, 2, 3, 3, ],
3299            "str" => ["a", "a", "b", "b", "c", "c"]
3300        }
3301        .unwrap();
3302
3303        df.vstack_mut(&df.slice(Slice::head(3))).unwrap();
3304        assert_eq!(df.first_col_n_chunks(), 2)
3305    }
3306
3307    #[test]
3308    fn test_chunk_alignment_with_scalar_column() {
3309        let mut df = df! { "int" => [0, 1] }.unwrap();
3310        df.vstack_mut(&df! { "int" => [2] }.unwrap()).unwrap();
3311        df.insert_column(
3312            0,
3313            Column::new_scalar("str".into(), Scalar::from(PlSmallStr::from_static("a")), 3),
3314        )
3315        .unwrap();
3316
3317        // A materialized scalar column still follows the layout of the other columns.
3318        for materialize in [false, true] {
3319            if materialize {
3320                df.columns()[0].as_materialized_series();
3321            }
3322            assert!(!df.should_rechunk());
3323
3324            let batch_lengths = |batches: Vec<RecordBatch>| {
3325                batches
3326                    .iter()
3327                    .map(|b| {
3328                        assert!(b.arrays().iter().all(|arr| arr.len() == b.len()));
3329                        b.len()
3330                    })
3331                    .collect::<Vec<_>>()
3332            };
3333            let batches = df.iter_chunks(CompatLevel::newest(), false).collect();
3334            assert_eq!(batch_lengths(batches), [2, 1]);
3335            let batches = df.iter_chunks_physical().collect();
3336            assert_eq!(batch_lengths(batches), [2, 1]);
3337        }
3338    }
3339
3340    #[test]
3341    fn test_vstack_scalar_chunk_alignment() {
3342        // Appending equal scalars keeps the column unmaterialized.
3343        let mut df = df! {
3344            "int" => [0],
3345            "str" => ["a"],
3346        }
3347        .unwrap();
3348        df.vstack_mut(&df! { "int" => [1], "str" => ["a"] }.unwrap())
3349            .unwrap();
3350
3351        assert_eq!(df.column("int").unwrap().n_chunks(), 2);
3352        assert!(df.column("str").unwrap().as_scalar_column().is_some());
3353        assert!(!df.should_rechunk());
3354
3355        let batches = df
3356            .iter_chunks(CompatLevel::newest(), false)
3357            .collect::<Vec<_>>();
3358        assert_eq!(batches.iter().map(|b| b.len()).collect::<Vec<_>>(), [1, 1]);
3359    }
3360
3361    #[test]
3362    fn test_vstack_on_empty_dataframe() {
3363        let mut df = DataFrame::empty();
3364
3365        let df_data = df! {
3366            "flt" => [1., 1., 2., 2., 3., 3.],
3367            "int" => [1, 1, 2, 2, 3, 3, ],
3368            "str" => ["a", "a", "b", "b", "c", "c"]
3369        }
3370        .unwrap();
3371
3372        df.vstack_mut(&df_data).unwrap();
3373        assert_eq!(df.height(), 6)
3374    }
3375
3376    #[test]
3377    fn test_unique_keep_none_with_slice() {
3378        let df = df! {
3379            "x" => [1, 2, 3, 2, 1]
3380        }
3381        .unwrap();
3382        let out = df
3383            .unique_stable(
3384                Some(&["x".to_string()][..]),
3385                UniqueKeepStrategy::None,
3386                Some(Slice::head(2)),
3387            )
3388            .unwrap();
3389        let expected = df! {
3390            "x" => [3]
3391        }
3392        .unwrap();
3393        assert!(out.equals(&expected));
3394    }
3395
3396    #[test]
3397    #[cfg(feature = "dtype-i8")]
3398    fn test_apply_result_schema() {
3399        let mut df = df! {
3400            "x" => [1, 2, 3, 2, 1]
3401        }
3402        .unwrap();
3403
3404        let schema_before = df.schema().clone();
3405        df.apply("x", |f| f.cast(&DataType::Int8).unwrap()).unwrap();
3406        assert_ne!(&schema_before, df.schema());
3407    }
3408}