1#![allow(unsafe_op_in_unsafe_fn)]
2use std::borrow::Cow;
4
5use either::Either;
6use polars_arrow::datatypes::ArrowSchemaRef;
7use polars_row::ArrayRef;
8use polars_utils::UnitVec;
9use polars_utils::itertools::Itertools;
10use rayon::prelude::*;
11
12use crate::chunked_array::flags::StatisticsFlags;
13#[cfg(feature = "algorithm_group_by")]
14use crate::chunked_array::ops::unique::is_unique_helper;
15use crate::prelude::gather::check_bounds_ca;
16use crate::prelude::*;
17#[cfg(feature = "row_hash")]
18use crate::utils::split_df;
19use crate::utils::{Container, NoNull, try_get_supertype};
20use crate::{HEAD_DEFAULT_LENGTH, TAIL_DEFAULT_LENGTH};
21
22#[cfg(feature = "dataframe_arithmetic")]
23mod arithmetic;
24pub mod builder;
25mod chunks;
26pub use chunks::chunk_df_for_writing;
27pub mod column;
28mod dataframe;
29mod filter;
30mod projection;
31pub use dataframe::DataFrame;
32use filter::filter_zero_width;
33use projection::{AmortizedColumnSelector, LINEAR_SEARCH_LIMIT};
34
35pub mod explode;
36mod from;
37#[cfg(feature = "algorithm_group_by")]
38pub mod group_by;
39pub(crate) mod horizontal;
40#[cfg(any(feature = "rows", feature = "object"))]
41pub mod row;
42mod top_k;
43mod upstream_traits;
44pub(crate) mod validation;
45
46use polars_arrow::record_batch::{RecordBatch, RecordBatchT};
47use polars_utils::pl_str::PlSmallStr;
48use polars_utils::slice_enum::Slice;
49#[cfg(feature = "serde")]
50use serde::{Deserialize, Serialize};
51use strum_macros::IntoStaticStr;
52
53#[cfg(feature = "row_hash")]
54use crate::hashing::_df_rows_to_hashes_threaded_vertical;
55use crate::prelude::sort::arg_sort;
56use crate::runtime::RAYON;
57use crate::series::IsSorted;
58use crate::series::arrow_export::{arrow_export_cut, contains_arrow_map, map_spans_fit};
59
60#[derive(Copy, Clone, Debug, PartialEq, Eq, Default, Hash, IntoStaticStr)]
61#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
62#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
63#[strum(serialize_all = "snake_case")]
64pub enum UniqueKeepStrategy {
65 First,
67 Last,
69 None,
71 #[default]
74 Any,
75}
76
77#[derive(Copy, Clone, Debug, PartialEq, Eq, Default, Hash, IntoStaticStr)]
78#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
79#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
80#[strum(serialize_all = "snake_case")]
81pub enum PivotColumnNaming {
83 Combine,
85 #[default]
88 Auto,
89}
90
91fn column_chunk_lengths(c: &Column) -> Option<Box<dyn Iterator<Item = usize> + '_>> {
94 match c {
95 Column::Series(_) => c.as_series().map(|s| Box::new(s.chunk_lengths()) as _),
96 Column::Dictionary(d) => Some(Box::new(d.codes().chunk_lengths())),
97 Column::Scalar(_) => None,
98 }
99}
100
101impl DataFrame {
102 fn chunk_lengths_of_leading_column(&self) -> Option<Box<dyn Iterator<Item = usize> + '_>> {
106 self.columns()
107 .iter()
108 .find(|c| matches!(c, Column::Series(_)))
109 .and_then(column_chunk_lengths)
110 .or_else(|| self.columns().iter().find_map(column_chunk_lengths))
111 }
112
113 pub fn materialized_column_iter(&self) -> impl ExactSizeIterator<Item = &Series> {
114 self.columns().iter().map(Column::as_materialized_series)
115 }
116
117 pub fn estimated_size(&self, expanded: bool) -> usize {
133 self.columns()
134 .iter()
135 .map(|c| c.estimated_size(expanded))
136 .sum()
137 }
138
139 pub fn try_apply_columns(
140 &self,
141 func: impl Fn(&Column) -> PolarsResult<Column> + Send + Sync,
142 ) -> PolarsResult<Vec<Column>> {
143 return inner(self, &func);
144
145 fn inner(
146 slf: &DataFrame,
147 func: &(dyn Fn(&Column) -> PolarsResult<Column> + Send + Sync),
148 ) -> PolarsResult<Vec<Column>> {
149 slf.columns().iter().map(func).collect()
150 }
151 }
152
153 pub fn apply_columns(&self, func: impl Fn(&Column) -> Column + Send + Sync) -> Vec<Column> {
154 return inner(self, &func);
155
156 fn inner(slf: &DataFrame, func: &(dyn Fn(&Column) -> Column + Send + Sync)) -> Vec<Column> {
157 slf.columns().iter().map(func).collect()
158 }
159 }
160
161 pub fn try_apply_columns_par(
162 &self,
163 func: impl Fn(&Column) -> PolarsResult<Column> + Send + Sync,
164 ) -> PolarsResult<Vec<Column>> {
165 return inner(self, &func);
166
167 fn inner(
168 slf: &DataFrame,
169 func: &(dyn Fn(&Column) -> PolarsResult<Column> + Send + Sync),
170 ) -> PolarsResult<Vec<Column>> {
171 RAYON.install(|| slf.columns().par_iter().map(func).collect())
172 }
173 }
174
175 pub fn apply_columns_par(&self, func: impl Fn(&Column) -> Column + Send + Sync) -> Vec<Column> {
176 return inner(self, &func);
177
178 fn inner(slf: &DataFrame, func: &(dyn Fn(&Column) -> Column + Send + Sync)) -> Vec<Column> {
179 RAYON.install(|| slf.columns().par_iter().map(func).collect())
180 }
181 }
182
183 pub fn materialize_dictionary_columns(&mut self) {
185 if !self
186 .columns()
187 .iter()
188 .any(|c| c.as_dictionary_column().is_some())
189 {
190 return;
191 }
192 for c in unsafe { self.columns_mut_retain_schema() } {
194 if c.as_dictionary_column().is_some() {
195 c.into_materialized_series();
196 }
197 }
198 }
199
200 pub(crate) fn reserve_chunks(&mut self, additional: usize) {
202 for s in unsafe { self.columns_mut_retain_schema() } {
203 if let Column::Series(s) = s {
204 unsafe { s.chunks_mut().reserve(additional) }
207 }
208 }
209 }
210 pub fn new_from_index(&self, index: usize, height: usize) -> Self {
211 let new_cols = self.apply_columns(|c| c.new_from_index(index, height));
212
213 unsafe { Self::_new_unchecked_impl(height, new_cols).with_schema_from(self) }
214 }
215
216 pub fn full_null(schema: &Schema, height: usize) -> Self {
218 let columns = schema
219 .iter_fields()
220 .map(|f| Column::full_null(f.name().clone(), height, f.dtype()))
221 .collect();
222
223 unsafe { DataFrame::_new_unchecked_impl(height, columns) }
224 }
225
226 pub fn ensure_matches_schema(&mut self, schema: &Schema) -> PolarsResult<()> {
229 let mut did_cast = false;
230 let cached_schema = self.cached_schema().cloned();
231
232 for (col, (name, dt)) in unsafe { self.columns_mut() }.iter_mut().zip(schema.iter()) {
233 polars_ensure!(
234 col.name() == name,
235 SchemaMismatch: "column name mismatch: expected {:?}, found {:?}",
236 name,
237 col.name()
238 );
239
240 let needs_cast = col.dtype().matches_schema_type(dt)?;
241
242 if needs_cast {
243 *col = col.cast(dt)?;
244 did_cast = true;
245 }
246 }
247
248 if !did_cast {
249 unsafe { self.set_opt_schema(cached_schema) };
250 }
251
252 Ok(())
253 }
254
255 pub fn with_row_index(&self, name: PlSmallStr, offset: Option<IdxSize>) -> PolarsResult<Self> {
290 let mut new_columns = Vec::with_capacity(self.width() + 1);
291 let offset = offset.unwrap_or(0);
292
293 if self.get_column_index(&name).is_some() {
294 polars_bail!(duplicate = name)
295 }
296
297 let col = Column::new_row_index(name, offset, self.height())?;
298 new_columns.push(col);
299 new_columns.extend_from_slice(self.columns());
300
301 Ok(unsafe { DataFrame::new_unchecked(self.height(), new_columns) })
302 }
303
304 pub unsafe fn with_row_index_mut(
312 &mut self,
313 name: PlSmallStr,
314 offset: Option<IdxSize>,
315 ) -> &mut Self {
316 debug_assert!(
317 self.get_column_index(&name).is_none(),
318 "with_row_index_mut(): column with name {} already exists",
319 name
320 );
321
322 let offset = offset.unwrap_or(0);
323 let col = Column::new_row_index(name, offset, self.height()).unwrap();
324
325 unsafe { self.columns_mut() }.insert(0, col);
326 self
327 }
328
329 pub fn shrink_to_fit(&mut self) {
331 for s in unsafe { self.columns_mut_retain_schema() } {
333 s.shrink_to_fit();
334 }
335 }
336
337 pub fn rechunk_mut_par(&mut self) -> &mut Self {
340 if self.columns().iter().any(|c| c.n_chunks() > 1) {
341 RAYON.install(|| {
342 unsafe { self.columns_mut_retain_schema() }
343 .par_iter_mut()
344 .for_each(|c| *c = c.rechunk());
345 })
346 }
347
348 self
349 }
350
351 pub fn rechunk_mut(&mut self) -> &mut Self {
353 let columns = unsafe { self.columns_mut() };
355
356 for col in columns.iter_mut().filter(|c| c.n_chunks() > 1) {
357 *col = col.rechunk();
358 }
359
360 self
361 }
362
363 pub fn should_rechunk(&self) -> bool {
365 if !self
371 .columns()
372 .iter()
373 .filter_map(column_chunk_lengths)
374 .map(|cl| cl.size_hint().0)
375 .all_equal()
376 {
377 return true;
378 }
379
380 let mut chunk_lengths = self.columns().iter().filter_map(column_chunk_lengths);
381 match chunk_lengths.next() {
382 None => false,
383 Some(first_column_chunk_lengths) => {
384 if first_column_chunk_lengths.size_hint().0 == 1 {
386 return chunk_lengths.any(|cl| cl.size_hint().0 != 1);
387 }
388 let height = self.height();
391 let n_chunks = first_column_chunk_lengths.size_hint().0;
392 if n_chunks > height && !(height == 0 && n_chunks == 1) {
393 return true;
394 }
395 let v: Vec<_> = first_column_chunk_lengths.collect();
397 for cl in chunk_lengths {
398 if cl.enumerate().any(|(idx, el)| Some(&el) != v.get(idx)) {
399 return true;
400 }
401 }
402 false
403 },
404 }
405 }
406
407 pub fn align_chunks_par(&mut self) -> &mut Self {
409 if self.should_rechunk() {
410 self.rechunk_mut_par()
411 } else {
412 self
413 }
414 }
415
416 pub fn align_chunks(&mut self) -> &mut Self {
418 if self.should_rechunk() {
419 self.rechunk_mut()
420 } else {
421 self
422 }
423 }
424
425 pub fn get_column_names(&self) -> Vec<&PlSmallStr> {
436 self.columns().iter().map(|s| s.name()).collect()
437 }
438
439 pub fn get_column_names_owned(&self) -> Vec<PlSmallStr> {
441 self.columns().iter().map(|s| s.name().clone()).collect()
442 }
443
444 pub fn set_column_names<T>(&mut self, new_names: &[T]) -> PolarsResult<()>
456 where
457 T: AsRef<str>,
458 {
459 polars_ensure!(
460 new_names.len() == self.width(),
461 ShapeMismatch: "{} column names provided for a DataFrame of width {}",
462 new_names.len(), self.width()
463 );
464
465 validation::ensure_names_unique(new_names)?;
466
467 *unsafe { self.columns_mut() } = std::mem::take(unsafe { self.columns_mut() })
468 .into_iter()
469 .zip(new_names)
470 .map(|(c, name)| c.with_name(PlSmallStr::from_str(name.as_ref())))
471 .collect();
472
473 Ok(())
474 }
475
476 pub fn dtypes(&self) -> Vec<DataType> {
489 self.columns().iter().map(|s| s.dtype().clone()).collect()
490 }
491
492 pub fn first_col_n_chunks(&self) -> usize {
494 match self.chunk_lengths_of_leading_column() {
495 None if self.width() == 0 => 0,
496 None => 1,
497 Some(cl) => cl.size_hint().0,
498 }
499 }
500
501 pub(crate) fn first_col_chunk_lengths(&self) -> impl Iterator<Item = usize> + '_ {
503 match self.chunk_lengths_of_leading_column() {
504 None => Either::Right((self.width() > 0).then_some(self.height()).into_iter()),
505 Some(cl) => Either::Left(cl),
506 }
507 }
508
509 pub fn max_n_chunks(&self) -> usize {
511 self.columns()
512 .iter()
513 .map(|c| column_chunk_lengths(c).map_or(1, |cl| cl.size_hint().0))
514 .max()
515 .unwrap_or(0)
516 }
517
518 pub fn fields(&self) -> Vec<Field> {
534 self.columns()
535 .iter()
536 .map(|s| s.field().into_owned())
537 .collect()
538 }
539
540 pub fn hstack(&self, columns: &[Column]) -> PolarsResult<Self> {
574 let mut new_cols = Vec::with_capacity(self.width() + columns.len());
575
576 new_cols.extend(self.columns().iter().cloned());
577 new_cols.extend_from_slice(columns);
578
579 DataFrame::new(self.height(), new_cols)
580 }
581 pub fn vstack(&self, other: &DataFrame) -> PolarsResult<Self> {
622 let mut df = self.clone();
623 df.vstack_mut(other)?;
624 Ok(df)
625 }
626
627 pub fn vstack_mut(&mut self, other: &DataFrame) -> PolarsResult<&mut Self> {
668 if self.width() != other.width() {
669 polars_ensure!(
670 self.shape() == (0, 0),
671 ShapeMismatch:
672 "unable to append to a DataFrame of shape {:?} with a DataFrame of width {}",
673 self.shape(), other.width(),
674 );
675
676 self.clone_from(other);
677
678 return Ok(self);
679 }
680
681 let new_height = usize::checked_add(self.height(), other.height()).unwrap();
682
683 unsafe { self.columns_mut_retain_schema() }
684 .iter_mut()
685 .zip(other.columns())
686 .try_for_each::<_, PolarsResult<_>>(|(left, right)| {
687 ensure_can_extend(&*left, right)?;
688 left.append(right)
689 .with_context(|| format!("failed to vstack column '{}'", right.name()))?;
690 Ok(())
691 })?;
692
693 unsafe { self.set_height(new_height) };
694
695 Ok(self)
696 }
697
698 pub fn vstack_mut_owned(&mut self, other: DataFrame) -> PolarsResult<&mut Self> {
699 if self.width() != other.width() {
700 polars_ensure!(
701 self.shape() == (0, 0),
702 ShapeMismatch:
703 "unable to append to a DataFrame of width {} with a DataFrame of width {}",
704 self.width(), other.width(),
705 );
706
707 *self = other;
708
709 return Ok(self);
710 }
711
712 let new_height = usize::checked_add(self.height(), other.height()).unwrap();
713
714 unsafe { self.columns_mut_retain_schema() }
715 .iter_mut()
716 .zip(other.into_columns())
717 .try_for_each::<_, PolarsResult<_>>(|(left, right)| {
718 ensure_can_extend(&*left, &right)?;
719 let right_name = right.name().clone();
720 left.append_owned(right)
721 .with_context(|| format!("failed to vstack column '{right_name}'"))?;
722 Ok(())
723 })?;
724
725 unsafe { self.set_height(new_height) };
726
727 Ok(self)
728 }
729
730 pub fn vstack_mut_unchecked(&mut self, other: &DataFrame) -> &mut Self {
737 let new_height = usize::checked_add(self.height(), other.height()).unwrap();
738
739 unsafe { self.columns_mut_retain_schema() }
740 .iter_mut()
741 .zip(other.columns())
742 .for_each(|(left, right)| {
743 left.append(right)
744 .with_context(|| format!("failed to vstack column '{}'", right.name()))
745 .expect("should not fail");
746 });
747
748 unsafe { self.set_height(new_height) };
749
750 self
751 }
752
753 pub fn vstack_mut_owned_unchecked(&mut self, other: DataFrame) -> &mut Self {
760 let new_height = usize::checked_add(self.height(), other.height()).unwrap();
761
762 unsafe { self.columns_mut_retain_schema() }
763 .iter_mut()
764 .zip(other.into_columns())
765 .for_each(|(left, right)| {
766 left.append_owned(right).expect("should not fail");
767 });
768
769 unsafe { self.set_height(new_height) };
770
771 self
772 }
773
774 pub fn extend(&mut self, other: &DataFrame) -> PolarsResult<()> {
789 polars_ensure!(
790 self.width() == other.width(),
791 ShapeMismatch:
792 "unable to extend a DataFrame of width {} with a DataFrame of width {}",
793 self.width(), other.width(),
794 );
795
796 let new_height = usize::checked_add(self.height(), other.height()).unwrap();
797
798 unsafe { self.columns_mut_retain_schema() }
799 .iter_mut()
800 .zip(other.columns())
801 .try_for_each::<_, PolarsResult<_>>(|(left, right)| {
802 ensure_can_extend(&*left, right)?;
803 left.extend(right)
804 .with_context(|| format!("failed to extend column '{}'", right.name()))?;
805 Ok(())
806 })?;
807
808 unsafe { self.set_height(new_height) };
809
810 Ok(())
811 }
812
813 pub fn drop_in_place(&mut self, name: &str) -> PolarsResult<Column> {
830 let idx = self.try_get_column_index(name)?;
831 Ok(unsafe { self.columns_mut() }.remove(idx))
832 }
833
834 pub fn drop_nulls<S>(&self, subset: Option<&[S]>) -> PolarsResult<Self>
863 where
864 for<'a> &'a S: AsRef<str>,
865 {
866 if let Some(v) = subset {
867 let v = self.select_to_vec(v)?;
868 self._drop_nulls_impl(v.as_slice())
869 } else {
870 self._drop_nulls_impl(self.columns())
871 }
872 }
873
874 fn _drop_nulls_impl(&self, subset: &[Column]) -> PolarsResult<Self> {
875 if subset.iter().all(|s| !s.has_nulls()) {
877 return Ok(self.clone());
878 }
879
880 let mut iter = subset.iter();
881
882 let mask = iter
883 .next()
884 .ok_or_else(|| polars_err!(NoData: "no data to drop nulls from"))?;
885 let mut mask = mask.is_not_null();
886
887 for c in iter {
888 mask = mask & c.is_not_null();
889 }
890 self.filter(&mask)
891 }
892
893 pub fn drop(&self, name: &str) -> PolarsResult<Self> {
908 let idx = self.try_get_column_index(name)?;
909 let mut new_cols = Vec::with_capacity(self.width() - 1);
910
911 self.columns().iter().enumerate().for_each(|(i, s)| {
912 if i != idx {
913 new_cols.push(s.clone())
914 }
915 });
916
917 Ok(unsafe { DataFrame::_new_unchecked_impl(self.height(), new_cols) })
918 }
919
920 pub fn drop_many<I, S>(&self, names: I) -> Self
922 where
923 I: IntoIterator<Item = S>,
924 S: Into<PlSmallStr>,
925 {
926 let names: PlHashSet<PlSmallStr> = names.into_iter().map(|s| s.into()).collect();
927 self.drop_many_amortized(&names)
928 }
929
930 pub fn drop_many_amortized(&self, names: &PlHashSet<PlSmallStr>) -> DataFrame {
932 if names.is_empty() {
933 return self.clone();
934 }
935 let mut new_cols = Vec::with_capacity(self.width().saturating_sub(names.len()));
936 self.columns().iter().for_each(|s| {
937 if !names.contains(s.name()) {
938 new_cols.push(s.clone())
939 }
940 });
941
942 unsafe { DataFrame::new_unchecked(self.height(), new_cols) }
943 }
944
945 fn insert_column_no_namecheck(
948 &mut self,
949 index: usize,
950 column: Column,
951 ) -> PolarsResult<&mut Self> {
952 polars_ensure!(
953 column.len() == self.height(),
954 ShapeMismatch:
955 "unable to add a column of length {} to a DataFrame of height {}",
956 column.len(), self.height(),
957 );
958
959 unsafe { self.columns_mut() }.insert(index, column);
960 Ok(self)
961 }
962
963 pub fn insert_column(&mut self, index: usize, column: Column) -> PolarsResult<&mut Self> {
965 let name = column.name();
966
967 polars_ensure!(
968 self.get_column_index(name).is_none(),
969 Duplicate:
970 "column with name {:?} is already present in the DataFrame", name
971 );
972
973 self.insert_column_no_namecheck(index, column)
974 }
975
976 pub fn with_column(&mut self, mut column: Column) -> PolarsResult<&mut Self> {
979 column.broadcast_in_place_to(self.height())?;
980
981 if let Some(i) = self.get_column_index(column.name()) {
982 *unsafe { self.columns_mut() }.get_mut(i).unwrap() = column
983 } else {
984 unsafe { self.columns_mut() }.push(column)
985 };
986
987 Ok(self)
988 }
989
990 pub unsafe fn push_column_unchecked(&mut self, column: Column) -> &mut Self {
996 unsafe { self.columns_mut() }.push(column);
997 self
998 }
999
1000 pub fn with_columns_mut(
1003 &mut self,
1004 columns: impl IntoIterator<Item = Column>,
1005 output_schema: &Schema,
1006 ) -> PolarsResult<()> {
1007 let columns = columns.into_iter();
1008
1009 unsafe {
1010 self.columns_mut_retain_schema()
1011 .reserve(columns.size_hint().0)
1012 }
1013
1014 for c in columns {
1015 self.with_column_and_schema_mut(c, output_schema)?;
1016 }
1017
1018 Ok(())
1019 }
1020
1021 fn with_column_and_schema_mut(
1022 &mut self,
1023 mut column: Column,
1024 output_schema: &Schema,
1025 ) -> PolarsResult<&mut Self> {
1026 column.broadcast_in_place_to(self.height())?;
1027
1028 let i = output_schema
1029 .index_of(column.name())
1030 .or_else(|| self.get_column_index(column.name()))
1031 .unwrap_or(self.width());
1032
1033 if i < self.width() {
1034 *unsafe { self.columns_mut() }.get_mut(i).unwrap() = column
1035 } else if i == self.width() {
1036 unsafe { self.columns_mut() }.push(column)
1037 } else {
1038 panic!("{:?}, {}", output_schema, column.name());
1040 }
1041
1042 Ok(self)
1043 }
1044
1045 pub fn get(&self, idx: usize) -> Option<Vec<AnyValue<'_>>> {
1056 (idx < self.height()).then(|| self.columns().iter().map(|c| c.get(idx).unwrap()).collect())
1057 }
1058
1059 pub fn select_at_idx(&self, idx: usize) -> Option<&Column> {
1075 self.columns().get(idx)
1076 }
1077
1078 pub fn get_column_index(&self, name: &str) -> Option<usize> {
1096 if let Some(schema) = self.cached_schema() {
1097 schema.index_of(name)
1098 } else if self.width() <= LINEAR_SEARCH_LIMIT {
1099 self.columns().iter().position(|s| s.name() == name)
1100 } else {
1101 self.schema().index_of(name)
1102 }
1103 }
1104
1105 pub fn try_get_column_index(&self, name: &str) -> PolarsResult<usize> {
1107 self.get_column_index(name)
1108 .ok_or_else(|| polars_err!(col_not_found = name))
1109 }
1110
1111 pub fn column(&self, name: &str) -> PolarsResult<&Column> {
1125 let idx = self.try_get_column_index(name)?;
1126 Ok(self.select_at_idx(idx).unwrap())
1127 }
1128
1129 pub fn select<I, S>(&self, names: I) -> PolarsResult<Self>
1140 where
1141 I: IntoIterator<Item = S>,
1142 S: AsRef<str>,
1143 {
1144 DataFrame::new(self.height(), self.select_to_vec(names)?)
1145 }
1146
1147 pub unsafe fn select_unchecked<I, S>(&self, names: I) -> PolarsResult<Self>
1152 where
1153 I: IntoIterator<Item = S>,
1154 S: AsRef<str>,
1155 {
1156 Ok(unsafe { DataFrame::new_unchecked(self.height(), self.select_to_vec(names)?) })
1157 }
1158
1159 pub fn select_to_vec(
1177 &self,
1178 selection: impl IntoIterator<Item = impl AsRef<str>>,
1179 ) -> PolarsResult<Vec<Column>> {
1180 AmortizedColumnSelector::new(self).select_multiple(selection)
1181 }
1182
1183 pub fn filter(&self, mask: &BooleanChunked) -> PolarsResult<Self> {
1195 if self.width() == 0 {
1196 filter_zero_width(self.height(), mask)
1197 } else if mask.len() == 1 && self.len() >= 1 {
1198 if mask.all() && mask.null_count() == 0 {
1199 Ok(self.clone())
1200 } else {
1201 Ok(self.clear())
1202 }
1203 } else {
1204 let all_chunks_aligned =
1207 !self.should_rechunk() && self.first_col_chunk_lengths().eq(mask.chunk_lengths());
1208
1209 let mask = if all_chunks_aligned {
1210 Cow::Borrowed(mask)
1211 } else {
1212 mask.rechunk()
1213 };
1214
1215 let new_columns: Vec<Column> =
1216 self.try_apply_columns_par(|s| s.filter(mask.as_ref()))?;
1217 let out = unsafe {
1218 DataFrame::new_unchecked(new_columns[0].len(), new_columns).with_schema_from(self)
1219 };
1220
1221 Ok(out)
1222 }
1223 }
1224
1225 pub fn filter_seq(&self, mask: &BooleanChunked) -> PolarsResult<Self> {
1227 if self.width() == 0 {
1228 filter_zero_width(self.height(), mask)
1229 } else if mask.len() == 1 && mask.null_count() == 0 && self.len() >= 1 {
1230 if mask.all() && mask.null_count() == 0 {
1231 Ok(self.clone())
1232 } else {
1233 Ok(self.clear())
1234 }
1235 } else {
1236 let all_chunks_aligned =
1237 !self.should_rechunk() && self.first_col_chunk_lengths().eq(mask.chunk_lengths());
1238
1239 let mask = if all_chunks_aligned {
1240 Cow::Borrowed(mask)
1241 } else {
1242 mask.rechunk()
1243 };
1244
1245 let new_columns: Vec<Column> = self.try_apply_columns(|s| s.filter(mask.as_ref()))?;
1246 let out = unsafe {
1247 DataFrame::new_unchecked(new_columns[0].len(), new_columns).with_schema_from(self)
1248 };
1249
1250 Ok(out)
1251 }
1252 }
1253
1254 pub fn take(&self, indices: &IdxCa) -> PolarsResult<Self> {
1266 check_bounds_ca(indices, self.height().try_into().unwrap_or(IdxSize::MAX))?;
1267
1268 let new_cols = self.apply_columns_par(|c| {
1269 assert_eq!(c.len(), self.height());
1270 unsafe { c.take_unchecked(indices) }
1271 });
1272
1273 Ok(unsafe { DataFrame::new_unchecked(indices.len(), new_cols).with_schema_from(self) })
1274 }
1275
1276 pub unsafe fn take_unchecked(&self, idx: &IdxCa) -> Self {
1279 self.take_unchecked_impl(idx, true)
1280 }
1281
1282 #[cfg(feature = "algorithm_group_by")]
1285 pub unsafe fn gather_group_unchecked(&self, group: &GroupsIndicator) -> Self {
1286 match group {
1287 GroupsIndicator::Idx((_, indices)) => unsafe {
1288 self.take_slice_unchecked_impl(indices.as_slice(), false)
1289 },
1290 GroupsIndicator::Slice([offset, len]) => self.slice((*offset as usize, *len as usize)),
1291 }
1292 }
1293
1294 pub unsafe fn take_unchecked_impl(&self, idx: &IdxCa, allow_threads: bool) -> Self {
1297 let cols = if allow_threads && RAYON.current_num_threads() > 1 {
1298 RAYON.install(|| {
1299 if RAYON.current_num_threads() > self.width() {
1300 let stride = usize::max(idx.len().div_ceil(RAYON.current_num_threads()), 256);
1301 if self.height() / stride >= 2 {
1302 self.apply_columns_par(|c| {
1303 let c = if c.dtype().is_nested() {
1306 &c.rechunk()
1307 } else {
1308 c
1309 };
1310
1311 (0..idx.len().div_ceil(stride))
1312 .into_par_iter()
1313 .map(|i| c.take_unchecked(&idx.slice((i * stride, stride))))
1314 .reduce(
1315 || Column::new_empty(c.name().clone(), c.dtype()),
1316 |mut a, b| {
1317 a.append_owned(b).unwrap();
1318 a
1319 },
1320 )
1321 })
1322 } else {
1323 self.apply_columns_par(|c| c.take_unchecked(idx))
1324 }
1325 } else {
1326 self.apply_columns_par(|c| c.take_unchecked(idx))
1327 }
1328 })
1329 } else {
1330 self.apply_columns(|s| s.take_unchecked(idx))
1331 };
1332
1333 unsafe { DataFrame::new_unchecked(idx.len(), cols).with_schema_from(self) }
1334 }
1335
1336 pub unsafe fn take_slice_unchecked(&self, idx: &[IdxSize]) -> Self {
1339 self.take_slice_unchecked_impl(idx, true)
1340 }
1341
1342 pub unsafe fn take_slice_unchecked_impl(&self, idx: &[IdxSize], allow_threads: bool) -> Self {
1345 let cols = if allow_threads && RAYON.current_num_threads() > 1 {
1346 RAYON.install(|| {
1347 if RAYON.current_num_threads() > self.width() {
1348 let stride = usize::max(idx.len().div_ceil(RAYON.current_num_threads()), 256);
1349 if self.height() / stride >= 2 {
1350 self.apply_columns_par(|c| {
1351 let c = if c.dtype().is_nested() {
1354 &c.rechunk()
1355 } else {
1356 c
1357 };
1358
1359 (0..idx.len().div_ceil(stride))
1360 .into_par_iter()
1361 .map(|i| {
1362 let idx = &idx[i * stride..];
1363 let idx = &idx[..idx.len().min(stride)];
1364 c.take_slice_unchecked(idx)
1365 })
1366 .reduce(
1367 || Column::new_empty(c.name().clone(), c.dtype()),
1368 |mut a, b| {
1369 a.append_owned(b).unwrap();
1370 a
1371 },
1372 )
1373 })
1374 } else {
1375 self.apply_columns_par(|s| s.take_slice_unchecked(idx))
1376 }
1377 } else {
1378 self.apply_columns_par(|s| s.take_slice_unchecked(idx))
1379 }
1380 })
1381 } else {
1382 self.apply_columns(|s| s.take_slice_unchecked(idx))
1383 };
1384 unsafe { DataFrame::new_unchecked(idx.len(), cols).with_schema_from(self) }
1385 }
1386
1387 pub fn rename(&mut self, column: &str, name: PlSmallStr) -> PolarsResult<&mut Self> {
1402 if column == name.as_str() {
1403 return Ok(self);
1404 }
1405 polars_ensure!(
1406 !self.schema().contains(&name),
1407 Duplicate: "column rename attempted with already existing name \"{name}\""
1408 );
1409
1410 self.get_column_index(column)
1411 .and_then(|idx| unsafe { self.columns_mut() }.get_mut(idx))
1412 .ok_or_else(|| polars_err!(col_not_found = column))
1413 .map(|c| c.rename(name))?;
1414
1415 Ok(self)
1416 }
1417
1418 pub fn rename_many<'a>(
1419 mut self,
1420 renames: impl Iterator<Item = (&'a str, PlSmallStr)>,
1421 ) -> PolarsResult<Self> {
1422 let schema = self.schema().clone();
1423
1424 for (from, to) in renames {
1425 if from == to.as_str() {
1426 continue;
1427 }
1428
1429 let idx = schema
1430 .index_of(from)
1431 .ok_or_else(|| polars_err!(col_not_found = from))?;
1432
1433 unsafe { self.columns_mut() }
1434 .get_mut(idx)
1435 .unwrap()
1436 .rename(to);
1437 }
1438
1439 let schema = Schema::from_iter_check_duplicates(
1441 self.columns()
1442 .iter()
1443 .map(|c| c.name().clone())
1444 .zip_eq(schema.iter_values().cloned()),
1445 )?;
1446
1447 unsafe { self.set_schema(Arc::new(schema)) };
1448
1449 Ok(self)
1450 }
1451
1452 pub fn sort_in_place(
1456 &mut self,
1457 by: impl IntoIterator<Item = impl AsRef<str>>,
1458 sort_options: SortMultipleOptions,
1459 ) -> PolarsResult<&mut Self> {
1460 let by: Vec<_> = by.into_iter().collect();
1461 if by.len() > 1 {
1464 self.rechunk_mut_par();
1465 }
1466 let by_column = self.select_to_vec(by)?;
1467
1468 let mut out = self.clone().sort_impl(by_column, sort_options, None)?;
1469 unsafe { out.set_schema_from(self) };
1470
1471 *self = out;
1472
1473 Ok(self)
1474 }
1475
1476 #[doc(hidden)]
1477 pub fn sort_impl(
1479 self,
1480 mut by_column: Vec<Column>,
1481 sort_options: SortMultipleOptions,
1482 slice: Option<Slice>,
1483 ) -> PolarsResult<Self> {
1484 if by_column.is_empty() {
1485 return Ok(match slice {
1487 Some(slice) => self.slice(slice),
1488 None => self,
1489 });
1490 }
1491
1492 for column in &by_column {
1493 if column.dtype().is_object() {
1494 polars_bail!(
1495 InvalidOperation: "column '{}' has a dtype of '{}', which does not support sorting", column.name(), column.dtype()
1496 )
1497 }
1498 }
1499
1500 let first_descending = sort_options.descending[0];
1505 let first_by_column = by_column[0].name().to_string();
1506
1507 let set_sorted = |df: &mut DataFrame| {
1508 let _ = df.apply(&first_by_column, |s| {
1511 let mut s = s.clone();
1512 if first_descending {
1513 s.set_sorted_flag(IsSorted::Descending)
1514 } else {
1515 s.set_sorted_flag(IsSorted::Ascending)
1516 }
1517 s
1518 });
1519 };
1520
1521 if self.shape_has_zero() {
1522 let mut out = self;
1523 set_sorted(&mut out);
1524 return Ok(out);
1525 }
1526
1527 if let Some(Slice::Positive { offset: 0, len: k }) = slice {
1528 if k < self.height() {
1529 return self.bottom_k_impl(k, by_column, sort_options);
1530 }
1531 }
1532 #[cfg(feature = "dtype-categorical")]
1536 let is_not_categorical_enum =
1537 !(matches!(by_column[0].dtype(), DataType::Categorical(_, _))
1538 || matches!(by_column[0].dtype(), DataType::Enum(_, _)));
1539
1540 #[cfg(not(feature = "dtype-categorical"))]
1541 #[allow(non_upper_case_globals)]
1542 const is_not_categorical_enum: bool = true;
1543
1544 if by_column.len() == 1 && is_not_categorical_enum {
1545 let required_sorting = if sort_options.descending[0] {
1546 IsSorted::Descending
1547 } else {
1548 IsSorted::Ascending
1549 };
1550 let no_sorting_required = (by_column[0].is_sorted_flag() == required_sorting)
1553 && ((by_column[0].null_count() == 0)
1554 || by_column[0].get(by_column[0].len() - 1).unwrap().is_null()
1555 == sort_options.nulls_last[0]);
1556
1557 if no_sorting_required {
1558 return Ok(match slice {
1559 Some(slice) => self.slice(slice),
1560 None => self,
1561 });
1562 }
1563 }
1564
1565 let has_nested = by_column.iter().any(|s| s.dtype().is_nested());
1566 let allow_threads = sort_options.multithreaded;
1567
1568 let mut df = self;
1571 if df.columns().iter().any(|c| c.n_chunks() > 1) {
1572 let frame_key_idxs: Vec<Option<usize>> = by_column
1575 .iter()
1576 .map(|c| {
1577 let s = c.as_series()?;
1578 let idx = df.get_column_index(s.name())?;
1579 let ds = df.columns()[idx].as_series()?;
1580 Arc::ptr_eq(&ds.0, &s.0).then_some(idx)
1581 })
1582 .collect();
1583 df.rechunk_mut_par();
1584 for (c, idx) in by_column.iter_mut().zip(frame_key_idxs) {
1585 if let Some(idx) = idx {
1586 *c = df.columns()[idx].clone();
1587 }
1588 }
1589 }
1590 let df = &df;
1591 let mut take = match (by_column.len(), has_nested) {
1592 (1, false) => {
1593 let s = &by_column[0];
1594 let options = SortOptions {
1595 descending: sort_options.descending[0],
1596 nulls_last: sort_options.nulls_last[0],
1597 multithreaded: sort_options.multithreaded,
1598 maintain_order: sort_options.maintain_order,
1599 limit: sort_options.limit,
1600 };
1601 if df.width() == 1 && df.try_get_column_index(s.name().as_str()).is_ok() {
1605 let mut out = s.sort_with(options)?;
1606 if let Some(slice) = slice {
1607 out = out.slice(slice);
1608 }
1609 return Ok(out.into_frame());
1610 }
1611 s.arg_sort(options)
1612 },
1613 _ => arg_sort(&by_column, sort_options)?,
1614 };
1615
1616 if let Some(slice) = slice {
1617 take = take.slice(slice);
1618 }
1619
1620 let mut df = unsafe { df.take_unchecked_impl(&take, allow_threads) };
1623 set_sorted(&mut df);
1624 Ok(df)
1625 }
1626
1627 pub fn _to_metadata(&self) -> DataFrame {
1632 let num_columns = self.width();
1633
1634 let mut column_names =
1635 StringChunkedBuilder::new(PlSmallStr::from_static("column_name"), num_columns);
1636 let mut repr_ca = StringChunkedBuilder::new(PlSmallStr::from_static("repr"), num_columns);
1637 let mut sorted_asc_ca =
1638 BooleanChunkedBuilder::new(PlSmallStr::from_static("sorted_asc"), num_columns);
1639 let mut sorted_dsc_ca =
1640 BooleanChunkedBuilder::new(PlSmallStr::from_static("sorted_dsc"), num_columns);
1641 let mut fast_explode_list_ca =
1642 BooleanChunkedBuilder::new(PlSmallStr::from_static("fast_explode_list"), num_columns);
1643 let mut materialized_at_ca =
1644 StringChunkedBuilder::new(PlSmallStr::from_static("materialized_at"), num_columns);
1645
1646 for col in self.columns() {
1647 let flags = col.get_flags();
1648
1649 let (repr, materialized_at) = match col {
1650 Column::Series(s) => ("series", s.materialized_at()),
1651 Column::Scalar(_) => ("scalar", None),
1652 Column::Dictionary(_) => ("dictionary", None),
1653 };
1654 let sorted_asc = flags.contains(StatisticsFlags::IS_SORTED_ASC);
1655 let sorted_dsc = flags.contains(StatisticsFlags::IS_SORTED_DSC);
1656 let fast_explode_list = flags.contains(StatisticsFlags::CAN_FAST_EXPLODE_LIST);
1657
1658 column_names.append_value(col.name().clone());
1659 repr_ca.append_value(repr);
1660 sorted_asc_ca.append_value(sorted_asc);
1661 sorted_dsc_ca.append_value(sorted_dsc);
1662 fast_explode_list_ca.append_value(fast_explode_list);
1663 materialized_at_ca.append_option(materialized_at.map(|v| format!("{v:#?}")));
1664 }
1665
1666 unsafe {
1667 DataFrame::new_unchecked(
1668 self.width(),
1669 vec![
1670 column_names.finish().into_column(),
1671 repr_ca.finish().into_column(),
1672 sorted_asc_ca.finish().into_column(),
1673 sorted_dsc_ca.finish().into_column(),
1674 fast_explode_list_ca.finish().into_column(),
1675 materialized_at_ca.finish().into_column(),
1676 ],
1677 )
1678 }
1679 }
1680 pub fn sort(
1718 &self,
1719 by: impl IntoIterator<Item = impl AsRef<str>>,
1720 sort_options: SortMultipleOptions,
1721 ) -> PolarsResult<Self> {
1722 let mut df = self.clone();
1723 df.sort_in_place(by, sort_options)?;
1724 Ok(df)
1725 }
1726
1727 pub fn replace(&mut self, column: &str, new_col: Column) -> PolarsResult<&mut Self> {
1742 self.apply(column, |_| new_col)
1743 }
1744
1745 pub fn replace_column(&mut self, index: usize, new_column: Column) -> PolarsResult<&mut Self> {
1760 polars_ensure!(
1761 index < self.width(),
1762 ShapeMismatch:
1763 "unable to replace at index {}, the DataFrame has only {} columns",
1764 index, self.width(),
1765 );
1766
1767 polars_ensure!(
1768 new_column.len() == self.height(),
1769 ShapeMismatch:
1770 "unable to replace a column, series length {} doesn't match the DataFrame height {}",
1771 new_column.len(), self.height(),
1772 );
1773
1774 unsafe { *self.columns_mut().get_mut(index).unwrap() = new_column };
1775
1776 Ok(self)
1777 }
1778
1779 pub fn apply<F, C>(&mut self, name: &str, f: F) -> PolarsResult<&mut Self>
1820 where
1821 F: FnOnce(&Column) -> C,
1822 C: IntoColumn,
1823 {
1824 let idx = self.try_get_column_index(name)?;
1825 self.apply_at_idx(idx, f)?;
1826 Ok(self)
1827 }
1828
1829 pub fn apply_at_idx<F, C>(&mut self, idx: usize, f: F) -> PolarsResult<&mut Self>
1860 where
1861 F: FnOnce(&Column) -> C,
1862 C: IntoColumn,
1863 {
1864 let df_height = self.height();
1865 let width = self.width();
1866
1867 let cached_schema = self.cached_schema().cloned();
1868
1869 let col = unsafe { self.columns_mut() }.get_mut(idx).ok_or_else(|| {
1870 polars_err!(
1871 ComputeError: "invalid column index: {} for a DataFrame with {} columns",
1872 idx, width
1873 )
1874 })?;
1875
1876 let new_col = f(col)
1877 .into_column()
1878 .with_name(col.name().clone())
1879 .broadcast_owned_to(df_height)?;
1880 let col_before = std::mem::replace(col, new_col);
1881
1882 if col.dtype() == col_before.dtype() {
1883 unsafe { self.set_opt_schema(cached_schema) };
1884 }
1885
1886 Ok(self)
1887 }
1888
1889 pub fn try_apply_at_idx<F, C>(&mut self, idx: usize, f: F) -> PolarsResult<&mut Self>
1930 where
1931 F: FnOnce(&Column) -> PolarsResult<C>,
1932 C: IntoColumn,
1933 {
1934 let df_height = self.height();
1935 let width = self.width();
1936
1937 let cached_schema = self.cached_schema().cloned();
1938
1939 let col = unsafe { self.columns_mut() }.get_mut(idx).ok_or_else(|| {
1940 polars_err!(
1941 ComputeError: "invalid column index: {} for a DataFrame with {} columns",
1942 idx, width
1943 )
1944 })?;
1945
1946 let mut new_col = f(col).map(|c| c.into_column())?;
1947
1948 polars_ensure!(
1949 new_col.len() == df_height,
1950 ShapeMismatch:
1951 "try_apply_at_idx: resulting Series has length {} while the DataFrame has height {}",
1952 new_col.len(), df_height
1953 );
1954
1955 new_col = new_col.with_name(col.name().clone());
1957 let col_before = std::mem::replace(col, new_col);
1958
1959 if col.dtype() == col_before.dtype() {
1960 unsafe { self.set_opt_schema(cached_schema) };
1961 }
1962
1963 Ok(self)
1964 }
1965
1966 pub fn try_apply<F, C>(&mut self, column: &str, f: F) -> PolarsResult<&mut Self>
2009 where
2010 F: FnOnce(&Series) -> PolarsResult<C>,
2011 C: IntoColumn,
2012 {
2013 let idx = self.try_get_column_index(column)?;
2014 self.try_apply_at_idx(idx, |c| f(c.as_materialized_series()))
2015 }
2016
2017 #[must_use]
2047 pub fn slice(&self, slice: impl Into<Slice>) -> Self {
2048 let range = slice.into().resolve(self.height());
2049
2050 if range.len() == self.height() {
2051 return self.clone();
2052 }
2053
2054 if range.is_empty() {
2055 return self.clear();
2056 }
2057
2058 let cols = self.apply_columns(|s| s.slice(range.clone()));
2059
2060 unsafe { DataFrame::_new_unchecked_impl(range.len(), cols).with_schema_from(self) }
2061 }
2062
2063 pub fn split_at(&self, offset: i64) -> (Self, Self) {
2065 let (a, b) = self.columns().iter().map(|s| s.split_at(offset)).unzip();
2066
2067 let idx = Slice::resolve_offset(offset, self.height());
2068
2069 let a = unsafe { DataFrame::new_unchecked(idx, a).with_schema_from(self) };
2070 let b = unsafe { DataFrame::new_unchecked(self.height() - idx, b).with_schema_from(self) };
2071 (a, b)
2072 }
2073
2074 #[must_use]
2075 pub fn clear(&self) -> Self {
2076 let cols = self.columns().iter().map(|s| s.clear()).collect::<Vec<_>>();
2077 unsafe { DataFrame::_new_unchecked_impl(0, cols).with_schema_from(self) }
2078 }
2079
2080 #[must_use]
2114 pub fn head(&self, length: Option<usize>) -> Self {
2115 let new_height = usize::min(self.height(), length.unwrap_or(HEAD_DEFAULT_LENGTH));
2116 let new_cols = self.apply_columns(|c| c.head(Some(new_height)));
2117
2118 unsafe { DataFrame::new_unchecked(new_height, new_cols).with_schema_from(self) }
2119 }
2120
2121 #[must_use]
2152 pub fn tail(&self, length: Option<usize>) -> Self {
2153 let new_height = usize::min(self.height(), length.unwrap_or(TAIL_DEFAULT_LENGTH));
2154 let new_cols = self.apply_columns(|c| c.tail(Some(new_height)));
2155
2156 unsafe { DataFrame::new_unchecked(new_height, new_cols).with_schema_from(self) }
2157 }
2158
2159 pub fn iter_chunks(
2171 &self,
2172 compat_level: CompatLevel,
2173 parallel: bool,
2174 ) -> impl Iterator<Item = RecordBatch> + '_ {
2175 self.try_iter_chunks(compat_level, parallel)
2176 .map(|batch| batch.unwrap())
2177 }
2178
2179 pub fn try_iter_chunks(
2184 &self,
2185 compat_level: CompatLevel,
2186 parallel: bool,
2187 ) -> impl Iterator<Item = PolarsResult<RecordBatch>> + Send + '_ {
2188 let parallel = parallel
2191 && compat_level.0 == 0
2192 && self
2193 .columns()
2194 .iter()
2195 .any(|s| matches!(s.dtype(), DataType::String | DataType::Binary));
2196 RecordBatchIterWrap::new(Cow::Borrowed(self), compat_level, parallel)
2197 }
2198
2199 pub fn into_try_iter_chunks(
2205 self,
2206 compat_level: CompatLevel,
2207 parallel: bool,
2208 ) -> impl Iterator<Item = PolarsResult<RecordBatch>> + Send + 'static {
2209 RecordBatchIterWrap::new(Cow::Owned(self), compat_level, parallel)
2210 }
2211
2212 pub fn iter_chunks_physical(&self) -> impl Iterator<Item = RecordBatch> + '_ {
2222 debug_assert!(!self.should_rechunk());
2223
2224 let iter = if self.width() == 0 {
2225 RecordBatchIterWrap::new_zero_width(self.height())
2226 } else {
2227 RecordBatchIterWrap::PhysicalBatches(PhysRecordBatchIter {
2228 df: self,
2229 schema: Arc::new(
2230 self.columns()
2231 .iter()
2232 .map(|c| c.field().to_arrow(CompatLevel::newest()))
2233 .collect(),
2234 ),
2235 cursor: ChunkCursor::new(self),
2236 offset: 0,
2237 })
2238 };
2239 iter.map(|batch| batch.unwrap())
2240 }
2241
2242 #[must_use]
2244 pub fn reverse(&self) -> Self {
2245 let new_cols = self.apply_columns(Column::reverse);
2246 unsafe { DataFrame::new_unchecked(self.height(), new_cols).with_schema_from(self) }
2247 }
2248
2249 #[must_use]
2254 pub fn shift(&self, periods: i64) -> Self {
2255 let col = self.apply_columns_par(|s| s.shift(periods));
2256 unsafe { DataFrame::new_unchecked(self.height(), col).with_schema_from(self) }
2257 }
2258
2259 pub fn fill_null(&self, strategy: FillNullStrategy) -> PolarsResult<Self> {
2268 let col = self.try_apply_columns_par(|s| s.fill_null(strategy))?;
2269
2270 Ok(unsafe { DataFrame::new_unchecked(self.height(), col) })
2271 }
2272
2273 #[cfg(feature = "algorithm_group_by")]
2307 pub fn unique_stable(
2308 &self,
2309 subset: Option<&[String]>,
2310 keep: UniqueKeepStrategy,
2311 slice: Option<Slice>,
2312 ) -> PolarsResult<DataFrame> {
2313 self.unique_impl(
2314 true,
2315 subset.map(|v| v.iter().map(|x| PlSmallStr::from_str(x.as_str())).collect()),
2316 keep,
2317 slice,
2318 )
2319 }
2320
2321 #[cfg(feature = "algorithm_group_by")]
2323 pub fn unique<I, S>(
2324 &self,
2325 subset: Option<&[String]>,
2326 keep: UniqueKeepStrategy,
2327 slice: Option<Slice>,
2328 ) -> PolarsResult<DataFrame> {
2329 self.unique_impl(
2330 false,
2331 subset.map(|v| v.iter().map(|x| PlSmallStr::from_str(x.as_str())).collect()),
2332 keep,
2333 slice,
2334 )
2335 }
2336
2337 #[cfg(feature = "algorithm_group_by")]
2338 pub fn unique_impl(
2339 &self,
2340 maintain_order: bool,
2341 subset: Option<Vec<PlSmallStr>>,
2342 keep: UniqueKeepStrategy,
2343 slice: Option<Slice>,
2344 ) -> PolarsResult<Self> {
2345 if self.width() == 0 {
2346 let height = usize::min(self.height(), 1);
2347 return Ok(DataFrame::empty_with_height(height));
2348 }
2349
2350 let names = subset.unwrap_or_else(|| self.get_column_names_owned());
2351 let mut df = self.clone();
2352 df.rechunk_mut_par();
2354
2355 fn optionally_slice(
2356 groups: &'_ GroupPositions,
2357 slice: Option<Slice>,
2358 ) -> Cow<'_, GroupPositions> {
2359 match slice {
2360 Some(s) => Cow::Owned(groups.slice(s)),
2361 None => Cow::Borrowed(groups),
2362 }
2363 }
2364
2365 let columns = match (keep, maintain_order) {
2366 (UniqueKeepStrategy::First | UniqueKeepStrategy::Any, true) => {
2367 let gb = df.group_by_stable(names)?;
2368 let groups = gb.get_groups();
2369 let groups = optionally_slice(groups, slice);
2370 df.apply_columns_par(|s| unsafe { s.agg_first(&groups) })
2371 },
2372 (UniqueKeepStrategy::Last, true) => {
2373 let gb = df.group_by_stable(names)?;
2376 let groups = gb.get_groups();
2377
2378 let last_idx: NoNull<IdxCa> = groups
2379 .iter()
2380 .map(|g| match g {
2381 GroupsIndicator::Idx((_first, idx)) => idx[idx.len() - 1],
2382 GroupsIndicator::Slice([first, len]) => first + len - 1,
2383 })
2384 .collect();
2385
2386 let mut last_idx = last_idx.into_inner().sort(false);
2387
2388 if let Some(slice) = slice {
2389 last_idx = last_idx.slice(slice);
2390 }
2391
2392 let last_idx = NoNull::new(last_idx);
2393 let out = unsafe { df.take_unchecked(&last_idx) };
2394 return Ok(out);
2395 },
2396 (UniqueKeepStrategy::First | UniqueKeepStrategy::Any, false) => {
2397 let gb = df.group_by(names)?;
2398 let groups = gb.get_groups();
2399 let groups = optionally_slice(groups, slice);
2400 df.apply_columns_par(|s| unsafe { s.agg_first(&groups) })
2401 },
2402 (UniqueKeepStrategy::Last, false) => {
2403 let gb = df.group_by(names)?;
2404 let groups = gb.get_groups();
2405 let groups = optionally_slice(groups, slice);
2406 df.apply_columns_par(|s| unsafe { s.agg_last(&groups) })
2407 },
2408 (UniqueKeepStrategy::None, _) => {
2409 let df_part = df.select(names)?;
2410 let mask = df_part.is_unique()?;
2411 let mut filtered = df.filter(&mask)?;
2412
2413 if let Some(slice) = slice {
2414 filtered = filtered.slice(slice);
2415 }
2416 return Ok(filtered);
2417 },
2418 };
2419 Ok(unsafe { DataFrame::new_unchecked_infer_height(columns).with_schema_from(self) })
2420 }
2421
2422 #[cfg(feature = "algorithm_group_by")]
2436 pub fn is_unique(&self) -> PolarsResult<BooleanChunked> {
2437 let gb = self.group_by(self.get_column_names_owned())?;
2438 let groups = gb.get_groups();
2439 Ok(is_unique_helper(
2440 groups,
2441 self.height() as IdxSize,
2442 true,
2443 false,
2444 ))
2445 }
2446
2447 #[cfg(feature = "algorithm_group_by")]
2461 pub fn is_duplicated(&self) -> PolarsResult<BooleanChunked> {
2462 let gb = self.group_by(self.get_column_names_owned())?;
2463 let groups = gb.get_groups();
2464 Ok(is_unique_helper(
2465 groups,
2466 self.height() as IdxSize,
2467 false,
2468 true,
2469 ))
2470 }
2471
2472 #[must_use]
2474 pub fn null_count(&self) -> Self {
2475 let cols =
2476 self.apply_columns(|c| Column::new(c.name().clone(), [c.null_count() as IdxSize]));
2477 unsafe { Self::new_unchecked(1, cols) }
2478 }
2479
2480 #[cfg(feature = "row_hash")]
2482 pub fn hash_rows(
2483 &mut self,
2484 hasher_builder: Option<PlSeedableRandomStateQuality>,
2485 ) -> PolarsResult<UInt64Chunked> {
2486 let dfs = split_df(self, RAYON.current_num_threads(), false);
2487 let (cas, _) = _df_rows_to_hashes_threaded_vertical(&dfs, hasher_builder)?;
2488
2489 let mut iter = cas.into_iter();
2490 let mut acc_ca = iter.next().unwrap();
2491 for ca in iter {
2492 acc_ca.append(&ca)?;
2493 }
2494 Ok(acc_ca.rechunk().into_owned())
2495 }
2496
2497 pub fn get_supertype(&self) -> Option<PolarsResult<DataType>> {
2499 self.columns()
2500 .iter()
2501 .map(|s| Ok(s.dtype().clone()))
2502 .reduce(|acc, b| try_get_supertype(&acc?, &b.unwrap()))
2503 }
2504
2505 #[doc(hidden)]
2510 pub unsafe fn _take_unchecked_slice(&self, idx: &[IdxSize], allow_threads: bool) -> Self {
2511 self._take_unchecked_slice_sorted(idx, allow_threads, IsSorted::Not)
2512 }
2513
2514 #[doc(hidden)]
2521 pub unsafe fn _take_unchecked_slice_sorted(
2522 &self,
2523 idx: &[IdxSize],
2524 allow_threads: bool,
2525 sorted: IsSorted,
2526 ) -> Self {
2527 #[cfg(debug_assertions)]
2528 {
2529 if idx.len() > 2 {
2530 use crate::series::IsSorted;
2531
2532 match sorted {
2533 IsSorted::Ascending => {
2534 assert!(idx[0] <= idx[idx.len() - 1]);
2535 },
2536 IsSorted::Descending => {
2537 assert!(idx[0] >= idx[idx.len() - 1]);
2538 },
2539 _ => {},
2540 }
2541 }
2542 }
2543 let mut ca = IdxCa::mmap_slice(PlSmallStr::EMPTY, idx);
2544 ca.set_sorted_flag(sorted);
2545 self.take_unchecked_impl(&ca, allow_threads)
2546 }
2547 #[cfg(all(feature = "partition_by", feature = "algorithm_group_by"))]
2548 #[doc(hidden)]
2549 pub fn _partition_by_impl(
2550 &self,
2551 cols: &[PlSmallStr],
2552 stable: bool,
2553 include_key: bool,
2554 parallel: bool,
2555 ) -> PolarsResult<Vec<DataFrame>> {
2556 let selected_keys = self.select_to_vec(cols.iter().cloned())?;
2557 let groups = self.group_by_with_series(selected_keys, parallel, stable)?;
2558 let groups = groups.into_groups();
2559
2560 let df = if include_key {
2562 self.clone()
2563 } else {
2564 self.drop_many(cols.iter().cloned())
2565 };
2566
2567 if parallel {
2568 RAYON.install(|| {
2571 match groups.as_ref() {
2572 GroupsType::Idx(idx) => {
2573 let mut df = df.clone();
2575 df.rechunk_mut_par();
2576 Ok(idx
2577 .into_par_iter()
2578 .map(|(_, group)| {
2579 unsafe {
2581 df._take_unchecked_slice_sorted(
2582 group,
2583 false,
2584 IsSorted::Ascending,
2585 )
2586 }
2587 })
2588 .collect())
2589 },
2590 GroupsType::Slice { groups, .. } => Ok(groups
2591 .into_par_iter()
2592 .map(|[first, len]| df.slice((*first as usize, *len as usize)))
2593 .collect()),
2594 }
2595 })
2596 } else {
2597 match groups.as_ref() {
2598 GroupsType::Idx(idx) => {
2599 let mut df = df;
2601 df.rechunk_mut();
2602 Ok(idx
2603 .into_iter()
2604 .map(|(_, group)| {
2605 unsafe {
2607 df._take_unchecked_slice_sorted(group, false, IsSorted::Ascending)
2608 }
2609 })
2610 .collect())
2611 },
2612 GroupsType::Slice { groups, .. } => Ok(groups
2613 .iter()
2614 .map(|[first, len]| df.slice((*first as usize, *len as usize)))
2615 .collect()),
2616 }
2617 }
2618 }
2619
2620 #[cfg(feature = "partition_by")]
2622 pub fn partition_by<I, S>(&self, cols: I, include_key: bool) -> PolarsResult<Vec<DataFrame>>
2623 where
2624 I: IntoIterator<Item = S>,
2625 S: Into<PlSmallStr>,
2626 {
2627 let cols: UnitVec<PlSmallStr> = cols.into_iter().map(Into::into).collect();
2628 self._partition_by_impl(cols.as_slice(), false, include_key, true)
2629 }
2630
2631 #[cfg(feature = "partition_by")]
2634 pub fn partition_by_stable<I, S>(
2635 &self,
2636 cols: I,
2637 include_key: bool,
2638 ) -> PolarsResult<Vec<DataFrame>>
2639 where
2640 I: IntoIterator<Item = S>,
2641 S: Into<PlSmallStr>,
2642 {
2643 let cols: UnitVec<PlSmallStr> = cols.into_iter().map(Into::into).collect();
2644 self._partition_by_impl(cols.as_slice(), true, include_key, true)
2645 }
2646
2647 #[cfg(feature = "dtype-struct")]
2650 pub fn unnest(
2651 &self,
2652 cols: impl IntoIterator<Item = impl Into<PlSmallStr>>,
2653 separator: Option<&str>,
2654 ) -> PolarsResult<DataFrame> {
2655 self.unnest_impl(cols.into_iter().map(Into::into).collect(), separator)
2656 }
2657
2658 #[cfg(feature = "dtype-struct")]
2659 fn unnest_impl(
2660 &self,
2661 cols: PlHashSet<PlSmallStr>,
2662 separator: Option<&str>,
2663 ) -> PolarsResult<DataFrame> {
2664 let mut new_cols = Vec::with_capacity(std::cmp::min(self.width() * 2, self.width() + 128));
2665 let mut count = 0;
2666 for s in self.columns() {
2667 if cols.contains(s.name()) {
2668 let ca = s.struct_()?.clone();
2669 new_cols.extend(ca.fields_as_series().into_iter().map(|mut f| {
2670 if let Some(separator) = &separator {
2671 f.rename(polars_utils::format_pl_smallstr!(
2672 "{}{}{}",
2673 s.name(),
2674 separator,
2675 f.name()
2676 ));
2677 }
2678 Column::from(f)
2679 }));
2680 count += 1;
2681 } else {
2682 new_cols.push(s.clone())
2683 }
2684 }
2685 if count != cols.len() {
2686 let schema = self.schema();
2689 for col in cols {
2690 let _ = schema
2691 .get(col.as_str())
2692 .ok_or_else(|| polars_err!(col_not_found = col))?;
2693 }
2694 }
2695
2696 DataFrame::new(self.height(), new_cols)
2697 }
2698
2699 pub fn append_record_batch(&mut self, rb: RecordBatchT<ArrayRef>) -> PolarsResult<()> {
2700 let df = DataFrame::from(rb);
2703 polars_ensure!(
2704 self.schema() == df.schema(),
2705 SchemaMismatch: "cannot append record batch with different schema\n\n
2706 Got {:?}\nexpected: {:?}", df.schema(), self.schema(),
2707 );
2708 self.vstack_mut_owned_unchecked(df);
2709 Ok(())
2710 }
2711}
2712
2713struct ChunkCursor {
2715 chunks: Vec<usize>,
2718 height: usize,
2719 idx: usize,
2720}
2721
2722impl ChunkCursor {
2723 fn new(df: &DataFrame) -> Self {
2724 Self {
2725 chunks: df.first_col_chunk_lengths().collect(),
2726 height: df.height(),
2727 idx: 0,
2728 }
2729 }
2730
2731 fn n_chunks(&self) -> usize {
2732 usize::max(1, self.chunks.len())
2733 }
2734
2735 fn next(&mut self) -> Option<(usize, usize)> {
2737 let idx = self.idx;
2738 if idx >= self.n_chunks() {
2739 return None;
2740 }
2741
2742 self.idx += 1;
2743 Some((idx, self.chunks.get(idx).copied().unwrap_or(self.height)))
2744 }
2745
2746 fn size_hint(&self) -> (usize, Option<usize>) {
2747 let n = self.n_chunks() - self.idx;
2748 (n, Some(n))
2749 }
2750}
2751
2752pub struct RecordBatchIter<'a> {
2753 df: Cow<'a, DataFrame>,
2754 schema: ArrowSchemaRef,
2755 cursor: ChunkCursor,
2756 compat_level: CompatLevel,
2757 parallel: bool,
2758 map_columns: Vec<MapColumn>,
2761 split: Option<SplitChunk>,
2762 offset: usize,
2764}
2765
2766struct MapColumn {
2768 idx: usize,
2769 row: Option<ArrayRef>,
2772}
2773
2774struct SplitChunk {
2776 chunk: DataFrame,
2777 converted: Vec<Option<ArrayRef>>,
2780 start: usize,
2782}
2783
2784fn materialize_rows(c: &Column, rows: std::ops::Range<usize>) -> Series {
2786 let s = if rows.len() == c.len() {
2788 c.as_materialized_series().clone()
2789 } else {
2790 c.slice(rows).take_materialized_series()
2791 };
2792 if s.n_chunks() > 1 { s.rechunk() } else { s }
2793}
2794
2795fn column_to_arrow(
2797 c: &Column,
2798 idx: usize,
2799 rows: std::ops::Range<usize>,
2800 compat_level: CompatLevel,
2801) -> PolarsResult<ArrayRef> {
2802 match c.as_series() {
2803 Some(s) => s.try_to_arrow(idx, compat_level),
2804 None => materialize_rows(c, rows).try_to_arrow(0, compat_level),
2806 }
2807}
2808
2809impl<'a> RecordBatchIter<'a> {
2810 pub(crate) fn new(df: Cow<'a, DataFrame>, compat_level: CompatLevel, parallel: bool) -> Self {
2812 let parallel = parallel && df.width() > 1;
2813
2814 let schema: ArrowSchemaRef = Arc::new(
2815 df.columns()
2816 .iter()
2817 .map(|c| c.field().to_arrow(compat_level))
2818 .collect(),
2819 );
2820 let map_columns = df
2821 .columns()
2822 .iter()
2823 .zip(schema.iter_values())
2824 .enumerate()
2825 .filter(|(_, (_, f))| contains_arrow_map(f.dtype()))
2826 .filter_map(|(idx, (c, _))| {
2827 let Some(sc) = c.as_scalar_column() else {
2828 return Some(MapColumn { idx, row: None });
2829 };
2830 let row = sc.as_single_value_series().rechunk();
2831 let row = row.chunks().first().filter(|row| row.len() == 1)?.clone();
2833 Some(MapColumn {
2834 idx,
2835 row: Some(row),
2836 })
2837 })
2838 .collect();
2839
2840 Self {
2841 cursor: ChunkCursor::new(&df),
2842 df,
2843 schema,
2844 compat_level,
2845 parallel,
2846 map_columns,
2847 split: None,
2848 offset: 0,
2849 }
2850 }
2851
2852 fn per_column<T: Send>(
2854 &self,
2855 width: usize,
2856 f: impl Fn(usize) -> PolarsResult<T> + Send + Sync,
2857 ) -> PolarsResult<Vec<T>> {
2858 if self.parallel {
2859 let error = std::sync::Mutex::new(None);
2861 let out: Vec<Option<T>> = RAYON.install(|| {
2862 (0..width)
2863 .into_par_iter()
2864 .map(|i| f(i).map_err(|err| *error.lock().unwrap() = Some(err)).ok())
2865 .collect()
2866 });
2867 if let Some(err) = error.into_inner().unwrap() {
2868 return Err(err);
2869 }
2870 Ok(out.into_iter().map(Option::unwrap).collect())
2871 } else {
2872 let mut out = Vec::with_capacity(width);
2875 for i in 0..width {
2876 out.push(f(i)?);
2877 }
2878 Ok(out)
2879 }
2880 }
2881
2882 fn export(
2884 &self,
2885 df: &DataFrame,
2886 idx: usize,
2887 rows: std::ops::Range<usize>,
2888 ) -> PolarsResult<RecordBatch> {
2889 let length = rows.len();
2890 let arrays = self.per_column(df.width(), |i| {
2891 column_to_arrow(&df.columns()[i], idx, rows.clone(), self.compat_level)
2892 })?;
2893 Ok(RecordBatch::new(length, self.schema.clone(), arrays))
2894 }
2895
2896 fn export_piece(
2898 &self,
2899 split: &SplitChunk,
2900 range: std::ops::Range<usize>,
2901 ) -> PolarsResult<RecordBatch> {
2902 let length = range.len();
2903 let arrays = self.per_column(split.chunk.width(), |i| match &split.converted[i] {
2904 Some(arr) => Ok(arr.sliced(range.start, length)),
2905 None => {
2906 let c = &split.chunk.columns()[i];
2907 let piece = match c.as_series() {
2908 Some(s) => s.slice(range.clone()),
2910 None => materialize_rows(c, range.clone()),
2911 };
2912 piece.try_piece_to_arrow(
2913 self.schema.get_at_index(i).unwrap().1.dtype(),
2914 self.compat_level,
2915 )
2916 },
2917 })?;
2918 Ok(RecordBatch::new(length, self.schema.clone(), arrays))
2919 }
2920
2921 pub(crate) fn fits(&self, df: &DataFrame, idx: usize, rows: std::ops::Range<usize>) -> bool {
2924 self.map_columns.iter().all(|m| {
2925 let dtype = self.schema.get_at_index(m.idx).unwrap().1.dtype();
2926 match (&m.row, df.columns()[m.idx].as_series()) {
2927 (Some(row), _) => map_spans_fit(row.as_ref(), dtype, 0..1, rows.len()),
2928 (None, Some(s)) => s.chunks().get(idx).is_some_and(|c| {
2929 rows.end <= c.len() && map_spans_fit(c.as_ref(), dtype, rows.clone(), 1)
2930 }),
2931 (None, None) => true,
2932 }
2933 })
2934 }
2935
2936 fn split(&self, idx: usize, rows: std::ops::Range<usize>) -> PolarsResult<SplitChunk> {
2938 for s in self.df.columns().iter().filter_map(Column::as_series) {
2941 polars_ensure!(
2942 s.chunks().get(idx).is_some_and(|c| c.len() == rows.len()),
2943 ShapeMismatch: "cannot export to arrow: the chunks of column '{}' are not \
2944 aligned with the other columns", s.name(),
2945 );
2946 }
2947 let chunk = self.df.chunk_frame(idx, rows);
2948 let length = chunk.height();
2949 let converted = self.per_column(chunk.width(), |i| {
2950 if self.map_columns.binary_search_by_key(&i, |m| m.idx).is_ok() {
2951 return Ok(None);
2952 }
2953 column_to_arrow(&chunk.columns()[i], 0, 0..length, self.compat_level).map(Some)
2954 })?;
2955 Ok(SplitChunk {
2956 chunk,
2957 converted,
2958 start: 0,
2959 })
2960 }
2961}
2962
2963impl Iterator for RecordBatchIter<'_> {
2964 type Item = PolarsResult<RecordBatch>;
2965
2966 fn next(&mut self) -> Option<Self::Item> {
2967 if let Some(split) = &self.split {
2968 let height = split.chunk.height();
2969 if split.start < height {
2970 let end = arrow_export_cut(split.start..height, |r| self.fits(&split.chunk, 0, r));
2971 let batch = self.export_piece(split, split.start..end);
2972 self.split.as_mut().unwrap().start = end;
2973 return Some(batch);
2974 }
2975 self.split = None;
2976 }
2977
2978 let (idx, length) = self.cursor.next()?;
2979 let rows = self.offset..self.offset + length;
2980 self.offset += length;
2981 if !self.fits(&self.df, idx, 0..length) {
2982 match self.split(idx, rows) {
2983 Ok(split) => {
2984 self.split = Some(split);
2985 return self.next();
2986 },
2987 Err(err) => return Some(Err(err)),
2988 }
2989 }
2990 Some(self.export(&self.df, idx, rows))
2991 }
2992
2993 fn size_hint(&self) -> (usize, Option<usize>) {
2994 let (chunks, _) = self.cursor.size_hint();
2995 let piece = self
2996 .split
2997 .as_ref()
2998 .is_some_and(|split| split.start < split.chunk.height());
2999 let lower = chunks + piece as usize;
3000 (lower, self.map_columns.is_empty().then_some(lower))
3001 }
3002}
3003
3004pub struct PhysRecordBatchIter<'a> {
3005 df: &'a DataFrame,
3006 schema: ArrowSchemaRef,
3007 cursor: ChunkCursor,
3008 offset: usize,
3010}
3011
3012impl Iterator for PhysRecordBatchIter<'_> {
3013 type Item = RecordBatch;
3014
3015 fn next(&mut self) -> Option<Self::Item> {
3016 let (idx, length) = self.cursor.next()?;
3017 let offset = self.offset;
3018 self.offset += length;
3019
3020 let arrs = self
3021 .df
3022 .columns()
3023 .iter()
3024 .map(|c| match c.as_series() {
3025 Some(s) => s.chunks()[idx].clone(),
3026 None => c
3027 .slice((offset, length))
3028 .take_materialized_series()
3029 .rechunk()
3030 .chunks()[0]
3031 .clone(),
3032 })
3033 .collect::<Vec<_>>();
3034
3035 Some(RecordBatch::new(length, self.schema.clone(), arrs))
3036 }
3037
3038 fn size_hint(&self) -> (usize, Option<usize>) {
3039 self.cursor.size_hint()
3040 }
3041}
3042
3043pub enum RecordBatchIterWrap<'a> {
3044 ZeroWidth {
3045 remaining_height: usize,
3046 chunk_size: usize,
3047 },
3048 Batches(RecordBatchIter<'a>),
3049 PhysicalBatches(PhysRecordBatchIter<'a>),
3050}
3051
3052impl<'a> RecordBatchIterWrap<'a> {
3053 fn new(df: Cow<'a, DataFrame>, compat_level: CompatLevel, parallel: bool) -> Self {
3054 debug_assert!(!df.should_rechunk(), "expected equal chunks");
3055
3056 if df.width() == 0 {
3057 return Self::new_zero_width(df.height());
3058 }
3059
3060 Self::Batches(RecordBatchIter::new(df, compat_level, parallel))
3061 }
3062
3063 fn new_zero_width(height: usize) -> Self {
3064 Self::ZeroWidth {
3065 remaining_height: height,
3066 chunk_size: polars_config::config().ideal_morsel_size() as usize,
3067 }
3068 }
3069}
3070
3071impl Iterator for RecordBatchIterWrap<'_> {
3072 type Item = PolarsResult<RecordBatch>;
3073
3074 fn next(&mut self) -> Option<Self::Item> {
3075 match self {
3076 Self::ZeroWidth {
3077 remaining_height,
3078 chunk_size,
3079 } => {
3080 let n = usize::min(*remaining_height, *chunk_size);
3081 *remaining_height -= n;
3082
3083 (n > 0).then(|| Ok(RecordBatch::new(n, ArrowSchemaRef::default(), vec![])))
3084 },
3085 Self::Batches(v) => v.next(),
3086 Self::PhysicalBatches(v) => v.next().map(Ok),
3087 }
3088 }
3089
3090 fn size_hint(&self) -> (usize, Option<usize>) {
3091 match self {
3092 Self::ZeroWidth {
3093 remaining_height,
3094 chunk_size,
3095 } => {
3096 let n = remaining_height.div_ceil(*chunk_size);
3097 (n, Some(n))
3098 },
3099 Self::Batches(v) => v.size_hint(),
3100 Self::PhysicalBatches(v) => v.size_hint(),
3101 }
3102 }
3103}
3104
3105fn ensure_can_extend(left: &Column, right: &Column) -> PolarsResult<()> {
3107 polars_ensure!(
3108 left.name() == right.name(),
3109 ShapeMismatch: "unable to vstack, column names don't match: {:?} and {:?}",
3110 left.name(), right.name(),
3111 );
3112 Ok(())
3113}
3114
3115#[cfg(test)]
3116mod test {
3117 use super::*;
3118
3119 fn create_frame() -> DataFrame {
3120 let s0 = Column::new("days".into(), [0, 1, 2].as_ref());
3121 let s1 = Column::new("temp".into(), [22.1, 19.9, 7.].as_ref());
3122 DataFrame::new_infer_height(vec![s0, s1]).unwrap()
3123 }
3124
3125 #[test]
3126 fn sort_in_place_keeps_the_chunks_of_a_frame_sorted_by_one_key() {
3127 let mut df = df!("a" => [1, 2], "b" => [1, 1]).unwrap();
3128 df.vstack_mut(&df!("a" => [3, 4], "b" => [1, 1]).unwrap())
3129 .unwrap();
3130 df.apply("a", |c| {
3131 let mut c = c.clone();
3132 c.set_sorted_flag(IsSorted::Ascending);
3133 c
3134 })
3135 .unwrap();
3136 assert_eq!(df.first_col_n_chunks(), 2);
3137
3138 df.sort_in_place(["a"], SortMultipleOptions::default())
3139 .unwrap();
3140 assert_eq!(df.first_col_n_chunks(), 2);
3141
3142 df.sort_in_place(["a", "b"], SortMultipleOptions::default())
3143 .unwrap();
3144 assert_eq!(df.first_col_n_chunks(), 1);
3145 let a = df.column("a").unwrap().as_materialized_series();
3146 assert_eq!(
3147 a.i32().unwrap().to_vec(),
3148 [Some(1), Some(2), Some(3), Some(4)]
3149 );
3150 }
3151
3152 #[test]
3153 #[cfg_attr(miri, ignore)]
3154 fn test_recordbatch_iterator() {
3155 let df = df!(
3156 "foo" => [1, 2, 3, 4, 5]
3157 )
3158 .unwrap();
3159 let mut iter = df.iter_chunks(CompatLevel::newest(), false);
3160 assert_eq!(5, iter.next().unwrap().len());
3161 assert!(iter.next().is_none());
3162 }
3163
3164 #[test]
3165 #[cfg_attr(miri, ignore)]
3166 fn test_select() {
3167 let df = create_frame();
3168 assert_eq!(
3169 df.column("days")
3170 .unwrap()
3171 .as_series()
3172 .unwrap()
3173 .equal(1)
3174 .unwrap()
3175 .sum(),
3176 Some(1)
3177 );
3178 }
3179
3180 #[test]
3181 #[cfg_attr(miri, ignore)]
3182 fn test_filter_broadcast_on_string_col() {
3183 let col_name = "some_col";
3184 let v = vec!["test".to_string()];
3185 let s0 = Column::new(PlSmallStr::from_str(col_name), v);
3186 let mut df = DataFrame::new_infer_height(vec![s0]).unwrap();
3187
3188 df = df
3189 .filter(
3190 &df.column(col_name)
3191 .unwrap()
3192 .as_materialized_series()
3193 .equal("")
3194 .unwrap(),
3195 )
3196 .unwrap();
3197 assert_eq!(
3198 df.column(col_name)
3199 .unwrap()
3200 .as_materialized_series()
3201 .n_chunks(),
3202 1
3203 );
3204 }
3205
3206 #[test]
3207 #[cfg_attr(miri, ignore)]
3208 fn test_filter_broadcast_on_list_col() {
3209 let s1 = Series::new(PlSmallStr::EMPTY, [true, false, true]);
3210 let ll: ListChunked = [&s1].iter().copied().collect();
3211
3212 let mask = BooleanChunked::from_slice(PlSmallStr::EMPTY, &[false]);
3213 let new = ll.filter(&mask).unwrap();
3214
3215 assert_eq!(new.chunks.len(), 1);
3216 assert_eq!(new.len(), 0);
3217 }
3218
3219 #[test]
3220 fn slice() {
3221 let df = create_frame();
3222 let sliced_df = df.slice(Slice::head(2));
3223 assert_eq!(sliced_df.shape(), (2, 2));
3224 }
3225
3226 #[test]
3227 fn rechunk_false() {
3228 let df = create_frame();
3229 assert!(!df.should_rechunk())
3230 }
3231
3232 #[test]
3233 fn rechunk_true() -> PolarsResult<()> {
3234 let mut base = df!(
3235 "a" => [1, 2, 3],
3236 "b" => [1, 2, 3]
3237 )?;
3238
3239 let mut s = Series::new("foo".into(), 0..2);
3241 let s2 = Series::new("bar".into(), 0..1);
3242 s.append(&s2)?;
3243
3244 let out = base.with_column(s.into_column())?;
3246
3247 assert!(out.should_rechunk());
3249 Ok(())
3250 }
3251
3252 #[test]
3253 fn test_duplicate_column() {
3254 let mut df = df! {
3255 "foo" => [1, 2, 3]
3256 }
3257 .unwrap();
3258 assert!(
3260 df.with_column(Column::new("foo".into(), &[1, 2, 3]))
3261 .is_ok()
3262 );
3263 assert!(
3264 df.with_column(Column::new("bar".into(), &[1, 2, 3]))
3265 .is_ok()
3266 );
3267 assert!(df.column("bar").is_ok())
3268 }
3269
3270 #[test]
3271 #[cfg_attr(miri, ignore)]
3272 fn distinct() {
3273 let df = df! {
3274 "flt" => [1., 1., 2., 2., 3., 3.],
3275 "int" => [1, 1, 2, 2, 3, 3, ],
3276 "str" => ["a", "a", "b", "b", "c", "c"]
3277 }
3278 .unwrap();
3279 let df = df
3280 .unique_stable(None, UniqueKeepStrategy::First, None)
3281 .unwrap()
3282 .sort(["flt"], SortMultipleOptions::default())
3283 .unwrap();
3284 let valid = df! {
3285 "flt" => [1., 2., 3.],
3286 "int" => [1, 2, 3],
3287 "str" => ["a", "b", "c"]
3288 }
3289 .unwrap();
3290 assert!(df.equals(&valid));
3291 }
3292
3293 #[test]
3294 fn test_vstack() {
3295 let mut df = df! {
3297 "flt" => [1., 1., 2., 2., 3., 3.],
3298 "int" => [1, 1, 2, 2, 3, 3, ],
3299 "str" => ["a", "a", "b", "b", "c", "c"]
3300 }
3301 .unwrap();
3302
3303 df.vstack_mut(&df.slice(Slice::head(3))).unwrap();
3304 assert_eq!(df.first_col_n_chunks(), 2)
3305 }
3306
3307 #[test]
3308 fn test_chunk_alignment_with_scalar_column() {
3309 let mut df = df! { "int" => [0, 1] }.unwrap();
3310 df.vstack_mut(&df! { "int" => [2] }.unwrap()).unwrap();
3311 df.insert_column(
3312 0,
3313 Column::new_scalar("str".into(), Scalar::from(PlSmallStr::from_static("a")), 3),
3314 )
3315 .unwrap();
3316
3317 for materialize in [false, true] {
3319 if materialize {
3320 df.columns()[0].as_materialized_series();
3321 }
3322 assert!(!df.should_rechunk());
3323
3324 let batch_lengths = |batches: Vec<RecordBatch>| {
3325 batches
3326 .iter()
3327 .map(|b| {
3328 assert!(b.arrays().iter().all(|arr| arr.len() == b.len()));
3329 b.len()
3330 })
3331 .collect::<Vec<_>>()
3332 };
3333 let batches = df.iter_chunks(CompatLevel::newest(), false).collect();
3334 assert_eq!(batch_lengths(batches), [2, 1]);
3335 let batches = df.iter_chunks_physical().collect();
3336 assert_eq!(batch_lengths(batches), [2, 1]);
3337 }
3338 }
3339
3340 #[test]
3341 fn test_vstack_scalar_chunk_alignment() {
3342 let mut df = df! {
3344 "int" => [0],
3345 "str" => ["a"],
3346 }
3347 .unwrap();
3348 df.vstack_mut(&df! { "int" => [1], "str" => ["a"] }.unwrap())
3349 .unwrap();
3350
3351 assert_eq!(df.column("int").unwrap().n_chunks(), 2);
3352 assert!(df.column("str").unwrap().as_scalar_column().is_some());
3353 assert!(!df.should_rechunk());
3354
3355 let batches = df
3356 .iter_chunks(CompatLevel::newest(), false)
3357 .collect::<Vec<_>>();
3358 assert_eq!(batches.iter().map(|b| b.len()).collect::<Vec<_>>(), [1, 1]);
3359 }
3360
3361 #[test]
3362 fn test_vstack_on_empty_dataframe() {
3363 let mut df = DataFrame::empty();
3364
3365 let df_data = df! {
3366 "flt" => [1., 1., 2., 2., 3., 3.],
3367 "int" => [1, 1, 2, 2, 3, 3, ],
3368 "str" => ["a", "a", "b", "b", "c", "c"]
3369 }
3370 .unwrap();
3371
3372 df.vstack_mut(&df_data).unwrap();
3373 assert_eq!(df.height(), 6)
3374 }
3375
3376 #[test]
3377 fn test_unique_keep_none_with_slice() {
3378 let df = df! {
3379 "x" => [1, 2, 3, 2, 1]
3380 }
3381 .unwrap();
3382 let out = df
3383 .unique_stable(
3384 Some(&["x".to_string()][..]),
3385 UniqueKeepStrategy::None,
3386 Some(Slice::head(2)),
3387 )
3388 .unwrap();
3389 let expected = df! {
3390 "x" => [3]
3391 }
3392 .unwrap();
3393 assert!(out.equals(&expected));
3394 }
3395
3396 #[test]
3397 #[cfg(feature = "dtype-i8")]
3398 fn test_apply_result_schema() {
3399 let mut df = df! {
3400 "x" => [1, 2, 3, 2, 1]
3401 }
3402 .unwrap();
3403
3404 let schema_before = df.schema().clone();
3405 df.apply("x", |f| f.cast(&DataType::Int8).unwrap()).unwrap();
3406 assert_ne!(&schema_before, df.schema());
3407 }
3408}