polars_core/frame/
chunks.rs1use polars_arrow::record_batch::RecordBatch;
2use rayon::prelude::*;
3
4use crate::prelude::*;
5use crate::runtime::RAYON;
6use crate::utils::{_split_offsets, accumulate_dataframes_vertical_unchecked, split_df_as_ref};
7
8impl From<RecordBatch> for DataFrame {
9 fn from(rb: RecordBatch) -> DataFrame {
10 let height = rb.height();
11 let (schema, arrays) = rb.into_schema_and_arrays();
12
13 let columns: Vec<Column> = arrays
14 .into_iter()
15 .zip(schema.iter())
16 .map(|(arr, (name, field))| {
17 unsafe {
20 Series::_try_from_arrow_unchecked_with_md(
21 name.clone(),
22 vec![arr],
23 field.dtype(),
24 field.metadata.as_deref(),
25 )
26 }
27 .unwrap()
28 .into_column()
29 })
30 .collect();
31
32 unsafe { DataFrame::new_unchecked(height, columns) }
34 }
35}
36
37impl DataFrame {
38 pub fn split_chunks(&mut self) -> impl Iterator<Item = DataFrame> + '_ {
39 self.align_chunks_par();
40
41 let chunk_lengths: Vec<usize> = self.first_col_chunk_lengths().collect();
44 let mut prev_height = 0;
45 (0..chunk_lengths.len()).map(move |i| unsafe {
46 let chunk_size = chunk_lengths[i];
47 let columns = self
48 .columns()
49 .iter()
50 .map(|col| match col {
51 Column::Series(s) => Column::from(s.select_chunk(i)),
52 Column::Scalar(_) | Column::Dictionary(_) => {
53 col.slice((prev_height, chunk_size))
54 },
55 })
56 .collect::<Vec<_>>();
57
58 prev_height += chunk_size;
59
60 DataFrame::new_unchecked(chunk_size, columns)
61 })
62 }
63
64 pub fn split_chunks_by_n(self, n: usize, parallel: bool) -> Vec<DataFrame> {
65 let split = _split_offsets(self.height(), n);
66
67 let split_fn = |(offset, len)| self.slice((offset, len));
68
69 if parallel {
70 RAYON.install(|| split.into_par_iter().map(split_fn).collect())
72 } else {
73 split.into_iter().map(split_fn).collect()
74 }
75 }
76
77 pub fn rechunk_to_arrow(&self, compat_level: CompatLevel) -> Vec<ArrayRef> {
79 self.columns()
80 .iter()
81 .map(|c| c.clone().rechunk_to_arrow(compat_level))
82 .collect()
83 }
84
85 pub fn rechunk_into_arrow(self, compat_level: CompatLevel) -> Vec<ArrayRef> {
87 self.into_columns()
88 .into_iter()
89 .map(|c| c.rechunk_to_arrow(compat_level))
90 .collect()
91 }
92}
93
94pub fn chunk_df_for_writing(
99 df: &mut DataFrame,
100 row_group_size: usize,
101) -> PolarsResult<std::borrow::Cow<'_, DataFrame>> {
102 df.align_chunks_par();
104
105 if !df.columns().is_empty()
108 && df
109 .first_col_chunk_lengths()
110 .take(5)
111 .all(|len| len < row_group_size)
112 {
113 fn finish(scratch: &mut Vec<DataFrame>, new_chunks: &mut Vec<DataFrame>) {
114 let mut new = accumulate_dataframes_vertical_unchecked(scratch.drain(..));
115 new.rechunk_mut_par();
116 new_chunks.push(new);
117 }
118
119 let mut new_chunks = Vec::with_capacity(df.first_col_n_chunks()); let mut scratch = vec![];
121 let mut remaining = row_group_size;
122
123 for df in df.split_chunks() {
124 remaining = remaining.saturating_sub(df.height());
125 scratch.push(df);
126
127 if remaining == 0 {
128 remaining = row_group_size;
129 finish(&mut scratch, &mut new_chunks);
130 }
131 }
132 if !scratch.is_empty() {
133 finish(&mut scratch, &mut new_chunks);
134 }
135 return Ok(std::borrow::Cow::Owned(
136 accumulate_dataframes_vertical_unchecked(new_chunks),
137 ));
138 }
139
140 let n_splits = df.height() / row_group_size;
141 let result = if n_splits > 0 {
142 let mut splits = split_df_as_ref(df, n_splits, false);
143
144 for df in splits.iter_mut() {
145 let n_chunks = df.first_col_n_chunks();
149 if n_chunks > 1 && (df.estimated_size(true) / n_chunks < 128 * 1024) {
150 df.rechunk_mut_par();
151 }
152 }
153
154 std::borrow::Cow::Owned(accumulate_dataframes_vertical_unchecked(splits))
155 } else {
156 std::borrow::Cow::Borrowed(df)
157 };
158 Ok(result)
159}