Skip to main content

polars_core/frame/
chunks.rs

1use polars_arrow::record_batch::RecordBatch;
2use rayon::prelude::*;
3
4use crate::prelude::*;
5use crate::runtime::RAYON;
6use crate::utils::{_split_offsets, accumulate_dataframes_vertical_unchecked, split_df_as_ref};
7
8impl From<RecordBatch> for DataFrame {
9    fn from(rb: RecordBatch) -> DataFrame {
10        let height = rb.height();
11        let (schema, arrays) = rb.into_schema_and_arrays();
12
13        let columns: Vec<Column> = arrays
14            .into_iter()
15            .zip(schema.iter())
16            .map(|(arr, (name, field))| {
17                // SAFETY: Record Batch has the invariant that the schema datatype matches the
18                // columns.
19                unsafe {
20                    Series::_try_from_arrow_unchecked_with_md(
21                        name.clone(),
22                        vec![arr],
23                        field.dtype(),
24                        field.metadata.as_deref(),
25                    )
26                }
27                .unwrap()
28                .into_column()
29            })
30            .collect();
31
32        // SAFETY: RecordBatch has the same invariants for names and heights as DataFrame.
33        unsafe { DataFrame::new_unchecked(height, columns) }
34    }
35}
36
37impl DataFrame {
38    pub fn split_chunks(&mut self) -> impl Iterator<Item = DataFrame> + '_ {
39        self.align_chunks_par();
40
41        // Follows the first column that has chunks (a series, or a dictionary column's codes);
42        // scalar columns have none and are sliced.
43        let chunk_lengths: Vec<usize> = self.first_col_chunk_lengths().collect();
44        let mut prev_height = 0;
45        (0..chunk_lengths.len()).map(move |i| unsafe {
46            let chunk_size = chunk_lengths[i];
47            let columns = self
48                .columns()
49                .iter()
50                .map(|col| match col {
51                    Column::Series(s) => Column::from(s.select_chunk(i)),
52                    Column::Scalar(_) | Column::Dictionary(_) => {
53                        col.slice((prev_height, chunk_size))
54                    },
55                })
56                .collect::<Vec<_>>();
57
58            prev_height += chunk_size;
59
60            DataFrame::new_unchecked(chunk_size, columns)
61        })
62    }
63
64    pub fn split_chunks_by_n(self, n: usize, parallel: bool) -> Vec<DataFrame> {
65        let split = _split_offsets(self.height(), n);
66
67        let split_fn = |(offset, len)| self.slice((offset, len));
68
69        if parallel {
70            // Parallel so that null_counts run in parallel
71            RAYON.install(|| split.into_par_iter().map(split_fn).collect())
72        } else {
73            split.into_iter().map(split_fn).collect()
74        }
75    }
76
77    /// Convert the columns of this [DataFrame] to arrow arrays.
78    pub fn rechunk_to_arrow(&self, compat_level: CompatLevel) -> Vec<ArrayRef> {
79        self.columns()
80            .iter()
81            .map(|c| c.clone().rechunk_to_arrow(compat_level))
82            .collect()
83    }
84
85    /// Convert the columns of this [DataFrame] to arrow arrays.
86    pub fn rechunk_into_arrow(self, compat_level: CompatLevel) -> Vec<ArrayRef> {
87        self.into_columns()
88            .into_iter()
89            .map(|c| c.rechunk_to_arrow(compat_level))
90            .collect()
91    }
92}
93
94/// Split DataFrame into chunks in preparation for writing. The chunks have a
95/// maximum number of rows per chunk to ensure reasonable memory efficiency when
96/// reading the resulting file, and a minimum size per chunk to ensure
97/// reasonable performance when writing.
98pub fn chunk_df_for_writing(
99    df: &mut DataFrame,
100    row_group_size: usize,
101) -> PolarsResult<std::borrow::Cow<'_, DataFrame>> {
102    // ensures all chunks are aligned.
103    df.align_chunks_par();
104
105    // Accumulate many small chunks to the row group size.
106    // See: #16403
107    if !df.columns().is_empty()
108        && df
109            .first_col_chunk_lengths()
110            .take(5)
111            .all(|len| len < row_group_size)
112    {
113        fn finish(scratch: &mut Vec<DataFrame>, new_chunks: &mut Vec<DataFrame>) {
114            let mut new = accumulate_dataframes_vertical_unchecked(scratch.drain(..));
115            new.rechunk_mut_par();
116            new_chunks.push(new);
117        }
118
119        let mut new_chunks = Vec::with_capacity(df.first_col_n_chunks()); // upper limit;
120        let mut scratch = vec![];
121        let mut remaining = row_group_size;
122
123        for df in df.split_chunks() {
124            remaining = remaining.saturating_sub(df.height());
125            scratch.push(df);
126
127            if remaining == 0 {
128                remaining = row_group_size;
129                finish(&mut scratch, &mut new_chunks);
130            }
131        }
132        if !scratch.is_empty() {
133            finish(&mut scratch, &mut new_chunks);
134        }
135        return Ok(std::borrow::Cow::Owned(
136            accumulate_dataframes_vertical_unchecked(new_chunks),
137        ));
138    }
139
140    let n_splits = df.height() / row_group_size;
141    let result = if n_splits > 0 {
142        let mut splits = split_df_as_ref(df, n_splits, false);
143
144        for df in splits.iter_mut() {
145            // If the chunks are small enough, writing many small chunks
146            // leads to slow writing performance, so in that case we
147            // merge them.
148            let n_chunks = df.first_col_n_chunks();
149            if n_chunks > 1 && (df.estimated_size(true) / n_chunks < 128 * 1024) {
150                df.rechunk_mut_par();
151            }
152        }
153
154        std::borrow::Cow::Owned(accumulate_dataframes_vertical_unchecked(splits))
155    } else {
156        std::borrow::Cow::Borrowed(df)
157    };
158    Ok(result)
159}