Skip to main content

polars_core/frame/
builder.rs

1use std::sync::Arc;
2
3use polars_arrow::array::builder::ShareStrategy;
4use polars_utils::IdxSize;
5
6use crate::frame::DataFrame;
7use crate::prelude::*;
8use crate::schema::Schema;
9use crate::series::builder::SeriesBuilder;
10
11pub struct DataFrameBuilder {
12    schema: Arc<Schema>,
13    builders: Vec<SeriesBuilder>,
14    height: usize,
15}
16
17impl DataFrameBuilder {
18    pub fn new(schema: Arc<Schema>) -> Self {
19        let builders = schema
20            .iter_values()
21            .map(|dt| SeriesBuilder::new(dt.clone()))
22            .collect();
23        Self {
24            schema,
25            builders,
26            height: 0,
27        }
28    }
29
30    pub fn reserve(&mut self, additional: usize) {
31        for builder in &mut self.builders {
32            builder.reserve(additional);
33        }
34    }
35
36    pub fn freeze(self) -> DataFrame {
37        let columns = self
38            .schema
39            .iter_names()
40            .zip(self.builders)
41            .map(|(n, b)| {
42                let s = b.freeze(n.clone());
43                assert_eq!(s.len(), self.height);
44                Column::from(s)
45            })
46            .collect();
47
48        // SAFETY: we checked the lengths and the names are unique because they
49        // come from Schema.
50        unsafe { DataFrame::new_unchecked(self.height, columns) }
51    }
52
53    pub fn freeze_reset(&mut self) -> DataFrame {
54        let columns = self
55            .schema
56            .iter_names()
57            .zip(&mut self.builders)
58            .map(|(n, b)| {
59                let s = b.freeze_reset(n.clone());
60                assert!(s.len() == self.height);
61                Column::from(s)
62            })
63            .collect();
64
65        // SAFETY: we checked the lengths and the names are unique because they
66        // come from Schema.
67        let out = unsafe { DataFrame::new_unchecked(self.height, columns) };
68        self.height = 0;
69        out
70    }
71
72    pub fn len(&self) -> usize {
73        self.height
74    }
75
76    pub fn is_empty(&self) -> bool {
77        self.height == 0
78    }
79
80    /// Extends this builder with the contents of the given dataframe. May panic
81    /// if other does not match the schema of this builder.
82    pub fn extend(&mut self, other: &DataFrame, share: ShareStrategy) {
83        self.subslice_extend(other, 0, other.height(), share);
84    }
85
86    /// Extends this builder with the contents of the given dataframe subslice.
87    /// May panic if other does not match the schema of this builder.
88    pub fn subslice_extend(
89        &mut self,
90        other: &DataFrame,
91        start: usize,
92        length: usize,
93        share: ShareStrategy,
94    ) {
95        let columns = other.columns();
96        assert!(self.builders.len() == columns.len());
97        for (builder, column) in self.builders.iter_mut().zip(columns) {
98            match column {
99                // Only the rows taken are materialized, not the whole dictionary column.
100                Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
101                    let s = d.slice((start, length)).take_materialized_series();
102                    builder.subslice_extend(&s, 0, s.len(), share);
103                },
104                Column::Series(_) | Column::Dictionary(_) => {
105                    let s = column.as_materialized_series();
106                    builder.subslice_extend(s, start, length, share);
107                },
108                Column::Scalar(sc) => {
109                    let len = sc.len().saturating_sub(start).min(length);
110                    let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
111                    builder.subslice_extend_each_repeated(&scalar_as_series, 0, 1, len, share);
112                },
113            }
114        }
115
116        self.height += length.min(other.height().saturating_sub(start));
117    }
118
119    /// Extends this builder with the contents of the given dataframe subslice, repeating it `repeats` times.
120    /// May panic if other does not match the schema of this builder.
121    pub fn subslice_extend_repeated(
122        &mut self,
123        other: &DataFrame,
124        start: usize,
125        length: usize,
126        repeats: usize,
127        share: ShareStrategy,
128    ) {
129        let columns = other.columns();
130        assert!(self.builders.len() == columns.len());
131        for (builder, column) in self.builders.iter_mut().zip(columns) {
132            match column {
133                Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
134                    let s = d.slice((start, length)).take_materialized_series();
135                    builder.subslice_extend_repeated(&s, 0, s.len(), repeats, share);
136                },
137                Column::Series(_) | Column::Dictionary(_) => {
138                    let s = column.as_materialized_series();
139                    builder.subslice_extend_repeated(s, start, length, repeats, share);
140                },
141                Column::Scalar(sc) => {
142                    let len = sc.len().saturating_sub(start).min(length);
143                    let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
144                    builder.subslice_extend_each_repeated(
145                        &scalar_as_series,
146                        0,
147                        1,
148                        len * repeats,
149                        share,
150                    );
151                },
152            }
153        }
154
155        self.height += length.min(other.height().saturating_sub(start)) * repeats;
156    }
157
158    /// Extends this builder with the contents of the given dataframe subslice.
159    /// Each element is repeated repeats times. May panic if other does not
160    /// match the schema of this builder.
161    pub fn subslice_extend_each_repeated(
162        &mut self,
163        other: &DataFrame,
164        start: usize,
165        length: usize,
166        repeats: usize,
167        share: ShareStrategy,
168    ) {
169        let columns = other.columns();
170        assert!(self.builders.len() == columns.len());
171        for (builder, column) in self.builders.iter_mut().zip(columns) {
172            match column {
173                Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
174                    let s = d.slice((start, length)).take_materialized_series();
175                    builder.subslice_extend_each_repeated(&s, 0, s.len(), repeats, share);
176                },
177                Column::Series(_) | Column::Dictionary(_) => {
178                    let s = column.as_materialized_series();
179                    builder.subslice_extend_each_repeated(s, start, length, repeats, share);
180                },
181                Column::Scalar(sc) => {
182                    let len = sc.len().saturating_sub(start).min(length);
183                    let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
184                    builder.subslice_extend_each_repeated(
185                        &scalar_as_series,
186                        0,
187                        1,
188                        len * repeats,
189                        share,
190                    );
191                },
192            }
193        }
194
195        self.height += length.min(other.height().saturating_sub(start)) * repeats;
196    }
197
198    /// Extends this builder with the contents of the given dataframe at the given
199    /// indices. That is, `other[idxs[i]]` is appended to this builder in order,
200    /// for each i=0..idxs.len(). May panic if other does not match the schema
201    /// of this builder, or if the other dataframe is not rechunked.
202    ///
203    /// # Safety
204    /// The indices must be in-bounds.
205    pub unsafe fn gather_extend(
206        &mut self,
207        other: &DataFrame,
208        idxs: &[IdxSize],
209        share: ShareStrategy,
210    ) {
211        let columns = other.columns();
212        assert!(self.builders.len() == columns.len());
213        for (builder, column) in self.builders.iter_mut().zip(columns) {
214            match column {
215                Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
216                    // SAFETY: the indices are in bounds.
217                    let s = unsafe { d.take_slice_unchecked(idxs) }.take_materialized_series();
218                    builder.subslice_extend(&s, 0, s.len(), share);
219                },
220                Column::Series(_) | Column::Dictionary(_) => {
221                    let s = column.as_materialized_series();
222                    builder.gather_extend(s, idxs, share);
223                },
224                Column::Scalar(sc) => {
225                    let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
226                    builder.subslice_extend_each_repeated(
227                        &scalar_as_series,
228                        0,
229                        1,
230                        idxs.len(),
231                        share,
232                    );
233                },
234            }
235        }
236
237        self.height += idxs.len();
238    }
239
240    /// Extends this builder with the contents of the given dataframe at the given
241    /// indices. That is, `other[idxs[i]]` is appended to this builder in order,
242    /// for each i=0..idxs.len(). Out-of-bounds indices extend with nulls.
243    /// May panic if other does not match the schema of this builder, or if the
244    /// other dataframe is not rechunked.
245    pub fn opt_gather_extend(&mut self, other: &DataFrame, idxs: &[IdxSize], share: ShareStrategy) {
246        let mut trans_idxs = Vec::new();
247        let columns = other.columns();
248        assert!(self.builders.len() == columns.len());
249        for (builder, column) in self.builders.iter_mut().zip(columns) {
250            match column {
251                Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
252                    // Out-of-bounds indices become null indices, which gather nulls.
253                    let idxs = idxs
254                        .iter()
255                        .map(|i| ((*i as usize) < d.len()).then_some(*i))
256                        .collect::<IdxCa>();
257                    // SAFETY: the valid indices are in bounds.
258                    let s = unsafe { d.take_unchecked(&idxs) }.take_materialized_series();
259                    builder.subslice_extend(&s, 0, s.len(), share);
260                },
261                Column::Series(_) | Column::Dictionary(_) => {
262                    let s = column.as_materialized_series();
263                    builder.opt_gather_extend(s, idxs, share);
264                },
265                Column::Scalar(sc) => {
266                    let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
267                    // Reduce call overhead by transforming indices to 0/1 and dispatching to
268                    // opt_gather_extend on the scalar as series.
269                    for idx_chunk in idxs.chunks(4096) {
270                        trans_idxs.clear();
271                        trans_idxs.extend(
272                            idx_chunk
273                                .iter()
274                                .map(|idx| ((*idx as usize) >= sc.len()) as IdxSize),
275                        );
276                        builder.opt_gather_extend(&scalar_as_series, &trans_idxs, share);
277                    }
278                },
279            }
280        }
281
282        self.height += idxs.len();
283    }
284}