1use std::sync::Arc;
2
3use polars_arrow::array::builder::ShareStrategy;
4use polars_utils::IdxSize;
5
6use crate::frame::DataFrame;
7use crate::prelude::*;
8use crate::schema::Schema;
9use crate::series::builder::SeriesBuilder;
10
11pub struct DataFrameBuilder {
12 schema: Arc<Schema>,
13 builders: Vec<SeriesBuilder>,
14 height: usize,
15}
16
17impl DataFrameBuilder {
18 pub fn new(schema: Arc<Schema>) -> Self {
19 let builders = schema
20 .iter_values()
21 .map(|dt| SeriesBuilder::new(dt.clone()))
22 .collect();
23 Self {
24 schema,
25 builders,
26 height: 0,
27 }
28 }
29
30 pub fn reserve(&mut self, additional: usize) {
31 for builder in &mut self.builders {
32 builder.reserve(additional);
33 }
34 }
35
36 pub fn freeze(self) -> DataFrame {
37 let columns = self
38 .schema
39 .iter_names()
40 .zip(self.builders)
41 .map(|(n, b)| {
42 let s = b.freeze(n.clone());
43 assert_eq!(s.len(), self.height);
44 Column::from(s)
45 })
46 .collect();
47
48 unsafe { DataFrame::new_unchecked(self.height, columns) }
51 }
52
53 pub fn freeze_reset(&mut self) -> DataFrame {
54 let columns = self
55 .schema
56 .iter_names()
57 .zip(&mut self.builders)
58 .map(|(n, b)| {
59 let s = b.freeze_reset(n.clone());
60 assert!(s.len() == self.height);
61 Column::from(s)
62 })
63 .collect();
64
65 let out = unsafe { DataFrame::new_unchecked(self.height, columns) };
68 self.height = 0;
69 out
70 }
71
72 pub fn len(&self) -> usize {
73 self.height
74 }
75
76 pub fn is_empty(&self) -> bool {
77 self.height == 0
78 }
79
80 pub fn extend(&mut self, other: &DataFrame, share: ShareStrategy) {
83 self.subslice_extend(other, 0, other.height(), share);
84 }
85
86 pub fn subslice_extend(
89 &mut self,
90 other: &DataFrame,
91 start: usize,
92 length: usize,
93 share: ShareStrategy,
94 ) {
95 let columns = other.columns();
96 assert!(self.builders.len() == columns.len());
97 for (builder, column) in self.builders.iter_mut().zip(columns) {
98 match column {
99 Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
101 let s = d.slice((start, length)).take_materialized_series();
102 builder.subslice_extend(&s, 0, s.len(), share);
103 },
104 Column::Series(_) | Column::Dictionary(_) => {
105 let s = column.as_materialized_series();
106 builder.subslice_extend(s, start, length, share);
107 },
108 Column::Scalar(sc) => {
109 let len = sc.len().saturating_sub(start).min(length);
110 let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
111 builder.subslice_extend_each_repeated(&scalar_as_series, 0, 1, len, share);
112 },
113 }
114 }
115
116 self.height += length.min(other.height().saturating_sub(start));
117 }
118
119 pub fn subslice_extend_repeated(
122 &mut self,
123 other: &DataFrame,
124 start: usize,
125 length: usize,
126 repeats: usize,
127 share: ShareStrategy,
128 ) {
129 let columns = other.columns();
130 assert!(self.builders.len() == columns.len());
131 for (builder, column) in self.builders.iter_mut().zip(columns) {
132 match column {
133 Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
134 let s = d.slice((start, length)).take_materialized_series();
135 builder.subslice_extend_repeated(&s, 0, s.len(), repeats, share);
136 },
137 Column::Series(_) | Column::Dictionary(_) => {
138 let s = column.as_materialized_series();
139 builder.subslice_extend_repeated(s, start, length, repeats, share);
140 },
141 Column::Scalar(sc) => {
142 let len = sc.len().saturating_sub(start).min(length);
143 let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
144 builder.subslice_extend_each_repeated(
145 &scalar_as_series,
146 0,
147 1,
148 len * repeats,
149 share,
150 );
151 },
152 }
153 }
154
155 self.height += length.min(other.height().saturating_sub(start)) * repeats;
156 }
157
158 pub fn subslice_extend_each_repeated(
162 &mut self,
163 other: &DataFrame,
164 start: usize,
165 length: usize,
166 repeats: usize,
167 share: ShareStrategy,
168 ) {
169 let columns = other.columns();
170 assert!(self.builders.len() == columns.len());
171 for (builder, column) in self.builders.iter_mut().zip(columns) {
172 match column {
173 Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
174 let s = d.slice((start, length)).take_materialized_series();
175 builder.subslice_extend_each_repeated(&s, 0, s.len(), repeats, share);
176 },
177 Column::Series(_) | Column::Dictionary(_) => {
178 let s = column.as_materialized_series();
179 builder.subslice_extend_each_repeated(s, start, length, repeats, share);
180 },
181 Column::Scalar(sc) => {
182 let len = sc.len().saturating_sub(start).min(length);
183 let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
184 builder.subslice_extend_each_repeated(
185 &scalar_as_series,
186 0,
187 1,
188 len * repeats,
189 share,
190 );
191 },
192 }
193 }
194
195 self.height += length.min(other.height().saturating_sub(start)) * repeats;
196 }
197
198 pub unsafe fn gather_extend(
206 &mut self,
207 other: &DataFrame,
208 idxs: &[IdxSize],
209 share: ShareStrategy,
210 ) {
211 let columns = other.columns();
212 assert!(self.builders.len() == columns.len());
213 for (builder, column) in self.builders.iter_mut().zip(columns) {
214 match column {
215 Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
216 let s = unsafe { d.take_slice_unchecked(idxs) }.take_materialized_series();
218 builder.subslice_extend(&s, 0, s.len(), share);
219 },
220 Column::Series(_) | Column::Dictionary(_) => {
221 let s = column.as_materialized_series();
222 builder.gather_extend(s, idxs, share);
223 },
224 Column::Scalar(sc) => {
225 let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
226 builder.subslice_extend_each_repeated(
227 &scalar_as_series,
228 0,
229 1,
230 idxs.len(),
231 share,
232 );
233 },
234 }
235 }
236
237 self.height += idxs.len();
238 }
239
240 pub fn opt_gather_extend(&mut self, other: &DataFrame, idxs: &[IdxSize], share: ShareStrategy) {
246 let mut trans_idxs = Vec::new();
247 let columns = other.columns();
248 assert!(self.builders.len() == columns.len());
249 for (builder, column) in self.builders.iter_mut().zip(columns) {
250 match column {
251 Column::Dictionary(d) if d.lazy_as_materialized_series().is_none() => {
252 let idxs = idxs
254 .iter()
255 .map(|i| ((*i as usize) < d.len()).then_some(*i))
256 .collect::<IdxCa>();
257 let s = unsafe { d.take_unchecked(&idxs) }.take_materialized_series();
259 builder.subslice_extend(&s, 0, s.len(), share);
260 },
261 Column::Series(_) | Column::Dictionary(_) => {
262 let s = column.as_materialized_series();
263 builder.opt_gather_extend(s, idxs, share);
264 },
265 Column::Scalar(sc) => {
266 let scalar_as_series = sc.scalar().clone().into_series(PlSmallStr::default());
267 for idx_chunk in idxs.chunks(4096) {
270 trans_idxs.clear();
271 trans_idxs.extend(
272 idx_chunk
273 .iter()
274 .map(|idx| ((*idx as usize) >= sc.len()) as IdxSize),
275 );
276 builder.opt_gather_extend(&scalar_as_series, &trans_idxs, share);
277 }
278 },
279 }
280 }
281
282 self.height += idxs.len();
283 }
284}