Skip to main content

polars_core/frame/row/
av_buffer.rs

1#![deny(unsafe_op_in_unsafe_fn)]
2
3use std::hint::unreachable_unchecked;
4
5use polars_arrow::array::builder::ShareStrategy;
6use polars_arrow::bitmap::BitmapBuilder;
7#[cfg(feature = "dtype-decimal")]
8use polars_compute::decimal::DecimalFmtBuffer;
9#[cfg(feature = "dtype-struct")]
10use polars_utils::pl_str::PlSmallStr;
11
12use super::*;
13use crate::chunked_array::builder::NullChunkedBuilder;
14use crate::chunked_array::flags::StatisticsFlags;
15#[cfg(feature = "dtype-struct")]
16use crate::prelude::any_value::arr_to_any_value;
17use crate::series::builder::SeriesBuilder;
18
19#[derive(Clone)]
20pub enum AnyValueBuffer<'a> {
21    Boolean(BooleanChunkedBuilder),
22    #[cfg(feature = "dtype-i8")]
23    Int8(PrimitiveChunkedBuilder<Int8Type>),
24    #[cfg(feature = "dtype-i16")]
25    Int16(PrimitiveChunkedBuilder<Int16Type>),
26    Int32(PrimitiveChunkedBuilder<Int32Type>),
27    Int64(PrimitiveChunkedBuilder<Int64Type>),
28    #[cfg(feature = "dtype-u8")]
29    UInt8(PrimitiveChunkedBuilder<UInt8Type>),
30    #[cfg(feature = "dtype-u16")]
31    UInt16(PrimitiveChunkedBuilder<UInt16Type>),
32    UInt32(PrimitiveChunkedBuilder<UInt32Type>),
33    UInt64(PrimitiveChunkedBuilder<UInt64Type>),
34    #[cfg(feature = "dtype-date")]
35    Date(PrimitiveChunkedBuilder<Int32Type>),
36    #[cfg(feature = "dtype-datetime")]
37    Datetime(
38        PrimitiveChunkedBuilder<Int64Type>,
39        TimeUnit,
40        Option<TimeZone>,
41    ),
42    #[cfg(feature = "dtype-duration")]
43    Duration(PrimitiveChunkedBuilder<Int64Type>, TimeUnit),
44    #[cfg(feature = "dtype-time")]
45    Time(PrimitiveChunkedBuilder<Int64Type>),
46    Float32(PrimitiveChunkedBuilder<Float32Type>),
47    Float64(PrimitiveChunkedBuilder<Float64Type>),
48    String(StringChunkedBuilder),
49    Null(NullChunkedBuilder),
50    All(DataType, Vec<AnyValue<'a>>),
51}
52
53impl<'a> AnyValueBuffer<'a> {
54    #[inline]
55    pub fn add(&mut self, val: AnyValue<'_>) -> Option<()> {
56        use AnyValueBuffer::*;
57        match (self, val) {
58            (Boolean(builder), AnyValue::Null) => builder.append_null(),
59            (Boolean(builder), AnyValue::Boolean(v)) => builder.append_value(v),
60            (Boolean(builder), val) => builder.append_value(val.extract_bool()?),
61            (Int32(builder), AnyValue::Null) => builder.append_null(),
62            (Int32(builder), val) => builder.append_value(val.extract()?),
63            (Int64(builder), AnyValue::Null) => builder.append_null(),
64            (Int64(builder), val) => builder.append_value(val.extract()?),
65            (UInt32(builder), AnyValue::Null) => builder.append_null(),
66            (UInt32(builder), val) => builder.append_value(val.extract()?),
67            (UInt64(builder), AnyValue::Null) => builder.append_null(),
68            (UInt64(builder), val) => builder.append_value(val.extract()?),
69            (Float32(builder), AnyValue::Null) => builder.append_null(),
70            (Float64(builder), AnyValue::Null) => builder.append_null(),
71            (Float32(builder), val) => builder.append_value(val.extract()?),
72            (Float64(builder), val) => builder.append_value(val.extract()?),
73            (String(builder), AnyValue::String(v)) => builder.append_value(v),
74            (String(builder), AnyValue::StringOwned(v)) => builder.append_value(v.as_str()),
75            (String(builder), AnyValue::Null) => builder.append_null(),
76            #[cfg(feature = "dtype-i8")]
77            (Int8(builder), AnyValue::Null) => builder.append_null(),
78            #[cfg(feature = "dtype-i8")]
79            (Int8(builder), val) => builder.append_value(val.extract()?),
80            #[cfg(feature = "dtype-i16")]
81            (Int16(builder), AnyValue::Null) => builder.append_null(),
82            #[cfg(feature = "dtype-i16")]
83            (Int16(builder), val) => builder.append_value(val.extract()?),
84            #[cfg(feature = "dtype-u8")]
85            (UInt8(builder), AnyValue::Null) => builder.append_null(),
86            #[cfg(feature = "dtype-u8")]
87            (UInt8(builder), val) => builder.append_value(val.extract()?),
88            #[cfg(feature = "dtype-u16")]
89            (UInt16(builder), AnyValue::Null) => builder.append_null(),
90            #[cfg(feature = "dtype-u16")]
91            (UInt16(builder), val) => builder.append_value(val.extract()?),
92            #[cfg(feature = "dtype-date")]
93            (Date(builder), AnyValue::Null) => builder.append_null(),
94            #[cfg(feature = "dtype-date")]
95            (Date(builder), AnyValue::Date(v)) => builder.append_value(v),
96            #[cfg(feature = "dtype-date")]
97            (Date(builder), val) if val.is_primitive_numeric() => {
98                builder.append_value(val.extract()?)
99            },
100            #[cfg(feature = "dtype-datetime")]
101            (Datetime(builder, _, _), AnyValue::Null) => builder.append_null(),
102            #[cfg(feature = "dtype-datetime")]
103            (
104                Datetime(builder, tu_l, _),
105                AnyValue::Datetime(v, tu_r, _) | AnyValue::DatetimeOwned(v, tu_r, _),
106            ) => {
107                // we convert right tu to left tu
108                // so we swap.
109                let v = crate::datatypes::time_unit::convert_time_units(v, tu_r, *tu_l);
110                builder.append_value(v)
111            },
112            #[cfg(feature = "dtype-datetime")]
113            (Datetime(builder, _, _), val) if val.is_primitive_numeric() => {
114                builder.append_value(val.extract()?)
115            },
116            #[cfg(feature = "dtype-duration")]
117            (Duration(builder, _), AnyValue::Null) => builder.append_null(),
118            #[cfg(feature = "dtype-duration")]
119            (Duration(builder, tu_l), AnyValue::Duration(v, tu_r)) => {
120                let v = crate::datatypes::time_unit::convert_time_units(v, tu_r, *tu_l);
121                builder.append_value(v)
122            },
123            #[cfg(feature = "dtype-duration")]
124            (Duration(builder, _), val) if val.is_primitive_numeric() => {
125                builder.append_value(val.extract()?)
126            },
127            #[cfg(feature = "dtype-time")]
128            (Time(builder), AnyValue::Time(v)) => builder.append_value(v),
129            #[cfg(feature = "dtype-time")]
130            (Time(builder), AnyValue::Null) => builder.append_null(),
131            #[cfg(feature = "dtype-time")]
132            (Time(builder), val) if val.is_primitive_numeric() => {
133                builder.append_value(val.extract()?)
134            },
135            (Null(builder), AnyValue::Null) => builder.append_null(),
136            // Struct and List can be recursive so use AnyValues for that
137            (All(_, vals), v) => vals.push(v.into_static()),
138
139            // dynamic types
140            (String(builder), av) => match av {
141                AnyValue::Int64(v) => builder.append_value(format!("{v}")),
142                AnyValue::Float64(v) => builder.append_value(format!("{v}")),
143                AnyValue::Boolean(true) => builder.append_value("true"),
144                AnyValue::Boolean(false) => builder.append_value("false"),
145                #[cfg(feature = "dtype-decimal")]
146                AnyValue::Decimal(v, _p, s) => {
147                    let mut fmt = DecimalFmtBuffer::new();
148                    builder.append_value(fmt.format_dec128(v, s, false, false));
149                },
150                _ => return None,
151            },
152            _ => return None,
153        };
154        Some(())
155    }
156
157    pub fn add_fallible(&mut self, val: &AnyValue<'a>) -> PolarsResult<()> {
158        self.add(val.as_borrowed()).ok_or_else(|| {
159            polars_err!(
160                ComputeError: "could not append value: {} of type: {} to the builder; make sure that all rows \
161                have the same schema or consider increasing `infer_schema_length`\n\
162                \n\
163                it might also be that a value overflows the data-type's capacity", val, val.dtype()
164            )
165        })
166    }
167
168    pub fn reset(&mut self, capacity: usize, strict: bool) -> PolarsResult<Series> {
169        use AnyValueBuffer::*;
170        let out = match self {
171            Boolean(b) => {
172                let mut new = BooleanChunkedBuilder::new(b.field.name().clone(), capacity);
173                std::mem::swap(&mut new, b);
174                new.finish().into_series()
175            },
176            Int32(b) => {
177                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
178                std::mem::swap(&mut new, b);
179                new.finish().into_series()
180            },
181            Int64(b) => {
182                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
183                std::mem::swap(&mut new, b);
184                new.finish().into_series()
185            },
186            UInt32(b) => {
187                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
188                std::mem::swap(&mut new, b);
189                new.finish().into_series()
190            },
191            UInt64(b) => {
192                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
193                std::mem::swap(&mut new, b);
194                new.finish().into_series()
195            },
196            #[cfg(feature = "dtype-date")]
197            Date(b) => {
198                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
199                std::mem::swap(&mut new, b);
200                new.finish().into_date().into_series()
201            },
202            #[cfg(feature = "dtype-datetime")]
203            Datetime(b, tu, tz) => {
204                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
205                std::mem::swap(&mut new, b);
206                let tz = if capacity > 0 {
207                    tz.clone()
208                } else {
209                    std::mem::take(tz)
210                };
211                new.finish().into_datetime(*tu, tz).into_series()
212            },
213            #[cfg(feature = "dtype-duration")]
214            Duration(b, tu) => {
215                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
216                std::mem::swap(&mut new, b);
217                new.finish().into_duration(*tu).into_series()
218            },
219            #[cfg(feature = "dtype-time")]
220            Time(b) => {
221                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
222                std::mem::swap(&mut new, b);
223                new.finish().into_time().into_series()
224            },
225            Float32(b) => {
226                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
227                std::mem::swap(&mut new, b);
228                new.finish().into_series()
229            },
230            Float64(b) => {
231                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
232                std::mem::swap(&mut new, b);
233                new.finish().into_series()
234            },
235            String(b) => {
236                let mut new = StringChunkedBuilder::new(b.field.name().clone(), capacity);
237                std::mem::swap(&mut new, b);
238                new.finish().into_series()
239            },
240            #[cfg(feature = "dtype-i8")]
241            Int8(b) => {
242                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
243                std::mem::swap(&mut new, b);
244                new.finish().into_series()
245            },
246            #[cfg(feature = "dtype-i16")]
247            Int16(b) => {
248                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
249                std::mem::swap(&mut new, b);
250                new.finish().into_series()
251            },
252            #[cfg(feature = "dtype-u8")]
253            UInt8(b) => {
254                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
255                std::mem::swap(&mut new, b);
256                new.finish().into_series()
257            },
258            #[cfg(feature = "dtype-u16")]
259            UInt16(b) => {
260                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
261                std::mem::swap(&mut new, b);
262                new.finish().into_series()
263            },
264            Null(b) => {
265                let mut new = NullChunkedBuilder::new(b.field.name().clone(), 0);
266                std::mem::swap(&mut new, b);
267                new.finish().into_series()
268            },
269            All(dtype, vals) => {
270                let out =
271                    Series::from_any_values_and_dtype(PlSmallStr::EMPTY, vals, dtype, strict)?;
272                let mut new = Vec::with_capacity(capacity);
273                std::mem::swap(&mut new, vals);
274                out
275            },
276        };
277        Ok(out)
278    }
279
280    pub fn into_series(mut self) -> PolarsResult<Series> {
281        self.reset(0, false)
282    }
283
284    pub fn new(dtype: &DataType, capacity: usize) -> AnyValueBuffer<'a> {
285        (dtype, capacity).into()
286    }
287}
288
289// datatype and length
290impl From<(&DataType, usize)> for AnyValueBuffer<'_> {
291    fn from(a: (&DataType, usize)) -> Self {
292        let (dt, len) = a;
293        use DataType::*;
294        match dt {
295            Boolean => AnyValueBuffer::Boolean(BooleanChunkedBuilder::new(PlSmallStr::EMPTY, len)),
296            Int32 => AnyValueBuffer::Int32(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
297            Int64 => AnyValueBuffer::Int64(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
298            UInt32 => AnyValueBuffer::UInt32(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
299            UInt64 => AnyValueBuffer::UInt64(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
300            #[cfg(feature = "dtype-i8")]
301            Int8 => AnyValueBuffer::Int8(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
302            #[cfg(feature = "dtype-i16")]
303            Int16 => AnyValueBuffer::Int16(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
304            #[cfg(feature = "dtype-u8")]
305            UInt8 => AnyValueBuffer::UInt8(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
306            #[cfg(feature = "dtype-u16")]
307            UInt16 => AnyValueBuffer::UInt16(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
308            #[cfg(feature = "dtype-date")]
309            Date => AnyValueBuffer::Date(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
310            #[cfg(feature = "dtype-datetime")]
311            Datetime(tu, tz) => AnyValueBuffer::Datetime(
312                PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len),
313                *tu,
314                tz.clone(),
315            ),
316            #[cfg(feature = "dtype-duration")]
317            Duration(tu) => {
318                AnyValueBuffer::Duration(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len), *tu)
319            },
320            #[cfg(feature = "dtype-time")]
321            Time => AnyValueBuffer::Time(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len)),
322            Float32 => {
323                AnyValueBuffer::Float32(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
324            },
325            Float64 => {
326                AnyValueBuffer::Float64(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
327            },
328            String => AnyValueBuffer::String(StringChunkedBuilder::new(PlSmallStr::EMPTY, len)),
329            Null => AnyValueBuffer::Null(NullChunkedBuilder::new(PlSmallStr::EMPTY, 0)),
330            // Struct and List can be recursive so use AnyValues for that
331            dt => AnyValueBuffer::All(dt.clone(), Vec::with_capacity(len)),
332        }
333    }
334}
335
336/// An [`AnyValueBuffer`] that converts nested values in batches (as each value retains
337/// a [`Series`] until finally converted, this can prevent excessive peak memory use).
338pub struct AnyValueBufferBatched<'a> {
339    values: AnyValueBuffer<'a>,
340    converted: Option<SeriesBuilder>,
341    fast_explode: bool,
342}
343
344impl<'a> AnyValueBufferBatched<'a> {
345    const BATCH_SIZE: usize = 1024;
346
347    pub fn new(dtype: &DataType, capacity: usize) -> Self {
348        if dtype.is_nested() && Self::can_batch(dtype) {
349            let mut converted = SeriesBuilder::new(dtype.clone());
350            converted.reserve(capacity);
351            Self {
352                values: AnyValueBuffer::new(dtype, capacity.min(Self::BATCH_SIZE)),
353                converted: Some(converted),
354                fast_explode: dtype.is_list(),
355            }
356        } else {
357            Self {
358                values: AnyValueBuffer::new(dtype, capacity),
359                converted: None,
360                fast_explode: false,
361            }
362        }
363    }
364
365    /// Whether values can convert in batches.
366    fn can_batch(dtype: &DataType) -> bool {
367        match dtype {
368            DataType::List(inner) => Self::can_batch(inner),
369            #[cfg(feature = "dtype-array")]
370            DataType::Array(inner, _) => Self::can_batch(inner),
371            #[cfg(feature = "dtype-struct")]
372            DataType::Struct(fields) => fields.iter().all(|f| Self::can_batch(f.dtype())),
373            _ => {
374                !(dtype.is_nested()
375                    || dtype.is_extension()
376                    || dtype.is_null()
377                    || dtype.is_unknown()
378                    || dtype.is_object())
379            },
380        }
381    }
382
383    #[inline]
384    pub fn add(&mut self, val: AnyValue<'_>) -> Option<()> {
385        self.values.add(val)
386    }
387
388    pub fn add_fallible(&mut self, val: &AnyValue<'a>) -> PolarsResult<()> {
389        self.values.add_fallible(val)
390    }
391
392    /// Whether a full batch of nested values is ready to [`flush`](Self::flush).
393    #[inline]
394    pub fn needs_flush(&self) -> bool {
395        self.converted.is_some()
396            && matches!(&self.values, AnyValueBuffer::All(_, vals) if vals.len() == Self::BATCH_SIZE)
397    }
398
399    /// Converts the buffered nested values, extending the converted [`SeriesBuilder`].
400    /// Frees the memory used by the buffered values.
401    pub fn flush(&mut self) -> PolarsResult<()> {
402        if let Some(converted) = &mut self.converted {
403            let batch = self.values.reset(Self::BATCH_SIZE, false)?;
404            self.fast_explode &= batch.is_empty() || batch.get_flags().can_fast_explode_list();
405            converted.extend(&batch, ShareStrategy::Never);
406        }
407        Ok(())
408    }
409
410    pub fn into_series(mut self) -> PolarsResult<Series> {
411        if self.converted.as_ref().is_none_or(SeriesBuilder::is_empty) {
412            return self.values.into_series();
413        }
414        self.flush()?;
415        let mut series = self.converted.unwrap().freeze(PlSmallStr::EMPTY);
416        if self.fast_explode {
417            // `freeze` drops the flags set by the batches' list builders; put them back
418            series.set_flags(series.get_flags() | StatisticsFlags::CAN_FAST_EXPLODE_LIST);
419        }
420        Ok(series)
421    }
422
423    /// Convert into a column of `height` rows.
424    pub fn into_column(self, name: PlSmallStr, height: usize) -> PolarsResult<Column> {
425        let series = self.into_series()?;
426        Ok(if series.is_empty() {
427            Column::full_null(name, height, series.dtype())
428        } else {
429            series.with_name(name).into_column()
430        })
431    }
432}
433
434/// An [`AnyValueBuffer`] that should be used when we trust the builder.
435#[derive(Clone)]
436pub enum AnyValueBufferTrusted<'a> {
437    Boolean(BooleanChunkedBuilder),
438    #[cfg(feature = "dtype-i8")]
439    Int8(PrimitiveChunkedBuilder<Int8Type>),
440    #[cfg(feature = "dtype-i16")]
441    Int16(PrimitiveChunkedBuilder<Int16Type>),
442    Int32(PrimitiveChunkedBuilder<Int32Type>),
443    Int64(PrimitiveChunkedBuilder<Int64Type>),
444    #[cfg(feature = "dtype-u8")]
445    UInt8(PrimitiveChunkedBuilder<UInt8Type>),
446    #[cfg(feature = "dtype-u16")]
447    UInt16(PrimitiveChunkedBuilder<UInt16Type>),
448    UInt32(PrimitiveChunkedBuilder<UInt32Type>),
449    UInt64(PrimitiveChunkedBuilder<UInt64Type>),
450    Float32(PrimitiveChunkedBuilder<Float32Type>),
451    Float64(PrimitiveChunkedBuilder<Float64Type>),
452    String(StringChunkedBuilder),
453    #[cfg(feature = "dtype-struct")]
454    // not the trusted variant!
455    Struct(BitmapBuilder, Vec<(AnyValueBuffer<'a>, PlSmallStr)>),
456    Null(NullChunkedBuilder),
457    All(DataType, Vec<AnyValue<'a>>),
458}
459
460impl<'a> AnyValueBufferTrusted<'a> {
461    pub fn new(dtype: &DataType, len: usize) -> Self {
462        (dtype, len).into()
463    }
464
465    #[inline]
466    fn add_null(&mut self) {
467        use AnyValueBufferTrusted::*;
468        match self {
469            Boolean(builder) => builder.append_null(),
470            #[cfg(feature = "dtype-i8")]
471            Int8(builder) => builder.append_null(),
472            #[cfg(feature = "dtype-i16")]
473            Int16(builder) => builder.append_null(),
474            Int32(builder) => builder.append_null(),
475            Int64(builder) => builder.append_null(),
476            #[cfg(feature = "dtype-u8")]
477            UInt8(builder) => builder.append_null(),
478            #[cfg(feature = "dtype-u16")]
479            UInt16(builder) => builder.append_null(),
480            UInt32(builder) => builder.append_null(),
481            UInt64(builder) => builder.append_null(),
482            Float32(builder) => builder.append_null(),
483            Float64(builder) => builder.append_null(),
484            String(builder) => builder.append_null(),
485            #[cfg(feature = "dtype-struct")]
486            Struct(outer_validity, builders) => {
487                outer_validity.push(false);
488                for (b, _) in builders.iter_mut() {
489                    b.add(AnyValue::Null);
490                }
491            },
492            Null(builder) => builder.append_null(),
493            All(_, vals) => vals.push(AnyValue::Null),
494        }
495    }
496
497    /// # Safety
498    /// The caller must ensure that the [`AnyValue`] type exactly matches the `Buffer` type.
499    #[inline]
500    unsafe fn add_physical(&mut self, val: &AnyValue<'_>) {
501        // SAFETY: All unsafe blocks rely directly on the function contract.
502
503        use AnyValueBufferTrusted::*;
504        match self {
505            Boolean(builder) => {
506                let AnyValue::Boolean(v) = val else {
507                    unsafe { unreachable_unchecked() }
508                };
509                builder.append_value(*v)
510            },
511            #[cfg(feature = "dtype-i8")]
512            Int8(builder) => {
513                let AnyValue::Int8(v) = val else {
514                    unsafe { unreachable_unchecked() }
515                };
516                builder.append_value(*v)
517            },
518            #[cfg(feature = "dtype-i16")]
519            Int16(builder) => {
520                let AnyValue::Int16(v) = val else {
521                    unsafe { unreachable_unchecked() }
522                };
523                builder.append_value(*v)
524            },
525            Int32(builder) => {
526                let AnyValue::Int32(v) = val else {
527                    unsafe { unreachable_unchecked() }
528                };
529                builder.append_value(*v)
530            },
531            Int64(builder) => {
532                let AnyValue::Int64(v) = val else {
533                    unsafe { unreachable_unchecked() }
534                };
535                builder.append_value(*v)
536            },
537            #[cfg(feature = "dtype-u8")]
538            UInt8(builder) => {
539                let AnyValue::UInt8(v) = val else {
540                    unsafe { unreachable_unchecked() }
541                };
542                builder.append_value(*v)
543            },
544            #[cfg(feature = "dtype-u16")]
545            UInt16(builder) => {
546                let AnyValue::UInt16(v) = val else {
547                    unsafe { unreachable_unchecked() }
548                };
549                builder.append_value(*v)
550            },
551            UInt32(builder) => {
552                let AnyValue::UInt32(v) = val else {
553                    unsafe { unreachable_unchecked() }
554                };
555                builder.append_value(*v)
556            },
557            UInt64(builder) => {
558                let AnyValue::UInt64(v) = val else {
559                    unsafe { unreachable_unchecked() }
560                };
561                builder.append_value(*v)
562            },
563            Float32(builder) => {
564                let AnyValue::Float32(v) = val else {
565                    unsafe { unreachable_unchecked() }
566                };
567                builder.append_value(*v)
568            },
569            Float64(builder) => {
570                let AnyValue::Float64(v) = val else {
571                    unsafe { unreachable_unchecked() }
572                };
573                builder.append_value(*v)
574            },
575            Null(builder) => {
576                let AnyValue::Null = val else {
577                    unsafe { unreachable_unchecked() }
578                };
579                builder.append_null()
580            },
581            _ => unreachable!(),
582        }
583    }
584
585    /// Will add the [`AnyValue`] into [`Self`] and unpack as the physical type belonging to
586    /// [`Self`]. This should only be used with physical buffers
587    ///
588    /// If a type is not primitive or String, the AnyValues will be converted to static
589    ///
590    /// # Safety
591    /// The caller must ensure that the [`AnyValue`] type exactly matches the `Buffer` type and is
592    /// owned.
593    #[inline]
594    pub unsafe fn add_unchecked_owned_physical(&mut self, val: &AnyValue<'a>) {
595        use AnyValueBufferTrusted::*;
596        match val {
597            AnyValue::Null => self.add_null(),
598            _ => {
599                match self {
600                    String(builder) => {
601                        let AnyValue::StringOwned(v) = val else {
602                            // SAFETY: Function contract.
603                            unsafe { unreachable_unchecked() }
604                        };
605                        builder.append_value(v.as_str())
606                    },
607                    #[cfg(feature = "dtype-struct")]
608                    Struct(outer_validity, builders) => {
609                        let AnyValue::StructOwned(payload) = val else {
610                            // SAFETY: Function contract.
611                            unsafe { unreachable_unchecked() }
612                        };
613                        let avs = &*payload.0;
614
615                        debug_assert_eq!(builders.len(), avs.len());
616                        for ((builder, _), av) in builders.iter_mut().zip(avs.iter().cloned()) {
617                            builder.add(av);
618                        }
619                        outer_validity.push(true);
620                    },
621                    All(_, vals) => vals.push(val.clone().into_static()),
622                    // SAFETY: Function contract.
623                    _ => unsafe { self.add_physical(val) },
624                }
625            },
626        }
627    }
628
629    /// # Safety
630    /// The caller must ensure that the [`AnyValue`] type exactly matches the `Buffer` type and is
631    /// borrowed and if `val` is a `AnyValue::Struct` that the values are internally consistent.
632    #[inline]
633    pub unsafe fn add_unchecked_borrowed_physical(&mut self, val: &AnyValue<'a>) {
634        use AnyValueBufferTrusted::*;
635        match val {
636            AnyValue::Null => self.add_null(),
637            _ => {
638                match self {
639                    String(builder) => {
640                        let AnyValue::String(v) = val else {
641                            unsafe { unreachable_unchecked() }
642                        };
643                        builder.append_value(v)
644                    },
645                    #[cfg(feature = "dtype-struct")]
646                    Struct(outer_validity, builders) => {
647                        let AnyValue::Struct(idx, arr, fields) = *val else {
648                            // SAFETY: Function contract.
649                            unsafe { unreachable_unchecked() }
650                        };
651                        let arrays = arr.values();
652                        debug_assert_eq!(builders.len(), arrays.len());
653                        debug_assert_eq!(fields.len(), arrays.len());
654                        for ((field, array), (builder, _)) in
655                            fields.iter().zip(arrays).zip(builders.iter_mut())
656                        {
657                            // SAFETY: The values inside `val` need to be consistent, `idx` MUST be
658                            // in-bounds for all `arr.values()` and `field.dtype` correct.
659                            let av_new = unsafe { arr_to_any_value(&**array, idx, &field.dtype) };
660                            builder.add(av_new);
661                        }
662                        outer_validity.push(true);
663                    },
664                    All(_, vals) => vals.push(val.clone().into_static()),
665                    // SAFETY: Function contract.
666                    _ => unsafe { self.add_physical(val) },
667                }
668            },
669        }
670    }
671
672    /// Clear `self` and give `capacity`, returning the old contents as a [`Series`].
673    pub fn reset(&mut self, capacity: usize, strict: bool) -> PolarsResult<Series> {
674        use AnyValueBufferTrusted::*;
675        let out = match self {
676            Boolean(b) => {
677                let mut new = BooleanChunkedBuilder::new(b.field.name().clone(), capacity);
678                std::mem::swap(&mut new, b);
679                new.finish().into_series()
680            },
681            Int32(b) => {
682                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
683                std::mem::swap(&mut new, b);
684                new.finish().into_series()
685            },
686            Int64(b) => {
687                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
688                std::mem::swap(&mut new, b);
689                new.finish().into_series()
690            },
691            UInt32(b) => {
692                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
693                std::mem::swap(&mut new, b);
694                new.finish().into_series()
695            },
696            UInt64(b) => {
697                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
698                std::mem::swap(&mut new, b);
699                new.finish().into_series()
700            },
701            Float32(b) => {
702                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
703                std::mem::swap(&mut new, b);
704                new.finish().into_series()
705            },
706            Float64(b) => {
707                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
708                std::mem::swap(&mut new, b);
709                new.finish().into_series()
710            },
711            String(b) => {
712                let mut new = StringChunkedBuilder::new(b.field.name().clone(), capacity);
713                std::mem::swap(&mut new, b);
714                new.finish().into_series()
715            },
716            #[cfg(feature = "dtype-i8")]
717            Int8(b) => {
718                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
719                std::mem::swap(&mut new, b);
720                new.finish().into_series()
721            },
722            #[cfg(feature = "dtype-i16")]
723            Int16(b) => {
724                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
725                std::mem::swap(&mut new, b);
726                new.finish().into_series()
727            },
728            #[cfg(feature = "dtype-u8")]
729            UInt8(b) => {
730                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
731                std::mem::swap(&mut new, b);
732                new.finish().into_series()
733            },
734            #[cfg(feature = "dtype-u16")]
735            UInt16(b) => {
736                let mut new = PrimitiveChunkedBuilder::new(b.field.name().clone(), capacity);
737                std::mem::swap(&mut new, b);
738                new.finish().into_series()
739            },
740            #[cfg(feature = "dtype-struct")]
741            Struct(outer_validity, b) => {
742                // Without fields, the outer validity has the length.
743                if b.is_empty() {
744                    let old_outer_validity = core::mem::take(outer_validity);
745                    outer_validity.reserve(capacity);
746                    let length = old_outer_validity.len();
747                    return Ok(
748                        StructChunked::from_series(PlSmallStr::EMPTY, length, [].iter())?
749                            .with_outer_validity(Some(old_outer_validity.freeze()))
750                            .into_series(),
751                    );
752                }
753
754                let mut min_len = usize::MAX;
755                let mut max_len = usize::MIN;
756
757                let v = b
758                    .iter_mut()
759                    .map(|(b, name)| {
760                        let mut s = b.reset(capacity, strict)?;
761
762                        min_len = min_len.min(s.len());
763                        max_len = max_len.max(s.len());
764
765                        s.rename(name.clone());
766                        Ok(s)
767                    })
768                    .collect::<PolarsResult<Vec<_>>>()?;
769
770                let length = if min_len == 0 { 0 } else { max_len };
771
772                let old_outer_validity = core::mem::take(outer_validity);
773                outer_validity.reserve(capacity);
774
775                StructChunked::from_series(PlSmallStr::EMPTY, length, v.iter())?
776                    .with_outer_validity(Some(old_outer_validity.freeze()))
777                    .into_series()
778            },
779            Null(b) => {
780                let mut new = NullChunkedBuilder::new(b.field.name().clone(), 0);
781                std::mem::swap(&mut new, b);
782                new.finish().into_series()
783            },
784            All(dtype, vals) => {
785                let mut swap_vals = Vec::with_capacity(capacity);
786                std::mem::swap(vals, &mut swap_vals);
787                Series::from_any_values_and_dtype(PlSmallStr::EMPTY, &swap_vals, dtype, false)?
788            },
789        };
790
791        Ok(out)
792    }
793
794    pub fn into_series(mut self) -> Series {
795        // unwrap: non-strict does not error.
796        self.reset(0, false).unwrap()
797    }
798}
799
800impl From<(&DataType, usize)> for AnyValueBufferTrusted<'_> {
801    fn from(a: (&DataType, usize)) -> Self {
802        let (dt, len) = a;
803        use DataType::*;
804        match dt {
805            Boolean => {
806                AnyValueBufferTrusted::Boolean(BooleanChunkedBuilder::new(PlSmallStr::EMPTY, len))
807            },
808            Int32 => {
809                AnyValueBufferTrusted::Int32(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
810            },
811            Int64 => {
812                AnyValueBufferTrusted::Int64(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
813            },
814            UInt32 => {
815                AnyValueBufferTrusted::UInt32(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
816            },
817            UInt64 => {
818                AnyValueBufferTrusted::UInt64(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
819            },
820            #[cfg(feature = "dtype-i8")]
821            Int8 => {
822                AnyValueBufferTrusted::Int8(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
823            },
824            #[cfg(feature = "dtype-i16")]
825            Int16 => {
826                AnyValueBufferTrusted::Int16(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
827            },
828            #[cfg(feature = "dtype-u8")]
829            UInt8 => {
830                AnyValueBufferTrusted::UInt8(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
831            },
832            #[cfg(feature = "dtype-u16")]
833            UInt16 => {
834                AnyValueBufferTrusted::UInt16(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
835            },
836            Float32 => {
837                AnyValueBufferTrusted::Float32(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
838            },
839            Float64 => {
840                AnyValueBufferTrusted::Float64(PrimitiveChunkedBuilder::new(PlSmallStr::EMPTY, len))
841            },
842            String => {
843                AnyValueBufferTrusted::String(StringChunkedBuilder::new(PlSmallStr::EMPTY, len))
844            },
845            #[cfg(feature = "dtype-struct")]
846            Struct(fields) => {
847                let outer_validity = BitmapBuilder::with_capacity(len);
848                let buffers = fields
849                    .iter()
850                    .map(|field| {
851                        let dtype = field.dtype().to_physical();
852                        let buffer: AnyValueBuffer = (&dtype, len).into();
853                        (buffer, field.name.clone())
854                    })
855                    .collect::<Vec<_>>();
856                AnyValueBufferTrusted::Struct(outer_validity, buffers)
857            },
858            // List can be recursive so use AnyValues for that
859            dt => AnyValueBufferTrusted::All(dt.clone(), Vec::with_capacity(len)),
860        }
861    }
862}