polars_io/csv/read/
builder.rs

1use arrow::array::MutableBinaryViewArray;
2#[cfg(feature = "dtype-decimal")]
3use polars_compute::decimal::str_to_dec128;
4#[cfg(feature = "dtype-categorical")]
5use polars_core::chunked_array::builder::CategoricalChunkedBuilder;
6use polars_core::prelude::*;
7use polars_error::to_compute_err;
8#[cfg(any(feature = "dtype-datetime", feature = "dtype-date"))]
9use polars_time::chunkedarray::string::Pattern;
10#[cfg(any(feature = "dtype-datetime", feature = "dtype-date"))]
11use polars_time::prelude::string::infer::{
12    DatetimeInfer, StrpTimeParser, TryFromWithUnit, infer_pattern_single,
13};
14#[cfg(feature = "dtype-f16")]
15use polars_utils::float16::pf16;
16use polars_utils::vec::PushUnchecked;
17
18use super::options::CsvEncoding;
19use super::parser::{could_be_whitespace_fast, skip_whitespace};
20use super::utils::escape_field;
21
22pub(crate) trait PrimitiveParser: PolarsNumericType {
23    fn parse(bytes: &[u8]) -> Option<Self::Native>;
24}
25
26#[cfg(feature = "dtype-f16")]
27impl PrimitiveParser for Float16Type {
28    #[inline]
29    fn parse(bytes: &[u8]) -> Option<pf16> {
30        use num_traits::FromPrimitive;
31
32        pf16::from_f32(fast_float2::parse(bytes).ok()?)
33    }
34}
35
36impl PrimitiveParser for Float32Type {
37    #[inline]
38    fn parse(bytes: &[u8]) -> Option<f32> {
39        fast_float2::parse(bytes).ok()
40    }
41}
42impl PrimitiveParser for Float64Type {
43    #[inline]
44    fn parse(bytes: &[u8]) -> Option<f64> {
45        fast_float2::parse(bytes).ok()
46    }
47}
48
49#[cfg(feature = "dtype-u8")]
50impl PrimitiveParser for UInt8Type {
51    #[inline]
52    fn parse(bytes: &[u8]) -> Option<u8> {
53        atoi_simd::parse::<_, true, true>(bytes).ok()
54    }
55}
56#[cfg(feature = "dtype-u16")]
57impl PrimitiveParser for UInt16Type {
58    #[inline]
59    fn parse(bytes: &[u8]) -> Option<u16> {
60        atoi_simd::parse::<_, true, true>(bytes).ok()
61    }
62}
63impl PrimitiveParser for UInt32Type {
64    #[inline]
65    fn parse(bytes: &[u8]) -> Option<u32> {
66        atoi_simd::parse::<_, true, true>(bytes).ok()
67    }
68}
69impl PrimitiveParser for UInt64Type {
70    #[inline]
71    fn parse(bytes: &[u8]) -> Option<u64> {
72        atoi_simd::parse::<_, true, true>(bytes).ok()
73    }
74}
75#[cfg(feature = "dtype-u128")]
76impl PrimitiveParser for UInt128Type {
77    #[inline]
78    fn parse(bytes: &[u8]) -> Option<u128> {
79        atoi_simd::parse::<_, true, true>(bytes).ok()
80    }
81}
82#[cfg(feature = "dtype-i8")]
83impl PrimitiveParser for Int8Type {
84    #[inline]
85    fn parse(bytes: &[u8]) -> Option<i8> {
86        atoi_simd::parse::<_, true, true>(bytes).ok()
87    }
88}
89#[cfg(feature = "dtype-i16")]
90impl PrimitiveParser for Int16Type {
91    #[inline]
92    fn parse(bytes: &[u8]) -> Option<i16> {
93        atoi_simd::parse::<_, true, true>(bytes).ok()
94    }
95}
96impl PrimitiveParser for Int32Type {
97    #[inline]
98    fn parse(bytes: &[u8]) -> Option<i32> {
99        atoi_simd::parse::<_, true, true>(bytes).ok()
100    }
101}
102impl PrimitiveParser for Int64Type {
103    #[inline]
104    fn parse(bytes: &[u8]) -> Option<i64> {
105        atoi_simd::parse::<_, true, true>(bytes).ok()
106    }
107}
108#[cfg(feature = "dtype-i128")]
109impl PrimitiveParser for Int128Type {
110    #[inline]
111    fn parse(bytes: &[u8]) -> Option<i128> {
112        atoi_simd::parse::<_, true, true>(bytes).ok()
113    }
114}
115
116trait ParsedBuilder {
117    fn parse_bytes(
118        &mut self,
119        bytes: &[u8],
120        ignore_errors: bool,
121        _needs_escaping: bool,
122        _missing_is_null: bool,
123        _time_unit: Option<TimeUnit>,
124    ) -> PolarsResult<()>;
125}
126
127impl<T> ParsedBuilder for PrimitiveChunkedBuilder<T>
128where
129    T: PolarsNumericType + PrimitiveParser,
130{
131    #[inline]
132    fn parse_bytes(
133        &mut self,
134        mut bytes: &[u8],
135        ignore_errors: bool,
136        needs_escaping: bool,
137        _missing_is_null: bool,
138        _time_unit: Option<TimeUnit>,
139    ) -> PolarsResult<()> {
140        if !bytes.is_empty() && needs_escaping {
141            bytes = &bytes[1..bytes.len() - 1];
142        }
143
144        if !bytes.is_empty() && could_be_whitespace_fast(bytes[0]) {
145            bytes = skip_whitespace(bytes);
146        }
147
148        if bytes.is_empty() {
149            self.append_null();
150            return Ok(());
151        }
152
153        match T::parse(bytes) {
154            Some(value) => self.append_value(value),
155            None => {
156                if ignore_errors {
157                    self.append_null()
158                } else {
159                    polars_bail!(ComputeError: "invalid primitive value found during CSV parsing")
160                }
161            },
162        }
163        Ok(())
164    }
165}
166
167pub struct Utf8Field {
168    name: PlSmallStr,
169    mutable: MutableBinaryViewArray<[u8]>,
170    scratch: Vec<u8>,
171    quote_char: u8,
172    encoding: CsvEncoding,
173}
174
175impl Utf8Field {
176    fn new(
177        name: PlSmallStr,
178        capacity: usize,
179        quote_char: Option<u8>,
180        encoding: CsvEncoding,
181    ) -> Self {
182        Self {
183            name,
184            mutable: MutableBinaryViewArray::with_capacity(capacity),
185            scratch: vec![],
186            quote_char: quote_char.unwrap_or(b'"'),
187            encoding,
188        }
189    }
190}
191
192#[inline]
193pub fn validate_utf8(bytes: &[u8]) -> bool {
194    simdutf8::basic::from_utf8(bytes).is_ok()
195}
196
197impl ParsedBuilder for Utf8Field {
198    #[inline]
199    fn parse_bytes(
200        &mut self,
201        bytes: &[u8],
202        ignore_errors: bool,
203        needs_escaping: bool,
204        missing_is_null: bool,
205        _time_unit: Option<TimeUnit>,
206    ) -> PolarsResult<()> {
207        if bytes.is_empty() {
208            if missing_is_null {
209                self.mutable.push_null()
210            } else {
211                self.mutable.push(Some([]))
212            }
213            return Ok(());
214        }
215
216        // note that one branch writes without updating the length, so we must do that later.
217        let escaped_bytes = if needs_escaping {
218            self.scratch.clear();
219            self.scratch.reserve(bytes.len());
220            polars_ensure!(bytes.len() > 1 && bytes.last() == Some(&self.quote_char), ComputeError: "invalid csv file\n\nField `{}` is not properly escaped.", std::str::from_utf8(bytes).map_err(to_compute_err)?);
221
222            // SAFETY:
223            // we just allocated enough capacity and data_len is correct.
224            unsafe {
225                let n_written =
226                    escape_field(bytes, self.quote_char, self.scratch.spare_capacity_mut());
227                self.scratch.set_len(n_written);
228            }
229
230            self.scratch.as_slice()
231        } else {
232            bytes
233        };
234
235        if matches!(self.encoding, CsvEncoding::LossyUtf8) | ignore_errors {
236            // It is important that this happens after escaping, as invalid escaped string can produce
237            // invalid utf8.
238            let parse_result = validate_utf8(escaped_bytes);
239
240            match parse_result {
241                true => {
242                    let value = escaped_bytes;
243                    self.mutable.push_value(value)
244                },
245                false => {
246                    if matches!(self.encoding, CsvEncoding::LossyUtf8) {
247                        // TODO! do this without allocating
248                        let s = String::from_utf8_lossy(escaped_bytes);
249                        self.mutable.push_value(s.as_ref().as_bytes())
250                    } else if ignore_errors {
251                        self.mutable.push_null()
252                    } else {
253                        // If field before escaping is valid utf8, the escaping is incorrect.
254                        if needs_escaping && validate_utf8(bytes) {
255                            polars_bail!(ComputeError: "string field is not properly escaped");
256                        } else {
257                            polars_bail!(ComputeError: "invalid utf-8 sequence");
258                        }
259                    }
260                },
261            }
262        } else {
263            self.mutable.push_value(escaped_bytes)
264        }
265
266        Ok(())
267    }
268}
269
270#[cfg(feature = "dtype-categorical")]
271pub struct CategoricalField<T: PolarsCategoricalType> {
272    escape_scratch: Vec<u8>,
273    quote_char: u8,
274    builder: CategoricalChunkedBuilder<T>,
275}
276
277#[cfg(feature = "dtype-categorical")]
278impl<T: PolarsCategoricalType> CategoricalField<T> {
279    fn new(name: PlSmallStr, capacity: usize, quote_char: Option<u8>, dtype: DataType) -> Self {
280        let mut builder = CategoricalChunkedBuilder::new(name, dtype);
281        builder.reserve(capacity);
282
283        Self {
284            escape_scratch: vec![],
285            quote_char: quote_char.unwrap_or(b'"'),
286            builder,
287        }
288    }
289
290    #[inline]
291    fn parse_bytes(
292        &mut self,
293        bytes: &[u8],
294        ignore_errors: bool,
295        needs_escaping: bool,
296        _missing_is_null: bool,
297        _time_unit: Option<TimeUnit>,
298    ) -> PolarsResult<()> {
299        if bytes.is_empty() {
300            self.builder.append_null();
301            return Ok(());
302        }
303        if validate_utf8(bytes) {
304            if needs_escaping {
305                polars_ensure!(bytes.len() > 1, ComputeError: "invalid csv file\n\nField `{}` is not properly escaped.", std::str::from_utf8(bytes).map_err(to_compute_err)?);
306                self.escape_scratch.clear();
307                self.escape_scratch.reserve(bytes.len());
308                // SAFETY:
309                // we just allocated enough capacity and data_len is correct.
310                unsafe {
311                    let n_written = escape_field(
312                        bytes,
313                        self.quote_char,
314                        self.escape_scratch.spare_capacity_mut(),
315                    );
316                    self.escape_scratch.set_len(n_written);
317                }
318
319                // SAFETY:
320                // just did utf8 check
321                let key = unsafe { std::str::from_utf8_unchecked(&self.escape_scratch) };
322                self.builder.append_str(key)?;
323            } else {
324                // SAFETY:
325                // just did utf8 check
326                let key = unsafe { std::str::from_utf8_unchecked(bytes) };
327                self.builder.append_str(key)?;
328            }
329        } else if ignore_errors {
330            self.builder.append_null()
331        } else {
332            polars_bail!(ComputeError: "invalid utf-8 sequence");
333        }
334        Ok(())
335    }
336}
337
338impl ParsedBuilder for BooleanChunkedBuilder {
339    #[inline]
340    fn parse_bytes(
341        &mut self,
342        bytes: &[u8],
343        ignore_errors: bool,
344        needs_escaping: bool,
345        _missing_is_null: bool,
346        _time_unit: Option<TimeUnit>,
347    ) -> PolarsResult<()> {
348        let bytes = if needs_escaping {
349            &bytes[1..bytes.len() - 1]
350        } else {
351            bytes
352        };
353        if bytes.eq_ignore_ascii_case(b"false") {
354            self.append_value(false);
355        } else if bytes.eq_ignore_ascii_case(b"true") {
356            self.append_value(true);
357        } else if ignore_errors || bytes.is_empty() {
358            self.append_null();
359        } else {
360            polars_bail!(
361                ComputeError: "error while parsing value {} as boolean",
362                String::from_utf8_lossy(bytes),
363            );
364        }
365        Ok(())
366    }
367}
368
369#[cfg(feature = "dtype-decimal")]
370pub struct DecimalField {
371    builder: PrimitiveChunkedBuilder<Int128Type>,
372    precision: usize,
373    scale: usize,
374    decimal_comma: bool,
375}
376
377#[cfg(feature = "dtype-decimal")]
378impl DecimalField {
379    fn new(
380        name: PlSmallStr,
381        capacity: usize,
382        precision: usize,
383        scale: usize,
384        decimal_comma: bool,
385    ) -> Self {
386        let builder = PrimitiveChunkedBuilder::<Int128Type>::new(name, capacity);
387        Self {
388            builder,
389            precision,
390            scale,
391            decimal_comma,
392        }
393    }
394}
395
396#[cfg(feature = "dtype-decimal")]
397impl ParsedBuilder for DecimalField {
398    #[inline]
399    fn parse_bytes(
400        &mut self,
401        mut bytes: &[u8],
402        ignore_errors: bool,
403        needs_escaping: bool,
404        _missing_is_null: bool,
405        _time_unit: Option<TimeUnit>,
406    ) -> PolarsResult<()> {
407        if !bytes.is_empty() && needs_escaping {
408            bytes = &bytes[1..bytes.len() - 1];
409        }
410
411        if !bytes.is_empty() && could_be_whitespace_fast(bytes[0]) {
412            bytes = skip_whitespace(bytes);
413        }
414
415        if bytes.is_empty() {
416            self.builder.append_null();
417            return Ok(());
418        }
419
420        match str_to_dec128(bytes, self.precision, self.scale, self.decimal_comma) {
421            Some(value) => self.builder.append_value(value),
422            None => {
423                if ignore_errors {
424                    self.builder.append_null()
425                } else {
426                    polars_bail!(ComputeError: "invalid decimal value found during CSV parsing")
427                }
428            },
429        }
430
431        Ok(())
432    }
433}
434
435#[cfg(any(feature = "dtype-datetime", feature = "dtype-date"))]
436pub struct DatetimeField<T: PolarsNumericType> {
437    compiled: Option<DatetimeInfer<T>>,
438    builder: PrimitiveChunkedBuilder<T>,
439}
440
441#[cfg(any(feature = "dtype-datetime", feature = "dtype-date"))]
442impl<T: PolarsNumericType> DatetimeField<T> {
443    fn new(name: PlSmallStr, capacity: usize) -> Self {
444        let builder = PrimitiveChunkedBuilder::<T>::new(name, capacity);
445        Self {
446            compiled: None,
447            builder,
448        }
449    }
450}
451
452#[cfg(any(feature = "dtype-datetime", feature = "dtype-date"))]
453fn slow_datetime_parser<T>(
454    buf: &mut DatetimeField<T>,
455    bytes: &[u8],
456    time_unit: Option<TimeUnit>,
457    ignore_errors: bool,
458) -> PolarsResult<()>
459where
460    T: PolarsNumericType,
461    DatetimeInfer<T>: TryFromWithUnit<Pattern>,
462{
463    let val = if bytes.is_ascii() {
464        // SAFETY:
465        // we just checked it is ascii
466        unsafe { std::str::from_utf8_unchecked(bytes) }
467    } else {
468        match std::str::from_utf8(bytes) {
469            Ok(val) => val,
470            Err(_) => {
471                if ignore_errors {
472                    buf.builder.append_null();
473                    return Ok(());
474                } else {
475                    polars_bail!(ComputeError: "invalid utf-8 sequence");
476                }
477            },
478        }
479    };
480
481    let pattern;
482    let parsed = match &mut buf.compiled {
483        // Retain the inferred candidate state across byte-parser misses. Rebuilding
484        // from `Pattern` here would reset the preferred format for every value.
485        Some(infer) => {
486            pattern = infer.pattern;
487            infer.parse(val)
488        },
489        None => {
490            pattern = match infer_pattern_single(val) {
491                Some(pattern) => pattern,
492                None => {
493                    if ignore_errors {
494                        buf.builder.append_null();
495                        return Ok(());
496                    } else {
497                        polars_bail!(ComputeError: "could not find a 'date/datetime' pattern for '{}'", val)
498                    }
499                },
500            };
501
502            let mut infer = match DatetimeInfer::try_from_with_unit(pattern, time_unit) {
503                Ok(infer) => infer,
504                Err(err) => {
505                    if ignore_errors {
506                        buf.builder.append_null();
507                        return Ok(());
508                    } else {
509                        return Err(err);
510                    }
511                },
512            };
513            let parsed = infer.parse(val);
514            if parsed.is_some() {
515                // Match the previous initialization behavior: only retain a parser
516                // after it has successfully parsed a value.
517                buf.compiled = Some(infer);
518            }
519            parsed
520        },
521    };
522
523    match parsed {
524        Some(parsed) => {
525            buf.builder.append_value(parsed);
526            Ok(())
527        },
528        None => {
529            if ignore_errors {
530                buf.builder.append_null();
531                Ok(())
532            } else {
533                polars_bail!(ComputeError: "could not parse '{}' with pattern '{:?}'", val, pattern)
534            }
535        },
536    }
537}
538
539#[cfg(any(feature = "dtype-datetime", feature = "dtype-date"))]
540impl<T> ParsedBuilder for DatetimeField<T>
541where
542    T: PolarsNumericType,
543    DatetimeInfer<T>: TryFromWithUnit<Pattern> + StrpTimeParser<T::Native>,
544{
545    #[inline]
546    fn parse_bytes(
547        &mut self,
548        mut bytes: &[u8],
549        ignore_errors: bool,
550        needs_escaping: bool,
551        _missing_is_null: bool,
552        time_unit: Option<TimeUnit>,
553    ) -> PolarsResult<()> {
554        if needs_escaping && bytes.len() >= 2 {
555            bytes = &bytes[1..bytes.len() - 1]
556        }
557
558        if bytes.is_empty() {
559            // for types other than string `_missing_is_null` is irrelevant; we always append null
560            self.builder.append_null();
561            return Ok(());
562        }
563
564        match &mut self.compiled {
565            None => slow_datetime_parser(self, bytes, time_unit, ignore_errors),
566            Some(compiled) => {
567                match compiled.parse_bytes(bytes, time_unit) {
568                    Some(parsed) => {
569                        self.builder.append_value(parsed);
570                        Ok(())
571                    },
572                    // fall back on chrono parser
573                    // this is a lot slower, we need to do utf8 checking and use
574                    // the slower parser
575                    None => slow_datetime_parser(self, bytes, time_unit, ignore_errors),
576                }
577            },
578        }
579    }
580}
581
582pub fn init_builders(
583    projection: &[usize],
584    capacity: usize,
585    schema: &Schema,
586    quote_char: Option<u8>,
587    encoding: CsvEncoding,
588    decimal_comma: bool,
589) -> PolarsResult<Vec<Builder>> {
590    projection
591        .iter()
592        .map(|&i| {
593            let (name, dtype) = schema.get_at_index(i).unwrap();
594            let name = name.clone();
595            let builder = match dtype {
596                &DataType::Boolean => Builder::Boolean(BooleanChunkedBuilder::new(name, capacity)),
597                #[cfg(feature = "dtype-i8")]
598                &DataType::Int8 => Builder::Int8(PrimitiveChunkedBuilder::new(name, capacity)),
599                #[cfg(feature = "dtype-i16")]
600                &DataType::Int16 => Builder::Int16(PrimitiveChunkedBuilder::new(name, capacity)),
601                &DataType::Int32 => Builder::Int32(PrimitiveChunkedBuilder::new(name, capacity)),
602                &DataType::Int64 => Builder::Int64(PrimitiveChunkedBuilder::new(name, capacity)),
603                #[cfg(feature = "dtype-i128")]
604                &DataType::Int128 => Builder::Int128(PrimitiveChunkedBuilder::new(name, capacity)),
605                #[cfg(feature = "dtype-u8")]
606                &DataType::UInt8 => Builder::UInt8(PrimitiveChunkedBuilder::new(name, capacity)),
607                #[cfg(feature = "dtype-u16")]
608                &DataType::UInt16 => Builder::UInt16(PrimitiveChunkedBuilder::new(name, capacity)),
609                &DataType::UInt32 => Builder::UInt32(PrimitiveChunkedBuilder::new(name, capacity)),
610                &DataType::UInt64 => Builder::UInt64(PrimitiveChunkedBuilder::new(name, capacity)),
611                #[cfg(feature = "dtype-u128")]
612                &DataType::UInt128 => {
613                    Builder::UInt128(PrimitiveChunkedBuilder::new(name, capacity))
614                },
615                #[cfg(feature = "dtype-f16")]
616                &DataType::Float16 => {
617                    if decimal_comma {
618                        Builder::DecimalFloat16(
619                            PrimitiveChunkedBuilder::new(name, capacity),
620                            Default::default(),
621                        )
622                    } else {
623                        Builder::Float16(PrimitiveChunkedBuilder::new(name, capacity))
624                    }
625                },
626                &DataType::Float32 => {
627                    if decimal_comma {
628                        Builder::DecimalFloat32(
629                            PrimitiveChunkedBuilder::new(name, capacity),
630                            Default::default(),
631                        )
632                    } else {
633                        Builder::Float32(PrimitiveChunkedBuilder::new(name, capacity))
634                    }
635                },
636                &DataType::Float64 => {
637                    if decimal_comma {
638                        Builder::DecimalFloat64(
639                            PrimitiveChunkedBuilder::new(name, capacity),
640                            Default::default(),
641                        )
642                    } else {
643                        Builder::Float64(PrimitiveChunkedBuilder::new(name, capacity))
644                    }
645                },
646                #[cfg(feature = "dtype-decimal")]
647                &DataType::Decimal(precision, scale) => Builder::Decimal(DecimalField::new(
648                    name,
649                    capacity,
650                    precision,
651                    scale,
652                    decimal_comma,
653                )),
654                &DataType::String => {
655                    Builder::Utf8(Utf8Field::new(name, capacity, quote_char, encoding))
656                },
657                #[cfg(feature = "dtype-datetime")]
658                DataType::Datetime(time_unit, time_zone) => Builder::Datetime {
659                    buf: DatetimeField::new(name, capacity),
660                    time_unit: *time_unit,
661                    time_zone: time_zone.clone(),
662                },
663                #[cfg(feature = "dtype-date")]
664                &DataType::Date => Builder::Date(DatetimeField::new(name, capacity)),
665                #[cfg(feature = "dtype-categorical")]
666                DataType::Categorical(_, _) | DataType::Enum(_, _) => {
667                    match dtype.cat_physical().unwrap() {
668                        CategoricalPhysical::U8 => {
669                            Builder::Categorical8(CategoricalField::<Categorical8Type>::new(
670                                name,
671                                capacity,
672                                quote_char,
673                                dtype.clone(),
674                            ))
675                        },
676                        CategoricalPhysical::U16 => {
677                            Builder::Categorical16(CategoricalField::<Categorical16Type>::new(
678                                name,
679                                capacity,
680                                quote_char,
681                                dtype.clone(),
682                            ))
683                        },
684                        CategoricalPhysical::U32 => {
685                            Builder::Categorical32(CategoricalField::<Categorical32Type>::new(
686                                name,
687                                capacity,
688                                quote_char,
689                                dtype.clone(),
690                            ))
691                        },
692                    }
693                },
694                dt => polars_bail!(
695                    ComputeError: "unsupported data type when reading CSV: {} when reading CSV", dt,
696                ),
697            };
698            Ok(builder)
699        })
700        .collect()
701}
702
703#[allow(clippy::large_enum_variant)]
704pub enum Builder {
705    Boolean(BooleanChunkedBuilder),
706    #[cfg(feature = "dtype-i8")]
707    Int8(PrimitiveChunkedBuilder<Int8Type>),
708    #[cfg(feature = "dtype-i16")]
709    Int16(PrimitiveChunkedBuilder<Int16Type>),
710    Int32(PrimitiveChunkedBuilder<Int32Type>),
711    Int64(PrimitiveChunkedBuilder<Int64Type>),
712    #[cfg(feature = "dtype-i128")]
713    Int128(PrimitiveChunkedBuilder<Int128Type>),
714    #[cfg(feature = "dtype-u8")]
715    UInt8(PrimitiveChunkedBuilder<UInt8Type>),
716    #[cfg(feature = "dtype-u16")]
717    UInt16(PrimitiveChunkedBuilder<UInt16Type>),
718    UInt32(PrimitiveChunkedBuilder<UInt32Type>),
719    UInt64(PrimitiveChunkedBuilder<UInt64Type>),
720    #[cfg(feature = "dtype-u128")]
721    UInt128(PrimitiveChunkedBuilder<UInt128Type>),
722    #[cfg(feature = "dtype-f16")]
723    Float16(PrimitiveChunkedBuilder<Float16Type>),
724    Float32(PrimitiveChunkedBuilder<Float32Type>),
725    Float64(PrimitiveChunkedBuilder<Float64Type>),
726    #[cfg(feature = "dtype-decimal")]
727    Decimal(DecimalField),
728    /// Stores the Utf8 fields and the total string length seen for that column
729    Utf8(Utf8Field),
730    #[cfg(feature = "dtype-datetime")]
731    Datetime {
732        buf: DatetimeField<Int64Type>,
733        time_unit: TimeUnit,
734        time_zone: Option<TimeZone>,
735    },
736    #[cfg(feature = "dtype-date")]
737    Date(DatetimeField<Int32Type>),
738    #[cfg(feature = "dtype-categorical")]
739    Categorical8(CategoricalField<Categorical8Type>),
740    #[cfg(feature = "dtype-categorical")]
741    Categorical16(CategoricalField<Categorical16Type>),
742    #[cfg(feature = "dtype-categorical")]
743    Categorical32(CategoricalField<Categorical32Type>),
744    #[cfg(feature = "dtype-f16")]
745    DecimalFloat16(PrimitiveChunkedBuilder<Float16Type>, Vec<u8>),
746    DecimalFloat32(PrimitiveChunkedBuilder<Float32Type>, Vec<u8>),
747    DecimalFloat64(PrimitiveChunkedBuilder<Float64Type>, Vec<u8>),
748}
749
750impl Builder {
751    pub fn into_series(self) -> PolarsResult<Series> {
752        let s = match self {
753            Builder::Boolean(v) => v.finish().into_series(),
754            #[cfg(feature = "dtype-i8")]
755            Builder::Int8(v) => v.finish().into_series(),
756            #[cfg(feature = "dtype-i16")]
757            Builder::Int16(v) => v.finish().into_series(),
758            Builder::Int32(v) => v.finish().into_series(),
759            Builder::Int64(v) => v.finish().into_series(),
760            #[cfg(feature = "dtype-i128")]
761            Builder::Int128(v) => v.finish().into_series(),
762            #[cfg(feature = "dtype-u8")]
763            Builder::UInt8(v) => v.finish().into_series(),
764            #[cfg(feature = "dtype-u16")]
765            Builder::UInt16(v) => v.finish().into_series(),
766            Builder::UInt32(v) => v.finish().into_series(),
767            Builder::UInt64(v) => v.finish().into_series(),
768            #[cfg(feature = "dtype-u128")]
769            Builder::UInt128(v) => v.finish().into_series(),
770            #[cfg(feature = "dtype-f16")]
771            Builder::Float16(v) => v.finish().into_series(),
772            Builder::Float32(v) => v.finish().into_series(),
773            Builder::Float64(v) => v.finish().into_series(),
774            #[cfg(feature = "dtype-f16")]
775            Builder::DecimalFloat16(v, _) => v.finish().into_series(),
776            Builder::DecimalFloat32(v, _) => v.finish().into_series(),
777            Builder::DecimalFloat64(v, _) => v.finish().into_series(),
778            #[cfg(feature = "dtype-decimal")]
779            Builder::Decimal(DecimalField {
780                builder,
781                precision,
782                scale,
783                ..
784            }) => unsafe {
785                builder
786                    .finish()
787                    .into_series()
788                    .from_physical_unchecked(&DataType::Decimal(precision, scale))
789                    .unwrap()
790            },
791            #[cfg(feature = "dtype-datetime")]
792            Builder::Datetime {
793                buf,
794                time_unit,
795                time_zone,
796            } => buf
797                .builder
798                .finish()
799                .into_series()
800                .cast(&DataType::Datetime(time_unit, time_zone))
801                .unwrap(),
802            #[cfg(feature = "dtype-date")]
803            Builder::Date(v) => v
804                .builder
805                .finish()
806                .into_series()
807                .cast(&DataType::Date)
808                .unwrap(),
809
810            Builder::Utf8(v) => {
811                let arr = v.mutable.freeze();
812                StringChunked::with_chunk(v.name, unsafe { arr.to_utf8view_unchecked() })
813                    .into_series()
814            },
815            #[cfg(feature = "dtype-categorical")]
816            Builder::Categorical8(buf) => buf.builder.finish().into_series(),
817            #[cfg(feature = "dtype-categorical")]
818            Builder::Categorical16(buf) => buf.builder.finish().into_series(),
819            #[cfg(feature = "dtype-categorical")]
820            Builder::Categorical32(buf) => buf.builder.finish().into_series(),
821        };
822        Ok(s)
823    }
824
825    pub fn add_null(&mut self, valid: bool) {
826        match self {
827            Builder::Boolean(v) => v.append_null(),
828            #[cfg(feature = "dtype-i8")]
829            Builder::Int8(v) => v.append_null(),
830            #[cfg(feature = "dtype-i16")]
831            Builder::Int16(v) => v.append_null(),
832            Builder::Int32(v) => v.append_null(),
833            Builder::Int64(v) => v.append_null(),
834            #[cfg(feature = "dtype-i128")]
835            Builder::Int128(v) => v.append_null(),
836            #[cfg(feature = "dtype-u8")]
837            Builder::UInt8(v) => v.append_null(),
838            #[cfg(feature = "dtype-u16")]
839            Builder::UInt16(v) => v.append_null(),
840            Builder::UInt32(v) => v.append_null(),
841            Builder::UInt64(v) => v.append_null(),
842            #[cfg(feature = "dtype-u128")]
843            Builder::UInt128(v) => v.append_null(),
844            #[cfg(feature = "dtype-f16")]
845            Builder::Float16(v) => v.append_null(),
846            Builder::Float32(v) => v.append_null(),
847            Builder::Float64(v) => v.append_null(),
848            #[cfg(feature = "dtype-decimal")]
849            Builder::Decimal(buf) => buf.builder.append_null(),
850            #[cfg(feature = "dtype-f16")]
851            Builder::DecimalFloat16(v, _) => v.append_null(),
852            Builder::DecimalFloat32(v, _) => v.append_null(),
853            Builder::DecimalFloat64(v, _) => v.append_null(),
854            Builder::Utf8(v) => {
855                if valid {
856                    v.mutable.push_value("")
857                } else {
858                    v.mutable.push_null()
859                }
860            },
861            #[cfg(feature = "dtype-datetime")]
862            Builder::Datetime { buf, .. } => buf.builder.append_null(),
863            #[cfg(feature = "dtype-date")]
864            Builder::Date(v) => v.builder.append_null(),
865            #[cfg(feature = "dtype-categorical")]
866            Builder::Categorical8(buf) => buf.builder.append_null(),
867            #[cfg(feature = "dtype-categorical")]
868            Builder::Categorical16(buf) => buf.builder.append_null(),
869            #[cfg(feature = "dtype-categorical")]
870            Builder::Categorical32(buf) => buf.builder.append_null(),
871        };
872    }
873
874    pub fn dtype(&self) -> DataType {
875        match self {
876            Builder::Boolean(_) => DataType::Boolean,
877            #[cfg(feature = "dtype-i8")]
878            Builder::Int8(_) => DataType::Int8,
879            #[cfg(feature = "dtype-i16")]
880            Builder::Int16(_) => DataType::Int16,
881            Builder::Int32(_) => DataType::Int32,
882            Builder::Int64(_) => DataType::Int64,
883            #[cfg(feature = "dtype-i128")]
884            Builder::Int128(_) => DataType::Int128,
885            #[cfg(feature = "dtype-u8")]
886            Builder::UInt8(_) => DataType::UInt8,
887            #[cfg(feature = "dtype-u16")]
888            Builder::UInt16(_) => DataType::UInt16,
889            Builder::UInt32(_) => DataType::UInt32,
890            Builder::UInt64(_) => DataType::UInt64,
891            #[cfg(feature = "dtype-u128")]
892            Builder::UInt128(_) => DataType::UInt128,
893            #[cfg(feature = "dtype-f16")]
894            Builder::Float16(_) | Builder::DecimalFloat16(_, _) => DataType::Float16,
895            Builder::Float32(_) | Builder::DecimalFloat32(_, _) => DataType::Float32,
896            Builder::Float64(_) | Builder::DecimalFloat64(_, _) => DataType::Float64,
897            #[cfg(feature = "dtype-decimal")]
898            Builder::Decimal(DecimalField {
899                precision, scale, ..
900            }) => DataType::Decimal(*precision, *scale),
901            Builder::Utf8(_) => DataType::String,
902            #[cfg(feature = "dtype-datetime")]
903            Builder::Datetime { time_unit, .. } => DataType::Datetime(*time_unit, None),
904            #[cfg(feature = "dtype-date")]
905            Builder::Date(_) => DataType::Date,
906            #[cfg(feature = "dtype-categorical")]
907            Builder::Categorical8(buf) => buf.builder.dtype().clone(),
908            #[cfg(feature = "dtype-categorical")]
909            Builder::Categorical16(buf) => buf.builder.dtype().clone(),
910            #[cfg(feature = "dtype-categorical")]
911            Builder::Categorical32(buf) => buf.builder.dtype().clone(),
912        }
913    }
914
915    #[inline]
916    pub fn add(
917        &mut self,
918        bytes: &[u8],
919        ignore_errors: bool,
920        needs_escaping: bool,
921        missing_is_null: bool,
922    ) -> PolarsResult<()> {
923        use Builder::*;
924        match self {
925            Boolean(buf) => <BooleanChunkedBuilder as ParsedBuilder>::parse_bytes(
926                buf,
927                bytes,
928                ignore_errors,
929                needs_escaping,
930                missing_is_null,
931                None,
932            ),
933            #[cfg(feature = "dtype-i8")]
934            Int8(buf) => <PrimitiveChunkedBuilder<Int8Type> as ParsedBuilder>::parse_bytes(
935                buf,
936                bytes,
937                ignore_errors,
938                needs_escaping,
939                missing_is_null,
940                None,
941            ),
942            #[cfg(feature = "dtype-i16")]
943            Int16(buf) => <PrimitiveChunkedBuilder<Int16Type> as ParsedBuilder>::parse_bytes(
944                buf,
945                bytes,
946                ignore_errors,
947                needs_escaping,
948                missing_is_null,
949                None,
950            ),
951            Int32(buf) => <PrimitiveChunkedBuilder<Int32Type> as ParsedBuilder>::parse_bytes(
952                buf,
953                bytes,
954                ignore_errors,
955                needs_escaping,
956                missing_is_null,
957                None,
958            ),
959            Int64(buf) => <PrimitiveChunkedBuilder<Int64Type> as ParsedBuilder>::parse_bytes(
960                buf,
961                bytes,
962                ignore_errors,
963                needs_escaping,
964                missing_is_null,
965                None,
966            ),
967            #[cfg(feature = "dtype-i128")]
968            Int128(buf) => <PrimitiveChunkedBuilder<Int128Type> as ParsedBuilder>::parse_bytes(
969                buf,
970                bytes,
971                ignore_errors,
972                needs_escaping,
973                missing_is_null,
974                None,
975            ),
976            #[cfg(feature = "dtype-u8")]
977            UInt8(buf) => <PrimitiveChunkedBuilder<UInt8Type> as ParsedBuilder>::parse_bytes(
978                buf,
979                bytes,
980                ignore_errors,
981                needs_escaping,
982                missing_is_null,
983                None,
984            ),
985            #[cfg(feature = "dtype-u16")]
986            UInt16(buf) => <PrimitiveChunkedBuilder<UInt16Type> as ParsedBuilder>::parse_bytes(
987                buf,
988                bytes,
989                ignore_errors,
990                needs_escaping,
991                missing_is_null,
992                None,
993            ),
994            UInt32(buf) => <PrimitiveChunkedBuilder<UInt32Type> as ParsedBuilder>::parse_bytes(
995                buf,
996                bytes,
997                ignore_errors,
998                needs_escaping,
999                missing_is_null,
1000                None,
1001            ),
1002            UInt64(buf) => <PrimitiveChunkedBuilder<UInt64Type> as ParsedBuilder>::parse_bytes(
1003                buf,
1004                bytes,
1005                ignore_errors,
1006                needs_escaping,
1007                missing_is_null,
1008                None,
1009            ),
1010            #[cfg(feature = "dtype-u128")]
1011            UInt128(buf) => <PrimitiveChunkedBuilder<UInt128Type> as ParsedBuilder>::parse_bytes(
1012                buf,
1013                bytes,
1014                ignore_errors,
1015                needs_escaping,
1016                missing_is_null,
1017                None,
1018            ),
1019            #[cfg(feature = "dtype-f16")]
1020            Float16(buf) => <PrimitiveChunkedBuilder<Float16Type> as ParsedBuilder>::parse_bytes(
1021                buf,
1022                bytes,
1023                ignore_errors,
1024                needs_escaping,
1025                missing_is_null,
1026                None,
1027            ),
1028            Float32(buf) => <PrimitiveChunkedBuilder<Float32Type> as ParsedBuilder>::parse_bytes(
1029                buf,
1030                bytes,
1031                ignore_errors,
1032                needs_escaping,
1033                missing_is_null,
1034                None,
1035            ),
1036            Float64(buf) => <PrimitiveChunkedBuilder<Float64Type> as ParsedBuilder>::parse_bytes(
1037                buf,
1038                bytes,
1039                ignore_errors,
1040                needs_escaping,
1041                missing_is_null,
1042                None,
1043            ),
1044            #[cfg(feature = "dtype-f16")]
1045            DecimalFloat16(buf, scratch) => {
1046                prepare_decimal_comma(bytes, scratch);
1047                <PrimitiveChunkedBuilder<Float16Type> as ParsedBuilder>::parse_bytes(
1048                    buf,
1049                    scratch,
1050                    ignore_errors,
1051                    needs_escaping,
1052                    missing_is_null,
1053                    None,
1054                )
1055            },
1056            DecimalFloat32(buf, scratch) => {
1057                prepare_decimal_comma(bytes, scratch);
1058                <PrimitiveChunkedBuilder<Float32Type> as ParsedBuilder>::parse_bytes(
1059                    buf,
1060                    scratch,
1061                    ignore_errors,
1062                    needs_escaping,
1063                    missing_is_null,
1064                    None,
1065                )
1066            },
1067            DecimalFloat64(buf, scratch) => {
1068                prepare_decimal_comma(bytes, scratch);
1069                <PrimitiveChunkedBuilder<Float64Type> as ParsedBuilder>::parse_bytes(
1070                    buf,
1071                    scratch,
1072                    ignore_errors,
1073                    needs_escaping,
1074                    missing_is_null,
1075                    None,
1076                )
1077            },
1078            #[cfg(feature = "dtype-decimal")]
1079            Decimal(buf) => <DecimalField as ParsedBuilder>::parse_bytes(
1080                buf,
1081                bytes,
1082                ignore_errors,
1083                needs_escaping,
1084                missing_is_null,
1085                None,
1086            ),
1087            Utf8(buf) => <Utf8Field as ParsedBuilder>::parse_bytes(
1088                buf,
1089                bytes,
1090                ignore_errors,
1091                needs_escaping,
1092                missing_is_null,
1093                None,
1094            ),
1095            #[cfg(feature = "dtype-datetime")]
1096            Datetime { buf, time_unit, .. } => {
1097                <DatetimeField<Int64Type> as ParsedBuilder>::parse_bytes(
1098                    buf,
1099                    bytes,
1100                    ignore_errors,
1101                    needs_escaping,
1102                    missing_is_null,
1103                    Some(*time_unit),
1104                )
1105            },
1106            #[cfg(feature = "dtype-date")]
1107            Date(buf) => <DatetimeField<Int32Type> as ParsedBuilder>::parse_bytes(
1108                buf,
1109                bytes,
1110                ignore_errors,
1111                needs_escaping,
1112                missing_is_null,
1113                None,
1114            ),
1115            #[cfg(feature = "dtype-categorical")]
1116            Categorical8(buf) => {
1117                buf.parse_bytes(bytes, ignore_errors, needs_escaping, missing_is_null, None)
1118            },
1119            #[cfg(feature = "dtype-categorical")]
1120            Categorical16(buf) => {
1121                buf.parse_bytes(bytes, ignore_errors, needs_escaping, missing_is_null, None)
1122            },
1123            #[cfg(feature = "dtype-categorical")]
1124            Categorical32(buf) => {
1125                buf.parse_bytes(bytes, ignore_errors, needs_escaping, missing_is_null, None)
1126            },
1127        }
1128    }
1129}
1130
1131#[inline]
1132fn prepare_decimal_comma(bytes: &[u8], scratch: &mut Vec<u8>) {
1133    scratch.clear();
1134    scratch.reserve(bytes.len());
1135
1136    // SAFETY: we pre-allocated.
1137    for &byte in bytes {
1138        if byte == b',' {
1139            unsafe { scratch.push_unchecked(b'.') }
1140        } else {
1141            unsafe { scratch.push_unchecked(byte) }
1142        }
1143    }
1144}
polars_io/csv/read/builder.rs

polars_io/csv/read/
builder.rs
