polars_core/series/
any_value.rs

1use std::fmt::Write;
2
3use num_traits::AsPrimitive;
4use polars_arrow::bitmap::Bitmap;
5use polars_compute::cast::SerPrimitive;
6
7#[cfg(feature = "dtype-categorical")]
8use crate::chunked_array::builder::CategoricalChunkedBuilder;
9use crate::chunked_array::builder::{AnonymousOwnedListBuilder, get_list_builder};
10#[cfg(feature = "dtype-struct")]
11use crate::prelude::any_value::arr_to_any_value;
12use crate::prelude::*;
13use crate::utils::any_values_to_supertype;
14
15impl<'a, T: AsRef<[AnyValue<'a>]>> NamedFrom<T, [AnyValue<'a>]> for Series {
16    /// Construct a new [`Series`] from a collection of [`AnyValue`].
17    ///
18    /// # Panics
19    ///
20    /// Panics if the values do not all share the same data type (with the exception
21    /// of [`DataType::Null`], which is always allowed).
22    ///
23    /// [`AnyValue`]: crate::datatypes::AnyValue
24    fn new(name: PlSmallStr, values: T) -> Self {
25        let values = values.as_ref();
26        Series::from_any_values(name, values, true).expect("data types of values should match")
27    }
28}
29
30impl Series {
31    /// Construct a new [`Series`] from a slice of AnyValues.
32    ///
33    /// The data type of the resulting Series is determined by the `values`
34    /// and the `strict` parameter:
35    /// - If `strict` is `true`, the data type is equal to the data type of the
36    ///   first non-null value. If any other non-null values do not match this
37    ///   data type, an error is raised. If the first non-null value is a
38    ///   decimal the slice is scanned for the maximum precision and scale possible.
39    /// - If `strict` is `false`, the data type is the supertype of the `values`.
40    ///   An error is returned if no supertype can be determined.
41    ///   **WARNING**: A full pass over the values is required to determine the supertype.
42    /// - If no values were passed, the resulting data type is `Null`.
43    pub fn from_any_values(
44        name: PlSmallStr,
45        values: &[AnyValue],
46        strict: bool,
47    ) -> PolarsResult<Self> {
48        fn get_first_non_null_dtype(values: &[AnyValue]) -> DataType {
49            let mut all_flat_null = true;
50            let first_non_null = values.iter().find(|av| {
51                if !av.is_null() {
52                    all_flat_null = false
53                };
54                !av.is_nested_null()
55            });
56            match first_non_null {
57                Some(av) => av.dtype(),
58                None => {
59                    if all_flat_null {
60                        DataType::Null
61                    } else {
62                        // Second pass to check for the nested null value that
63                        // toggled `all_flat_null` to false, e.g. a List(Null).
64                        let first_nested_null = values.iter().find(|av| !av.is_null()).unwrap();
65                        first_nested_null.dtype()
66                    }
67                },
68            }
69        }
70        let dtype = if strict {
71            match get_first_non_null_dtype(values) {
72                #[cfg(feature = "dtype-decimal")]
73                DataType::Decimal(mut prec, mut scale) => {
74                    for v in values {
75                        if let DataType::Decimal(p, s) = v.dtype() {
76                            prec = prec.max(p);
77                            scale = scale.max(s);
78                        }
79                    }
80                    DataType::Decimal(prec, scale)
81                },
82                dt => dt,
83            }
84        } else {
85            any_values_to_supertype(values)?
86        };
87
88        Self::from_any_values_and_dtype(name, values, &dtype, strict)
89    }
90
91    /// Construct a new [`Series`] with the given `dtype` from a slice of AnyValues.
92    ///
93    /// If `strict` is `true`, an error is returned if the values do not match the given
94    /// data type. If `strict` is `false`, values that do not match the given data type
95    /// are cast. If casting is not possible, the values are set to null instead.
96    pub fn from_any_values_and_dtype(
97        name: PlSmallStr,
98        values: &[AnyValue],
99        dtype: &DataType,
100        strict: bool,
101    ) -> PolarsResult<Self> {
102        Self::from_any_values_and_dtype_unnamed(values, dtype, strict)
103            .with_context(|| format!("while constructing Series '{name}'"))
104            .map(|s| s.with_name(name))
105    }
106
107    fn from_any_values_and_dtype_unnamed(
108        values: &[AnyValue],
109        dtype: &DataType,
110        strict: bool,
111    ) -> PolarsResult<Self> {
112        if values.is_empty() {
113            return Ok(Self::new_empty(PlSmallStr::EMPTY, dtype));
114        }
115
116        Ok(match dtype {
117            #[cfg(feature = "dtype-i8")]
118            DataType::Int8 => any_values_to_integer::<Int8Type>(values, strict)?.into_series(),
119            #[cfg(feature = "dtype-i16")]
120            DataType::Int16 => any_values_to_integer::<Int16Type>(values, strict)?.into_series(),
121            DataType::Int32 => any_values_to_integer::<Int32Type>(values, strict)?.into_series(),
122            DataType::Int64 => any_values_to_integer::<Int64Type>(values, strict)?.into_series(),
123            #[cfg(feature = "dtype-i128")]
124            DataType::Int128 => any_values_to_integer::<Int128Type>(values, strict)?.into_series(),
125            #[cfg(feature = "dtype-u8")]
126            DataType::UInt8 => any_values_to_integer::<UInt8Type>(values, strict)?.into_series(),
127            #[cfg(feature = "dtype-u16")]
128            DataType::UInt16 => any_values_to_integer::<UInt16Type>(values, strict)?.into_series(),
129            DataType::UInt32 => any_values_to_integer::<UInt32Type>(values, strict)?.into_series(),
130            DataType::UInt64 => any_values_to_integer::<UInt64Type>(values, strict)?.into_series(),
131            #[cfg(feature = "dtype-u128")]
132            DataType::UInt128 => {
133                any_values_to_integer::<UInt128Type>(values, strict)?.into_series()
134            },
135            #[cfg(feature = "dtype-f16")]
136            DataType::Float16 => any_values_to_f16(values, strict)?.into_series(),
137            DataType::Float32 => any_values_to_f32(values, strict)?.into_series(),
138            DataType::Float64 => any_values_to_f64(values, strict)?.into_series(),
139            DataType::Boolean => any_values_to_bool(values, strict)?.into_series(),
140            DataType::String => any_values_to_string(values, strict)?.into_series(),
141            DataType::Binary => any_values_to_binary(values, strict)?.into_series(),
142            DataType::BinaryOffset => any_values_to_binary_offset(values, strict)?.into_series(),
143            #[cfg(feature = "dtype-date")]
144            DataType::Date => any_values_to_date(values, strict)?.into_series(),
145            #[cfg(feature = "dtype-time")]
146            DataType::Time => any_values_to_time(values, strict)?.into_series(),
147            #[cfg(feature = "dtype-datetime")]
148            DataType::Datetime(tu, tz) => {
149                any_values_to_datetime(values, *tu, (*tz).clone(), strict)?.into_series()
150            },
151            #[cfg(feature = "dtype-duration")]
152            DataType::Duration(tu) => any_values_to_duration(values, *tu, strict)?.into_series(),
153            #[cfg(feature = "dtype-categorical")]
154            dt @ (DataType::Categorical(_, _) | DataType::Enum(_, _)) => {
155                any_values_to_categorical(values, dt, strict)?
156            },
157            #[cfg(feature = "dtype-decimal")]
158            DataType::Decimal(precision, scale) => {
159                any_values_to_decimal(values, *precision, *scale, strict)?.into_series()
160            },
161            #[cfg(feature = "dtype-map")]
162            DataType::Map(_, _) => {
163                let entries_dtype = dtype.map_entries_dtype().unwrap();
164                let entries = values
165                    .iter()
166                    .map(|av| match av {
167                        AnyValue::Map(entries) => Ok(AnyValue::List(entries.clone())),
168                        // An empty list of entries carries no field information.
169                        AnyValue::List(entries) if entries.dtype().is_nested_null() => {
170                            Ok(AnyValue::List(entries.clone()))
171                        },
172                        AnyValue::List(entries) => {
173                            ensure_map_entries_dtype(entries.dtype())?;
174                            Ok(AnyValue::List(entries.clone()))
175                        },
176                        av => Ok(av.clone()),
177                    })
178                    .collect::<PolarsResult<Vec<AnyValue>>>()?;
179                let storage = any_values_to_list(&entries, &entries_dtype, strict)?.into_series();
180                MapChunked::try_from_storage(dtype.clone(), storage)?.into_series()
181            },
182            #[cfg(feature = "dtype-extension")]
183            DataType::Extension(typ, storage) => {
184                Series::from_any_values_and_dtype_unnamed(values, storage, strict)?
185                    .into_extension(typ.clone())
186            },
187            DataType::List(inner) => any_values_to_list(values, inner, strict)?.into_series(),
188            #[cfg(feature = "dtype-array")]
189            DataType::Array(inner, size) => any_values_to_array(values, inner, strict, *size)?
190                .into_series()
191                .cast(&DataType::Array(inner.clone(), *size))?,
192            #[cfg(feature = "dtype-struct")]
193            DataType::Struct(fields) => any_values_to_struct(values, fields, strict)?,
194            #[cfg(feature = "object")]
195            DataType::Object(_) => any_values_to_object(values)?,
196            DataType::Null => Series::new_null(PlSmallStr::EMPTY, values.len()),
197            dt => {
198                polars_bail!(
199                    InvalidOperation:
200                    "constructing a Series with data type {dt:?} from AnyValues is not supported"
201                )
202            },
203        })
204    }
205}
206
207fn any_values_to_primitive_nonstrict<T: PolarsNumericType>(values: &[AnyValue]) -> ChunkedArray<T> {
208    values
209        .iter()
210        .map(|av| av.extract::<T::Native>())
211        .collect_trusted()
212}
213
214fn any_values_to_integer<T: PolarsIntegerType>(
215    values: &[AnyValue],
216    strict: bool,
217) -> PolarsResult<ChunkedArray<T>> {
218    fn any_values_to_integer_strict<T: PolarsIntegerType>(
219        values: &[AnyValue],
220    ) -> PolarsResult<ChunkedArray<T>> {
221        let mut builder = PrimitiveChunkedBuilder::<T>::new(PlSmallStr::EMPTY, values.len());
222        for av in values {
223            match &av {
224                av if av.is_integer() => {
225                    let opt_val = av.extract::<T::Native>();
226                    let val = match opt_val {
227                        Some(v) => v,
228                        None => return Err(invalid_value_error(&T::get_static_dtype(), av)),
229                    };
230                    builder.append_value(val)
231                },
232                AnyValue::Null => builder.append_null(),
233                av => return Err(invalid_value_error(&T::get_static_dtype(), av)),
234            }
235        }
236        Ok(builder.finish())
237    }
238
239    if strict {
240        any_values_to_integer_strict::<T>(values)
241    } else {
242        Ok(any_values_to_primitive_nonstrict::<T>(values))
243    }
244}
245
246#[cfg(feature = "dtype-f16")]
247fn any_values_to_f16(values: &[AnyValue], strict: bool) -> PolarsResult<Float16Chunked> {
248    fn any_values_to_f16_strict(values: &[AnyValue]) -> PolarsResult<Float16Chunked> {
249        let mut builder =
250            PrimitiveChunkedBuilder::<Float16Type>::new(PlSmallStr::EMPTY, values.len());
251        for av in values {
252            match av {
253                AnyValue::Float16(i) => builder.append_value(*i),
254                AnyValue::Null => builder.append_null(),
255                av => return Err(invalid_value_error(&DataType::Float16, av)),
256            }
257        }
258        Ok(builder.finish())
259    }
260    if strict {
261        any_values_to_f16_strict(values)
262    } else {
263        Ok(any_values_to_primitive_nonstrict::<Float16Type>(values))
264    }
265}
266
267fn any_values_to_f32(values: &[AnyValue], strict: bool) -> PolarsResult<Float32Chunked> {
268    fn any_values_to_f32_strict(values: &[AnyValue]) -> PolarsResult<Float32Chunked> {
269        let mut builder =
270            PrimitiveChunkedBuilder::<Float32Type>::new(PlSmallStr::EMPTY, values.len());
271        for av in values {
272            match av {
273                AnyValue::Float32(i) => builder.append_value(*i),
274                AnyValue::Float16(i) => builder.append_value(i.as_()),
275                AnyValue::Null => builder.append_null(),
276                av => return Err(invalid_value_error(&DataType::Float32, av)),
277            }
278        }
279        Ok(builder.finish())
280    }
281    if strict {
282        any_values_to_f32_strict(values)
283    } else {
284        Ok(any_values_to_primitive_nonstrict::<Float32Type>(values))
285    }
286}
287fn any_values_to_f64(values: &[AnyValue], strict: bool) -> PolarsResult<Float64Chunked> {
288    fn any_values_to_f64_strict(values: &[AnyValue]) -> PolarsResult<Float64Chunked> {
289        let mut builder =
290            PrimitiveChunkedBuilder::<Float64Type>::new(PlSmallStr::EMPTY, values.len());
291        for av in values {
292            match av {
293                AnyValue::Float64(i) => builder.append_value(*i),
294                AnyValue::Float32(i) => builder.append_value(*i as f64),
295                AnyValue::Float16(i) => builder.append_value(i.as_()),
296                AnyValue::Null => builder.append_null(),
297                av => return Err(invalid_value_error(&DataType::Float64, av)),
298            }
299        }
300        Ok(builder.finish())
301    }
302    if strict {
303        any_values_to_f64_strict(values)
304    } else {
305        Ok(any_values_to_primitive_nonstrict::<Float64Type>(values))
306    }
307}
308
309fn any_values_to_bool(values: &[AnyValue], strict: bool) -> PolarsResult<BooleanChunked> {
310    let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
311    for av in values {
312        match av {
313            AnyValue::Boolean(b) => builder.append_value(*b),
314            AnyValue::Null => builder.append_null(),
315            av => {
316                if strict {
317                    return Err(invalid_value_error(&DataType::Boolean, av));
318                }
319                match av.cast(&DataType::Boolean) {
320                    AnyValue::Boolean(b) => builder.append_value(b),
321                    _ => builder.append_null(),
322                }
323            },
324        }
325    }
326    Ok(builder.finish())
327}
328
329fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult<StringChunked> {
330    fn any_values_to_string_strict(values: &[AnyValue]) -> PolarsResult<StringChunked> {
331        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
332        for av in values {
333            match av {
334                AnyValue::String(s) => builder.append_value(s),
335                AnyValue::StringOwned(s) => builder.append_value(s),
336                AnyValue::Null => builder.append_null(),
337                av => return Err(invalid_value_error(&DataType::String, av)),
338            }
339        }
340        Ok(builder.finish())
341    }
342    fn any_values_to_string_nonstrict(values: &[AnyValue]) -> StringChunked {
343        fn _write_any_value(av: &AnyValue<'_>, buffer: &mut String) {
344            match av {
345                AnyValue::String(s) => buffer.push_str(s),
346                AnyValue::Float64(f) => {
347                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
348                },
349                AnyValue::Float32(f) => {
350                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
351                },
352                #[cfg(feature = "dtype-f16")]
353                AnyValue::Float16(f) => {
354                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
355                },
356                #[cfg(feature = "dtype-struct")]
357                AnyValue::StructOwned(payload) => {
358                    buffer.push('{');
359                    let mut iter = payload.0.iter().peekable();
360                    while let Some(child) = iter.next() {
361                        _write_any_value(child, buffer);
362                        if iter.peek().is_some() {
363                            buffer.push(',')
364                        }
365                    }
366                    buffer.push('}');
367                },
368                #[cfg(feature = "dtype-struct")]
369                AnyValue::Struct(_, _, flds) => {
370                    let mut vals = Vec::with_capacity(flds.len());
371                    av._materialize_struct_av(&mut vals);
372
373                    buffer.push('{');
374                    let mut iter = vals.iter().peekable();
375                    while let Some(child) = iter.next() {
376                        _write_any_value(child, buffer);
377                        if iter.peek().is_some() {
378                            buffer.push(',')
379                        }
380                    }
381                    buffer.push('}');
382                },
383                #[cfg(feature = "dtype-array")]
384                AnyValue::Array(vals, _) => {
385                    buffer.push('[');
386                    let mut iter = vals.iter().peekable();
387                    while let Some(child) = iter.next() {
388                        _write_any_value(&child, buffer);
389                        if iter.peek().is_some() {
390                            buffer.push(',');
391                        }
392                    }
393                    buffer.push(']');
394                },
395                AnyValue::List(vals) => {
396                    buffer.push('[');
397                    let mut iter = vals.iter().peekable();
398                    while let Some(child) = iter.next() {
399                        _write_any_value(&child, buffer);
400                        if iter.peek().is_some() {
401                            buffer.push(',');
402                        }
403                    }
404                    buffer.push(']');
405                },
406                av => {
407                    write!(buffer, "{av}").unwrap();
408                },
409            }
410        }
411
412        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
413        let mut owned = String::new(); // Amortize allocations.
414        for av in values {
415            owned.clear();
416
417            match av {
418                AnyValue::String(s) => builder.append_value(s),
419                AnyValue::StringOwned(s) => builder.append_value(s),
420                AnyValue::Null => builder.append_null(),
421                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(),
422
423                // Explicitly convert and dump floating-point values to strings
424                // to preserve as much precision as possible.
425                // Using write!(..., "{av}") steps through Display formatting
426                // which rounds to an arbitrary precision thus losing information.
427                av => {
428                    _write_any_value(av, &mut owned);
429                    builder.append_value(&owned);
430                },
431            }
432        }
433        builder.finish()
434    }
435    if strict {
436        any_values_to_string_strict(values)
437    } else {
438        Ok(any_values_to_string_nonstrict(values))
439    }
440}
441
442fn any_values_to_binary(values: &[AnyValue], strict: bool) -> PolarsResult<BinaryChunked> {
443    fn any_values_to_binary_strict(values: &[AnyValue]) -> PolarsResult<BinaryChunked> {
444        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
445        for av in values {
446            match av {
447                AnyValue::Binary(s) => builder.append_value(*s),
448                AnyValue::BinaryOwned(s) => builder.append_value(&**s),
449                AnyValue::Null => builder.append_null(),
450                av => return Err(invalid_value_error(&DataType::Binary, av)),
451            }
452        }
453        Ok(builder.finish())
454    }
455    fn any_values_to_binary_nonstrict(values: &[AnyValue]) -> BinaryChunked {
456        values
457            .iter()
458            .map(|av| match av {
459                AnyValue::Binary(b) => Some(*b),
460                AnyValue::BinaryOwned(b) => Some(&**b),
461                AnyValue::String(s) => Some(s.as_bytes()),
462                AnyValue::StringOwned(s) => Some(s.as_bytes()),
463                _ => None,
464            })
465            .collect_trusted()
466    }
467    if strict {
468        any_values_to_binary_strict(values)
469    } else {
470        Ok(any_values_to_binary_nonstrict(values))
471    }
472}
473
474fn any_values_to_binary_offset(
475    values: &[AnyValue],
476    strict: bool,
477) -> PolarsResult<BinaryOffsetChunked> {
478    let mut builder = MutableBinaryArray::<i64>::new();
479    for av in values {
480        match av {
481            AnyValue::Binary(s) => builder.push(Some(*s)),
482            AnyValue::BinaryOwned(s) => builder.push(Some(&**s)),
483            AnyValue::Null => builder.push_null(),
484            av => {
485                if strict {
486                    return Err(invalid_value_error(&DataType::Binary, av));
487                } else {
488                    builder.push_null();
489                };
490            },
491        }
492    }
493    Ok(BinaryOffsetChunked::with_chunk(
494        Default::default(),
495        builder.into(),
496    ))
497}
498
499#[cfg(feature = "dtype-date")]
500fn any_values_to_date(values: &[AnyValue], strict: bool) -> PolarsResult<DateChunked> {
501    let mut builder = PrimitiveChunkedBuilder::<Int32Type>::new(PlSmallStr::EMPTY, values.len());
502    for av in values {
503        match av {
504            AnyValue::Date(i) => builder.append_value(*i),
505            AnyValue::Null => builder.append_null(),
506            av => {
507                if strict {
508                    return Err(invalid_value_error(&DataType::Date, av));
509                }
510                match av.cast(&DataType::Date) {
511                    AnyValue::Date(i) => builder.append_value(i),
512                    _ => builder.append_null(),
513                }
514            },
515        }
516    }
517    Ok(builder.finish().into_date())
518}
519
520#[cfg(feature = "dtype-time")]
521fn any_values_to_time(values: &[AnyValue], strict: bool) -> PolarsResult<TimeChunked> {
522    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
523    for av in values {
524        match av {
525            AnyValue::Time(i) => builder.append_value(*i),
526            AnyValue::Null => builder.append_null(),
527            av => {
528                if strict {
529                    return Err(invalid_value_error(&DataType::Time, av));
530                }
531                match av.cast(&DataType::Time) {
532                    AnyValue::Time(i) => builder.append_value(i),
533                    _ => builder.append_null(),
534                }
535            },
536        }
537    }
538    Ok(builder.finish().into_time())
539}
540
541#[cfg(feature = "dtype-datetime")]
542fn any_values_to_datetime(
543    values: &[AnyValue],
544    time_unit: TimeUnit,
545    time_zone: Option<TimeZone>,
546    strict: bool,
547) -> PolarsResult<DatetimeChunked> {
548    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
549    let target_dtype = DataType::Datetime(time_unit, time_zone.clone());
550    for av in values {
551        match av {
552            AnyValue::Datetime(i, tu, _) if *tu == time_unit => builder.append_value(*i),
553            AnyValue::DatetimeOwned(i, tu, _) if *tu == time_unit => builder.append_value(*i),
554            AnyValue::Null => builder.append_null(),
555            av => {
556                if strict {
557                    return Err(invalid_value_error(&target_dtype, av));
558                }
559                match av.cast(&target_dtype) {
560                    AnyValue::Datetime(i, _, _) => builder.append_value(i),
561                    AnyValue::DatetimeOwned(i, _, _) => builder.append_value(i),
562                    _ => builder.append_null(),
563                }
564            },
565        }
566    }
567    Ok(builder.finish().into_datetime(time_unit, time_zone))
568}
569
570#[cfg(feature = "dtype-duration")]
571fn any_values_to_duration(
572    values: &[AnyValue],
573    time_unit: TimeUnit,
574    strict: bool,
575) -> PolarsResult<DurationChunked> {
576    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
577    let target_dtype = DataType::Duration(time_unit);
578    for av in values {
579        match av {
580            AnyValue::Duration(i, tu) if *tu == time_unit => builder.append_value(*i),
581            AnyValue::Null => builder.append_null(),
582            av => {
583                if strict {
584                    return Err(invalid_value_error(&target_dtype, av));
585                }
586                match av.cast(&target_dtype) {
587                    AnyValue::Duration(i, _) => builder.append_value(i),
588                    _ => builder.append_null(),
589                }
590            },
591        }
592    }
593    Ok(builder.finish().into_duration(time_unit))
594}
595
596#[cfg(feature = "dtype-categorical")]
597fn any_values_to_categorical(
598    values: &[AnyValue],
599    dtype: &DataType,
600    strict: bool,
601) -> PolarsResult<Series> {
602    with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
603        let mut builder = CategoricalChunkedBuilder::<$C>::new(PlSmallStr::EMPTY, dtype.clone());
604
605        let mut owned = String::new(); // Amortize allocations.
606        for av in values {
607            let ret = match av {
608                AnyValue::String(s) => builder.append_str(s),
609                AnyValue::StringOwned(s) => builder.append_str(s),
610
611                &AnyValue::Enum(cat, &ref map) |
612                &AnyValue::EnumOwned(cat, ref map) |
613                &AnyValue::Categorical(cat, &ref map) |
614                &AnyValue::CategoricalOwned(cat, ref map) => builder.append_cat(cat, map),
615
616                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) if !strict => {
617                    builder.append_null();
618                    Ok(())
619                },
620                AnyValue::Null => {
621                    builder.append_null();
622                    Ok(())
623                }
624
625                av => {
626                    if strict {
627                        return Err(invalid_value_error(&DataType::String, av));
628                    }
629
630                    owned.clear();
631                    write!(owned, "{av}").unwrap();
632                    builder.append_str(&owned)
633                },
634            };
635
636            if let Err(e) = ret {
637                if strict {
638                    return Err(e);
639                } else {
640                    builder.append_null();
641                }
642            }
643        }
644
645        let ca = builder.finish();
646        Ok(ca.into_series())
647    })
648}
649
650#[cfg(feature = "dtype-decimal")]
651fn any_values_to_decimal(
652    values: &[AnyValue],
653    precision: usize,
654    scale: usize,
655    strict: bool,
656) -> PolarsResult<DecimalChunked> {
657    let target_dtype = DataType::Decimal(precision, scale);
658
659    let mut builder = PrimitiveChunkedBuilder::<Int128Type>::new(PlSmallStr::EMPTY, values.len());
660    for av in values {
661        match av {
662            // Allow equal or less scale. We do want to support different scales even in 'strict' mode.
663            AnyValue::Decimal(v, p, s) if *s <= scale => {
664                if *p <= precision && *s == scale {
665                    builder.append_value(*v)
666                } else {
667                    match av.strict_cast(&target_dtype) {
668                        Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
669                        _ => builder.append_null(),
670                    }
671                }
672            },
673            AnyValue::Null => builder.append_null(),
674            av => {
675                if strict {
676                    return Err(invalid_value_error(&target_dtype, av));
677                }
678                match av.strict_cast(&target_dtype) {
679                    Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
680                    _ => builder.append_null(),
681                }
682            },
683        };
684    }
685
686    // Build the array and do a precision check if needed.
687    builder.finish().into_decimal(precision, scale)
688}
689
690fn any_values_to_list(
691    avs: &[AnyValue],
692    inner_type: &DataType,
693    strict: bool,
694) -> PolarsResult<ListChunked> {
695    // GB:
696    // Lord forgive for the sins I have committed in this function. The amount of strange
697    // exceptions that need to happen for this to work are insane and I feel like I am going crazy.
698    //
699    // This function is essentially a copy of the `<ListChunked as FromIterator>` where it does not
700    // sample the datatype from the first element and instead we give it explicitly. This allows
701    // this function to properly assign a datatype if `avs` starts with a `null` value. Previously,
702    // this was solved by assigning the `dtype` again afterwards, but why? We should not link the
703    // implementation of these functions. We still need to assign the dtype of the ListArray and
704    // such, anyways.
705    //
706    // Then, `collect_ca_with_dtype` does not possess the necessary exceptions shown in this
707    // function to use that. I have tried adding the exceptions there and it broke other things. I
708    // really do feel like this is the simplest solution.
709
710    let mut valid = true;
711    let capacity = avs.len();
712
713    let ca = match inner_type {
714        // AnyValues with empty lists in python can create
715        // Series of an unknown dtype.
716        // We use the anonymousbuilder without a dtype
717        // the empty arrays is then not added (we add an extra offset instead)
718        // the next non-empty series then must have the correct dtype.
719        DataType::Null => {
720            let mut builder = AnonymousOwnedListBuilder::new(PlSmallStr::EMPTY, capacity, None);
721            for av in avs {
722                match av {
723                    AnyValue::List(b) => builder.append_series(b)?,
724                    AnyValue::Null => builder.append_null(),
725                    _ => {
726                        valid = false;
727                        builder.append_null();
728                    },
729                }
730            }
731            builder.finish()
732        },
733
734        #[cfg(feature = "object")]
735        DataType::Object(_) => polars_bail!(nyi = "Nested object types"),
736
737        _ => {
738            let mut builder =
739                get_list_builder(inner_type, capacity * 5, capacity, PlSmallStr::EMPTY);
740            for av in avs {
741                match av {
742                    AnyValue::List(b) => match b.cast(inner_type) {
743                        Ok(casted) => {
744                            if casted.null_count() != b.null_count() {
745                                valid = !strict;
746                            }
747                            builder.append_series(&casted)?;
748                        },
749                        Err(_) => {
750                            valid = false;
751                            for _ in 0..b.len() {
752                                builder.append_null();
753                            }
754                        },
755                    },
756                    AnyValue::Null => builder.append_null(),
757                    _ => {
758                        valid = false;
759                        builder.append_null()
760                    },
761                }
762            }
763
764            builder.finish()
765        },
766    };
767
768    if strict && !valid {
769        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", DataType::List(Box::new(inner_type.clone())));
770    }
771
772    Ok(ca)
773}
774
775#[cfg(feature = "dtype-array")]
776fn any_values_to_array(
777    avs: &[AnyValue],
778    inner_type: &DataType,
779    strict: bool,
780    width: usize,
781) -> PolarsResult<ArrayChunked> {
782    fn to_arr(s: &Series) -> Option<ArrayRef> {
783        if s.chunks().len() > 1 {
784            let s = s.rechunk();
785            Some(s.chunks()[0].clone())
786        } else {
787            Some(s.chunks()[0].clone())
788        }
789    }
790
791    let target_dtype = DataType::Array(Box::new(inner_type.clone()), width);
792
793    // This is handled downstream. The builder will choose the first non null type.
794    let mut valid = true;
795    #[allow(unused_mut)]
796    let mut out: ArrayChunked = if inner_type == &DataType::Null {
797        avs.iter()
798            .map(|av| match av {
799                AnyValue::List(b) | AnyValue::Array(b, _) => to_arr(b),
800                AnyValue::Null => None,
801                _ => {
802                    valid = false;
803                    None
804                },
805            })
806            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
807    }
808    // Make sure that wrongly inferred AnyValues don't deviate from the datatype.
809    else {
810        avs.iter()
811            .map(|av| match av {
812                AnyValue::List(b) | AnyValue::Array(b, _) => {
813                    if b.dtype() == inner_type {
814                        to_arr(b)
815                    } else {
816                        let s = match b.cast(inner_type) {
817                            Ok(out) => out,
818                            Err(_) => Series::full_null(b.name().clone(), b.len(), inner_type),
819                        };
820                        to_arr(&s)
821                    }
822                },
823                AnyValue::Null => None,
824                _ => {
825                    valid = false;
826                    None
827                },
828            })
829            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
830    };
831
832    if strict && !valid {
833        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", target_dtype);
834    }
835    polars_ensure!(
836        out.width() == width,
837        SchemaMismatch: "got mixed size array widths where width {} was expected", width
838    );
839
840    // Ensure the logical type is correct for nested types.
841    #[cfg(feature = "dtype-struct")]
842    if !matches!(inner_type, DataType::Null) && out.inner_dtype().is_nested() {
843        unsafe {
844            out.set_dtype(target_dtype);
845        };
846    }
847
848    Ok(out)
849}
850
851/// A Struct's Fields, compared and hashed by name.
852#[cfg(feature = "dtype-struct")]
853#[derive(Clone, Copy)]
854struct FieldsByName<'a>(&'a [Field]);
855
856#[cfg(feature = "dtype-struct")]
857impl Eq for FieldsByName<'_> {}
858
859#[cfg(feature = "dtype-struct")]
860impl PartialEq for FieldsByName<'_> {
861    fn eq(&self, other: &Self) -> bool {
862        let names_eq = |(l, r): (&Field, &Field)| l.name == r.name;
863        self.0.len() == other.0.len() && self.0.iter().zip(other.0).all(names_eq)
864    }
865}
866
867#[cfg(feature = "dtype-struct")]
868impl std::hash::Hash for FieldsByName<'_> {
869    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
870        self.0.iter().for_each(|fld| fld.name.hash(state));
871    }
872}
873
874#[cfg(feature = "dtype-struct")]
875fn any_values_to_struct(
876    values: &[AnyValue],
877    fields: &[Field],
878    strict: bool,
879) -> PolarsResult<Series> {
880    // Values other than structs here (or lists/arrays of their fields)
881    // are null (or invalid if `strict`).
882    let n_fields = fields.len();
883    let is_valid = |av: &AnyValue| match av {
884        AnyValue::StructOwned(_) | AnyValue::Struct(..) => true,
885        AnyValue::List(s) => s.len() == n_fields,
886        #[cfg(feature = "dtype-array")]
887        AnyValue::Array(s, _) => s.len() == n_fields,
888        _ => false,
889    };
890    if strict && let Some(av) = values.iter().find(|av| !av.is_null() && !is_valid(av)) {
891        return Err(invalid_value_error(&DataType::Struct(fields.to_vec()), av));
892    }
893
894    // Empty struct fast path (no fields, but outer validity still required).
895    if n_fields == 0 {
896        let mut out = StructChunked::from_series(PlSmallStr::EMPTY, values.len(), [].iter())?;
897        out.set_outer_validity(Bitmap::opt_from_iter(values.iter().map(is_valid)));
898        return Ok(out.into_series());
899    }
900
901    // Resolve once per distinct ordered sequence of field names.
902    let mut field_order_ids = PlHashMap::from_iter([(FieldsByName(fields), 0)]);
903    let mut field_positions: Vec<_> = (0..n_fields).map(Some).collect();
904    let mut av_positions = PlHashMap::<&str, usize>::default();
905    let mut last = (FieldsByName(fields), 0);
906    let av_field_order_ids: Vec<usize> = values
907        .iter()
908        .map(|av| {
909            let av_fields = match av {
910                AnyValue::StructOwned(payload) => FieldsByName(&payload.1),
911                AnyValue::Struct(_, _, av_fields) => FieldsByName(av_fields),
912                _ => return 0,
913            };
914            if last.0 != av_fields {
915                let next_id = field_order_ids.len();
916                let id = *field_order_ids.entry(av_fields).or_insert_with(|| {
917                    av_positions.clear();
918                    for (j, fld) in av_fields.0.iter().enumerate() {
919                        av_positions.entry(fld.name.as_str()).or_insert(j);
920                    }
921                    let positions = fields.iter().map(|fld| av_positions.get(fld.name.as_str()));
922                    field_positions.extend(positions.map(|j| j.copied()));
923                    next_id
924                });
925                last = (av_fields, id);
926            }
927            last.1
928        })
929        .collect();
930
931    // The physical series fields of the struct.
932    let mut series_fields = Vec::with_capacity(n_fields);
933    let mut field_avs = Vec::with_capacity(values.len());
934    for (i, field) in fields.iter().enumerate() {
935        field_avs.clear();
936
937        for (av, &field_order_id) in values.iter().zip(&av_field_order_ids) {
938            let position = field_positions[field_order_id * n_fields + i];
939            match av {
940                AnyValue::StructOwned(payload) => {
941                    let av_values = &payload.0;
942                    field_avs.push(position.map_or(AnyValue::Null, |j| av_values[j].as_borrowed()));
943                },
944                AnyValue::Struct(idx, arr, av_fields) => {
945                    field_avs.push(position.map_or(AnyValue::Null, |j| {
946                        // SAFETY: `idx` and `j` are in bounds; `av_fields[j]` describes this child.
947                        unsafe { arr_to_any_value(&*arr.values()[j], *idx, av_fields[j].dtype()) }
948                    }));
949                },
950                AnyValue::List(s) if s.len() == n_fields => {
951                    let av = unsafe { s.get_unchecked(i) };
952                    field_avs.push(av);
953                },
954                #[cfg(feature = "dtype-array")]
955                AnyValue::Array(s, _) if s.len() == n_fields => {
956                    let av = unsafe { s.get_unchecked(i) };
957                    field_avs.push(av);
958                },
959                _ => field_avs.push(AnyValue::Null),
960            }
961        }
962        // If the inferred dtype is null, we let auto inference work.
963        let s = if matches!(field.dtype, DataType::Null) {
964            Series::from_any_values(field.name().clone(), &field_avs, strict)?
965        } else {
966            Series::from_any_values_and_dtype(
967                field.name().clone(),
968                &field_avs,
969                &field.dtype,
970                strict,
971            )?
972        };
973        series_fields.push(s)
974    }
975
976    let mut out =
977        StructChunked::from_series(PlSmallStr::EMPTY, values.len(), series_fields.iter())?;
978    let validity = Bitmap::opt_from_iter(values.iter().map(is_valid));
979    if validity.is_some() {
980        out.set_outer_validity(validity);
981    }
982    Ok(out.into_series())
983}
984
985#[cfg(feature = "object")]
986fn any_values_to_object(values: &[AnyValue]) -> PolarsResult<Series> {
987    use crate::chunked_array::object::registry;
988    let converter = registry::get_object_converter();
989    let mut builder = registry::get_object_builder(PlSmallStr::EMPTY, values.len());
990    for av in values {
991        match av {
992            AnyValue::Object(val) => builder.append_value(val.as_any()),
993            AnyValue::Null => builder.append_null(),
994            _ => {
995                // This is needed because in Python users can send mixed types.
996                // This only works if you set a global converter.
997                let any = converter(av.as_borrowed());
998                builder.append_value(&*any)
999            },
1000        }
1001    }
1002
1003    Ok(builder.to_series())
1004}
1005
1006fn invalid_value_error(dtype: &DataType, value: &AnyValue) -> PolarsError {
1007    polars_err!(
1008        SchemaMismatch:
1009        "unexpected value while building Series of type {:?}; found value of type {:?}: {}",
1010        dtype,
1011        value.dtype(),
1012        value
1013    )
1014}
polars_core/series/any_value.rs

polars_core/series/
any_value.rs
