polars_core/series/
any_value.rs

1use std::fmt::Write;
2
3use arrow::bitmap::Bitmap;
4use num_traits::AsPrimitive;
5use polars_compute::cast::SerPrimitive;
6
7#[cfg(feature = "dtype-categorical")]
8use crate::chunked_array::builder::CategoricalChunkedBuilder;
9use crate::chunked_array::builder::{AnonymousOwnedListBuilder, get_list_builder};
10use crate::prelude::*;
11use crate::utils::any_values_to_supertype;
12
13impl<'a, T: AsRef<[AnyValue<'a>]>> NamedFrom<T, [AnyValue<'a>]> for Series {
14    /// Construct a new [`Series`] from a collection of [`AnyValue`].
15    ///
16    /// # Panics
17    ///
18    /// Panics if the values do not all share the same data type (with the exception
19    /// of [`DataType::Null`], which is always allowed).
20    ///
21    /// [`AnyValue`]: crate::datatypes::AnyValue
22    fn new(name: PlSmallStr, values: T) -> Self {
23        let values = values.as_ref();
24        Series::from_any_values(name, values, true).expect("data types of values should match")
25    }
26}
27
28impl Series {
29    /// Construct a new [`Series`] from a slice of AnyValues.
30    ///
31    /// The data type of the resulting Series is determined by the `values`
32    /// and the `strict` parameter:
33    /// - If `strict` is `true`, the data type is equal to the data type of the
34    ///   first non-null value. If any other non-null values do not match this
35    ///   data type, an error is raised. If the first non-null value is a
36    ///   decimal the slice is scanned for the maximum precision and scale possible.
37    /// - If `strict` is `false`, the data type is the supertype of the `values`.
38    ///   An error is returned if no supertype can be determined.
39    ///   **WARNING**: A full pass over the values is required to determine the supertype.
40    /// - If no values were passed, the resulting data type is `Null`.
41    pub fn from_any_values(
42        name: PlSmallStr,
43        values: &[AnyValue],
44        strict: bool,
45    ) -> PolarsResult<Self> {
46        fn get_first_non_null_dtype(values: &[AnyValue]) -> DataType {
47            let mut all_flat_null = true;
48            let first_non_null = values.iter().find(|av| {
49                if !av.is_null() {
50                    all_flat_null = false
51                };
52                !av.is_nested_null()
53            });
54            match first_non_null {
55                Some(av) => av.dtype(),
56                None => {
57                    if all_flat_null {
58                        DataType::Null
59                    } else {
60                        // Second pass to check for the nested null value that
61                        // toggled `all_flat_null` to false, e.g. a List(Null).
62                        let first_nested_null = values.iter().find(|av| !av.is_null()).unwrap();
63                        first_nested_null.dtype()
64                    }
65                },
66            }
67        }
68        let dtype = if strict {
69            match get_first_non_null_dtype(values) {
70                #[cfg(feature = "dtype-decimal")]
71                DataType::Decimal(mut prec, mut scale) => {
72                    for v in values {
73                        if let DataType::Decimal(p, s) = v.dtype() {
74                            prec = prec.max(p);
75                            scale = scale.max(s);
76                        }
77                    }
78                    DataType::Decimal(prec, scale)
79                },
80                dt => dt,
81            }
82        } else {
83            any_values_to_supertype(values)?
84        };
85
86        Self::from_any_values_and_dtype(name, values, &dtype, strict)
87    }
88
89    /// Construct a new [`Series`] with the given `dtype` from a slice of AnyValues.
90    ///
91    /// If `strict` is `true`, an error is returned if the values do not match the given
92    /// data type. If `strict` is `false`, values that do not match the given data type
93    /// are cast. If casting is not possible, the values are set to null instead.
94    pub fn from_any_values_and_dtype(
95        name: PlSmallStr,
96        values: &[AnyValue],
97        dtype: &DataType,
98        strict: bool,
99    ) -> PolarsResult<Self> {
100        Self::from_any_values_and_dtype_unnamed(values, dtype, strict)
101            .with_context(|| format!("while constructing Series '{name}'"))
102            .map(|s| s.with_name(name))
103    }
104
105    fn from_any_values_and_dtype_unnamed(
106        values: &[AnyValue],
107        dtype: &DataType,
108        strict: bool,
109    ) -> PolarsResult<Self> {
110        if values.is_empty() {
111            return Ok(Self::new_empty(PlSmallStr::EMPTY, dtype));
112        }
113
114        Ok(match dtype {
115            #[cfg(feature = "dtype-i8")]
116            DataType::Int8 => any_values_to_integer::<Int8Type>(values, strict)?.into_series(),
117            #[cfg(feature = "dtype-i16")]
118            DataType::Int16 => any_values_to_integer::<Int16Type>(values, strict)?.into_series(),
119            DataType::Int32 => any_values_to_integer::<Int32Type>(values, strict)?.into_series(),
120            DataType::Int64 => any_values_to_integer::<Int64Type>(values, strict)?.into_series(),
121            #[cfg(feature = "dtype-i128")]
122            DataType::Int128 => any_values_to_integer::<Int128Type>(values, strict)?.into_series(),
123            #[cfg(feature = "dtype-u8")]
124            DataType::UInt8 => any_values_to_integer::<UInt8Type>(values, strict)?.into_series(),
125            #[cfg(feature = "dtype-u16")]
126            DataType::UInt16 => any_values_to_integer::<UInt16Type>(values, strict)?.into_series(),
127            DataType::UInt32 => any_values_to_integer::<UInt32Type>(values, strict)?.into_series(),
128            DataType::UInt64 => any_values_to_integer::<UInt64Type>(values, strict)?.into_series(),
129            #[cfg(feature = "dtype-u128")]
130            DataType::UInt128 => {
131                any_values_to_integer::<UInt128Type>(values, strict)?.into_series()
132            },
133            #[cfg(feature = "dtype-f16")]
134            DataType::Float16 => any_values_to_f16(values, strict)?.into_series(),
135            DataType::Float32 => any_values_to_f32(values, strict)?.into_series(),
136            DataType::Float64 => any_values_to_f64(values, strict)?.into_series(),
137            DataType::Boolean => any_values_to_bool(values, strict)?.into_series(),
138            DataType::String => any_values_to_string(values, strict)?.into_series(),
139            DataType::Binary => any_values_to_binary(values, strict)?.into_series(),
140            DataType::BinaryOffset => any_values_to_binary_offset(values, strict)?.into_series(),
141            #[cfg(feature = "dtype-date")]
142            DataType::Date => any_values_to_date(values, strict)?.into_series(),
143            #[cfg(feature = "dtype-time")]
144            DataType::Time => any_values_to_time(values, strict)?.into_series(),
145            #[cfg(feature = "dtype-datetime")]
146            DataType::Datetime(tu, tz) => {
147                any_values_to_datetime(values, *tu, (*tz).clone(), strict)?.into_series()
148            },
149            #[cfg(feature = "dtype-duration")]
150            DataType::Duration(tu) => any_values_to_duration(values, *tu, strict)?.into_series(),
151            #[cfg(feature = "dtype-categorical")]
152            dt @ (DataType::Categorical(_, _) | DataType::Enum(_, _)) => {
153                any_values_to_categorical(values, dt, strict)?
154            },
155            #[cfg(feature = "dtype-decimal")]
156            DataType::Decimal(precision, scale) => {
157                any_values_to_decimal(values, *precision, *scale, strict)?.into_series()
158            },
159            #[cfg(feature = "dtype-map")]
160            DataType::Map(_, _) => {
161                let entries_dtype = dtype.map_entries_dtype().unwrap();
162                let entries = values
163                    .iter()
164                    .map(|av| match av {
165                        AnyValue::Map(entries) => Ok(AnyValue::List(entries.clone())),
166                        // An empty list of entries carries no field information.
167                        AnyValue::List(entries) if entries.dtype().is_nested_null() => {
168                            Ok(AnyValue::List(entries.clone()))
169                        },
170                        AnyValue::List(entries) => {
171                            ensure_map_entries_dtype(entries.dtype())?;
172                            Ok(AnyValue::List(entries.clone()))
173                        },
174                        av => Ok(av.clone()),
175                    })
176                    .collect::<PolarsResult<Vec<AnyValue>>>()?;
177                let storage = any_values_to_list(&entries, &entries_dtype, strict)?.into_series();
178                MapChunked::try_from_storage(dtype.clone(), storage)?.into_series()
179            },
180            #[cfg(feature = "dtype-extension")]
181            DataType::Extension(typ, storage) => {
182                Series::from_any_values_and_dtype_unnamed(values, storage, strict)?
183                    .into_extension(typ.clone())
184            },
185            DataType::List(inner) => any_values_to_list(values, inner, strict)?.into_series(),
186            #[cfg(feature = "dtype-array")]
187            DataType::Array(inner, size) => any_values_to_array(values, inner, strict, *size)?
188                .into_series()
189                .cast(&DataType::Array(inner.clone(), *size))?,
190            #[cfg(feature = "dtype-struct")]
191            DataType::Struct(fields) => any_values_to_struct(values, fields, strict)?,
192            #[cfg(feature = "object")]
193            DataType::Object(_) => any_values_to_object(values)?,
194            DataType::Null => Series::new_null(PlSmallStr::EMPTY, values.len()),
195            dt => {
196                polars_bail!(
197                    InvalidOperation:
198                    "constructing a Series with data type {dt:?} from AnyValues is not supported"
199                )
200            },
201        })
202    }
203}
204
205fn any_values_to_primitive_nonstrict<T: PolarsNumericType>(values: &[AnyValue]) -> ChunkedArray<T> {
206    values
207        .iter()
208        .map(|av| av.extract::<T::Native>())
209        .collect_trusted()
210}
211
212fn any_values_to_integer<T: PolarsIntegerType>(
213    values: &[AnyValue],
214    strict: bool,
215) -> PolarsResult<ChunkedArray<T>> {
216    fn any_values_to_integer_strict<T: PolarsIntegerType>(
217        values: &[AnyValue],
218    ) -> PolarsResult<ChunkedArray<T>> {
219        let mut builder = PrimitiveChunkedBuilder::<T>::new(PlSmallStr::EMPTY, values.len());
220        for av in values {
221            match &av {
222                av if av.is_integer() => {
223                    let opt_val = av.extract::<T::Native>();
224                    let val = match opt_val {
225                        Some(v) => v,
226                        None => return Err(invalid_value_error(&T::get_static_dtype(), av)),
227                    };
228                    builder.append_value(val)
229                },
230                AnyValue::Null => builder.append_null(),
231                av => return Err(invalid_value_error(&T::get_static_dtype(), av)),
232            }
233        }
234        Ok(builder.finish())
235    }
236
237    if strict {
238        any_values_to_integer_strict::<T>(values)
239    } else {
240        Ok(any_values_to_primitive_nonstrict::<T>(values))
241    }
242}
243
244#[cfg(feature = "dtype-f16")]
245fn any_values_to_f16(values: &[AnyValue], strict: bool) -> PolarsResult<Float16Chunked> {
246    fn any_values_to_f16_strict(values: &[AnyValue]) -> PolarsResult<Float16Chunked> {
247        let mut builder =
248            PrimitiveChunkedBuilder::<Float16Type>::new(PlSmallStr::EMPTY, values.len());
249        for av in values {
250            match av {
251                AnyValue::Float16(i) => builder.append_value(*i),
252                AnyValue::Null => builder.append_null(),
253                av => return Err(invalid_value_error(&DataType::Float16, av)),
254            }
255        }
256        Ok(builder.finish())
257    }
258    if strict {
259        any_values_to_f16_strict(values)
260    } else {
261        Ok(any_values_to_primitive_nonstrict::<Float16Type>(values))
262    }
263}
264
265fn any_values_to_f32(values: &[AnyValue], strict: bool) -> PolarsResult<Float32Chunked> {
266    fn any_values_to_f32_strict(values: &[AnyValue]) -> PolarsResult<Float32Chunked> {
267        let mut builder =
268            PrimitiveChunkedBuilder::<Float32Type>::new(PlSmallStr::EMPTY, values.len());
269        for av in values {
270            match av {
271                AnyValue::Float32(i) => builder.append_value(*i),
272                AnyValue::Float16(i) => builder.append_value(i.as_()),
273                AnyValue::Null => builder.append_null(),
274                av => return Err(invalid_value_error(&DataType::Float32, av)),
275            }
276        }
277        Ok(builder.finish())
278    }
279    if strict {
280        any_values_to_f32_strict(values)
281    } else {
282        Ok(any_values_to_primitive_nonstrict::<Float32Type>(values))
283    }
284}
285fn any_values_to_f64(values: &[AnyValue], strict: bool) -> PolarsResult<Float64Chunked> {
286    fn any_values_to_f64_strict(values: &[AnyValue]) -> PolarsResult<Float64Chunked> {
287        let mut builder =
288            PrimitiveChunkedBuilder::<Float64Type>::new(PlSmallStr::EMPTY, values.len());
289        for av in values {
290            match av {
291                AnyValue::Float64(i) => builder.append_value(*i),
292                AnyValue::Float32(i) => builder.append_value(*i as f64),
293                AnyValue::Float16(i) => builder.append_value(i.as_()),
294                AnyValue::Null => builder.append_null(),
295                av => return Err(invalid_value_error(&DataType::Float64, av)),
296            }
297        }
298        Ok(builder.finish())
299    }
300    if strict {
301        any_values_to_f64_strict(values)
302    } else {
303        Ok(any_values_to_primitive_nonstrict::<Float64Type>(values))
304    }
305}
306
307fn any_values_to_bool(values: &[AnyValue], strict: bool) -> PolarsResult<BooleanChunked> {
308    let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
309    for av in values {
310        match av {
311            AnyValue::Boolean(b) => builder.append_value(*b),
312            AnyValue::Null => builder.append_null(),
313            av => {
314                if strict {
315                    return Err(invalid_value_error(&DataType::Boolean, av));
316                }
317                match av.cast(&DataType::Boolean) {
318                    AnyValue::Boolean(b) => builder.append_value(b),
319                    _ => builder.append_null(),
320                }
321            },
322        }
323    }
324    Ok(builder.finish())
325}
326
327fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult<StringChunked> {
328    fn any_values_to_string_strict(values: &[AnyValue]) -> PolarsResult<StringChunked> {
329        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
330        for av in values {
331            match av {
332                AnyValue::String(s) => builder.append_value(s),
333                AnyValue::StringOwned(s) => builder.append_value(s),
334                AnyValue::Null => builder.append_null(),
335                av => return Err(invalid_value_error(&DataType::String, av)),
336            }
337        }
338        Ok(builder.finish())
339    }
340    fn any_values_to_string_nonstrict(values: &[AnyValue]) -> StringChunked {
341        fn _write_any_value(av: &AnyValue<'_>, buffer: &mut String) {
342            match av {
343                AnyValue::String(s) => buffer.push_str(s),
344                AnyValue::Float64(f) => {
345                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
346                },
347                AnyValue::Float32(f) => {
348                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
349                },
350                #[cfg(feature = "dtype-f16")]
351                AnyValue::Float16(f) => {
352                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
353                },
354                #[cfg(feature = "dtype-struct")]
355                AnyValue::StructOwned(payload) => {
356                    buffer.push('{');
357                    let mut iter = payload.0.iter().peekable();
358                    while let Some(child) = iter.next() {
359                        _write_any_value(child, buffer);
360                        if iter.peek().is_some() {
361                            buffer.push(',')
362                        }
363                    }
364                    buffer.push('}');
365                },
366                #[cfg(feature = "dtype-struct")]
367                AnyValue::Struct(_, _, flds) => {
368                    let mut vals = Vec::with_capacity(flds.len());
369                    av._materialize_struct_av(&mut vals);
370
371                    buffer.push('{');
372                    let mut iter = vals.iter().peekable();
373                    while let Some(child) = iter.next() {
374                        _write_any_value(child, buffer);
375                        if iter.peek().is_some() {
376                            buffer.push(',')
377                        }
378                    }
379                    buffer.push('}');
380                },
381                #[cfg(feature = "dtype-array")]
382                AnyValue::Array(vals, _) => {
383                    buffer.push('[');
384                    let mut iter = vals.iter().peekable();
385                    while let Some(child) = iter.next() {
386                        _write_any_value(&child, buffer);
387                        if iter.peek().is_some() {
388                            buffer.push(',');
389                        }
390                    }
391                    buffer.push(']');
392                },
393                AnyValue::List(vals) => {
394                    buffer.push('[');
395                    let mut iter = vals.iter().peekable();
396                    while let Some(child) = iter.next() {
397                        _write_any_value(&child, buffer);
398                        if iter.peek().is_some() {
399                            buffer.push(',');
400                        }
401                    }
402                    buffer.push(']');
403                },
404                av => {
405                    write!(buffer, "{av}").unwrap();
406                },
407            }
408        }
409
410        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
411        let mut owned = String::new(); // Amortize allocations.
412        for av in values {
413            owned.clear();
414
415            match av {
416                AnyValue::String(s) => builder.append_value(s),
417                AnyValue::StringOwned(s) => builder.append_value(s),
418                AnyValue::Null => builder.append_null(),
419                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(),
420
421                // Explicitly convert and dump floating-point values to strings
422                // to preserve as much precision as possible.
423                // Using write!(..., "{av}") steps through Display formatting
424                // which rounds to an arbitrary precision thus losing information.
425                av => {
426                    _write_any_value(av, &mut owned);
427                    builder.append_value(&owned);
428                },
429            }
430        }
431        builder.finish()
432    }
433    if strict {
434        any_values_to_string_strict(values)
435    } else {
436        Ok(any_values_to_string_nonstrict(values))
437    }
438}
439
440fn any_values_to_binary(values: &[AnyValue], strict: bool) -> PolarsResult<BinaryChunked> {
441    fn any_values_to_binary_strict(values: &[AnyValue]) -> PolarsResult<BinaryChunked> {
442        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
443        for av in values {
444            match av {
445                AnyValue::Binary(s) => builder.append_value(*s),
446                AnyValue::BinaryOwned(s) => builder.append_value(&**s),
447                AnyValue::Null => builder.append_null(),
448                av => return Err(invalid_value_error(&DataType::Binary, av)),
449            }
450        }
451        Ok(builder.finish())
452    }
453    fn any_values_to_binary_nonstrict(values: &[AnyValue]) -> BinaryChunked {
454        values
455            .iter()
456            .map(|av| match av {
457                AnyValue::Binary(b) => Some(*b),
458                AnyValue::BinaryOwned(b) => Some(&**b),
459                AnyValue::String(s) => Some(s.as_bytes()),
460                AnyValue::StringOwned(s) => Some(s.as_bytes()),
461                _ => None,
462            })
463            .collect_trusted()
464    }
465    if strict {
466        any_values_to_binary_strict(values)
467    } else {
468        Ok(any_values_to_binary_nonstrict(values))
469    }
470}
471
472fn any_values_to_binary_offset(
473    values: &[AnyValue],
474    strict: bool,
475) -> PolarsResult<BinaryOffsetChunked> {
476    let mut builder = MutableBinaryArray::<i64>::new();
477    for av in values {
478        match av {
479            AnyValue::Binary(s) => builder.push(Some(*s)),
480            AnyValue::BinaryOwned(s) => builder.push(Some(&**s)),
481            AnyValue::Null => builder.push_null(),
482            av => {
483                if strict {
484                    return Err(invalid_value_error(&DataType::Binary, av));
485                } else {
486                    builder.push_null();
487                };
488            },
489        }
490    }
491    Ok(BinaryOffsetChunked::with_chunk(
492        Default::default(),
493        builder.into(),
494    ))
495}
496
497#[cfg(feature = "dtype-date")]
498fn any_values_to_date(values: &[AnyValue], strict: bool) -> PolarsResult<DateChunked> {
499    let mut builder = PrimitiveChunkedBuilder::<Int32Type>::new(PlSmallStr::EMPTY, values.len());
500    for av in values {
501        match av {
502            AnyValue::Date(i) => builder.append_value(*i),
503            AnyValue::Null => builder.append_null(),
504            av => {
505                if strict {
506                    return Err(invalid_value_error(&DataType::Date, av));
507                }
508                match av.cast(&DataType::Date) {
509                    AnyValue::Date(i) => builder.append_value(i),
510                    _ => builder.append_null(),
511                }
512            },
513        }
514    }
515    Ok(builder.finish().into_date())
516}
517
518#[cfg(feature = "dtype-time")]
519fn any_values_to_time(values: &[AnyValue], strict: bool) -> PolarsResult<TimeChunked> {
520    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
521    for av in values {
522        match av {
523            AnyValue::Time(i) => builder.append_value(*i),
524            AnyValue::Null => builder.append_null(),
525            av => {
526                if strict {
527                    return Err(invalid_value_error(&DataType::Time, av));
528                }
529                match av.cast(&DataType::Time) {
530                    AnyValue::Time(i) => builder.append_value(i),
531                    _ => builder.append_null(),
532                }
533            },
534        }
535    }
536    Ok(builder.finish().into_time())
537}
538
539#[cfg(feature = "dtype-datetime")]
540fn any_values_to_datetime(
541    values: &[AnyValue],
542    time_unit: TimeUnit,
543    time_zone: Option<TimeZone>,
544    strict: bool,
545) -> PolarsResult<DatetimeChunked> {
546    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
547    let target_dtype = DataType::Datetime(time_unit, time_zone.clone());
548    for av in values {
549        match av {
550            AnyValue::Datetime(i, tu, _) if *tu == time_unit => builder.append_value(*i),
551            AnyValue::DatetimeOwned(i, tu, _) if *tu == time_unit => builder.append_value(*i),
552            AnyValue::Null => builder.append_null(),
553            av => {
554                if strict {
555                    return Err(invalid_value_error(&target_dtype, av));
556                }
557                match av.cast(&target_dtype) {
558                    AnyValue::Datetime(i, _, _) => builder.append_value(i),
559                    AnyValue::DatetimeOwned(i, _, _) => builder.append_value(i),
560                    _ => builder.append_null(),
561                }
562            },
563        }
564    }
565    Ok(builder.finish().into_datetime(time_unit, time_zone))
566}
567
568#[cfg(feature = "dtype-duration")]
569fn any_values_to_duration(
570    values: &[AnyValue],
571    time_unit: TimeUnit,
572    strict: bool,
573) -> PolarsResult<DurationChunked> {
574    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
575    let target_dtype = DataType::Duration(time_unit);
576    for av in values {
577        match av {
578            AnyValue::Duration(i, tu) if *tu == time_unit => builder.append_value(*i),
579            AnyValue::Null => builder.append_null(),
580            av => {
581                if strict {
582                    return Err(invalid_value_error(&target_dtype, av));
583                }
584                match av.cast(&target_dtype) {
585                    AnyValue::Duration(i, _) => builder.append_value(i),
586                    _ => builder.append_null(),
587                }
588            },
589        }
590    }
591    Ok(builder.finish().into_duration(time_unit))
592}
593
594#[cfg(feature = "dtype-categorical")]
595fn any_values_to_categorical(
596    values: &[AnyValue],
597    dtype: &DataType,
598    strict: bool,
599) -> PolarsResult<Series> {
600    with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
601        let mut builder = CategoricalChunkedBuilder::<$C>::new(PlSmallStr::EMPTY, dtype.clone());
602
603        let mut owned = String::new(); // Amortize allocations.
604        for av in values {
605            let ret = match av {
606                AnyValue::String(s) => builder.append_str(s),
607                AnyValue::StringOwned(s) => builder.append_str(s),
608
609                &AnyValue::Enum(cat, &ref map) |
610                &AnyValue::EnumOwned(cat, ref map) |
611                &AnyValue::Categorical(cat, &ref map) |
612                &AnyValue::CategoricalOwned(cat, ref map) => builder.append_cat(cat, map),
613
614                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) if !strict => {
615                    builder.append_null();
616                    Ok(())
617                },
618                AnyValue::Null => {
619                    builder.append_null();
620                    Ok(())
621                }
622
623                av => {
624                    if strict {
625                        return Err(invalid_value_error(&DataType::String, av));
626                    }
627
628                    owned.clear();
629                    write!(owned, "{av}").unwrap();
630                    builder.append_str(&owned)
631                },
632            };
633
634            if let Err(e) = ret {
635                if strict {
636                    return Err(e);
637                } else {
638                    builder.append_null();
639                }
640            }
641        }
642
643        let ca = builder.finish();
644        Ok(ca.into_series())
645    })
646}
647
648#[cfg(feature = "dtype-decimal")]
649fn any_values_to_decimal(
650    values: &[AnyValue],
651    precision: usize,
652    scale: usize,
653    strict: bool,
654) -> PolarsResult<DecimalChunked> {
655    let target_dtype = DataType::Decimal(precision, scale);
656
657    let mut builder = PrimitiveChunkedBuilder::<Int128Type>::new(PlSmallStr::EMPTY, values.len());
658    for av in values {
659        match av {
660            // Allow equal or less scale. We do want to support different scales even in 'strict' mode.
661            AnyValue::Decimal(v, p, s) if *s <= scale => {
662                if *p <= precision && *s == scale {
663                    builder.append_value(*v)
664                } else {
665                    match av.strict_cast(&target_dtype) {
666                        Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
667                        _ => builder.append_null(),
668                    }
669                }
670            },
671            AnyValue::Null => builder.append_null(),
672            av => {
673                if strict {
674                    return Err(invalid_value_error(&target_dtype, av));
675                }
676                match av.strict_cast(&target_dtype) {
677                    Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
678                    _ => builder.append_null(),
679                }
680            },
681        };
682    }
683
684    // Build the array and do a precision check if needed.
685    builder.finish().into_decimal(precision, scale)
686}
687
688fn any_values_to_list(
689    avs: &[AnyValue],
690    inner_type: &DataType,
691    strict: bool,
692) -> PolarsResult<ListChunked> {
693    // GB:
694    // Lord forgive for the sins I have committed in this function. The amount of strange
695    // exceptions that need to happen for this to work are insane and I feel like I am going crazy.
696    //
697    // This function is essentially a copy of the `<ListChunked as FromIterator>` where it does not
698    // sample the datatype from the first element and instead we give it explicitly. This allows
699    // this function to properly assign a datatype if `avs` starts with a `null` value. Previously,
700    // this was solved by assigning the `dtype` again afterwards, but why? We should not link the
701    // implementation of these functions. We still need to assign the dtype of the ListArray and
702    // such, anyways.
703    //
704    // Then, `collect_ca_with_dtype` does not possess the necessary exceptions shown in this
705    // function to use that. I have tried adding the exceptions there and it broke other things. I
706    // really do feel like this is the simplest solution.
707
708    let mut valid = true;
709    let capacity = avs.len();
710
711    let ca = match inner_type {
712        // AnyValues with empty lists in python can create
713        // Series of an unknown dtype.
714        // We use the anonymousbuilder without a dtype
715        // the empty arrays is then not added (we add an extra offset instead)
716        // the next non-empty series then must have the correct dtype.
717        DataType::Null => {
718            let mut builder = AnonymousOwnedListBuilder::new(PlSmallStr::EMPTY, capacity, None);
719            for av in avs {
720                match av {
721                    AnyValue::List(b) => builder.append_series(b)?,
722                    AnyValue::Null => builder.append_null(),
723                    _ => {
724                        valid = false;
725                        builder.append_null();
726                    },
727                }
728            }
729            builder.finish()
730        },
731
732        #[cfg(feature = "object")]
733        DataType::Object(_) => polars_bail!(nyi = "Nested object types"),
734
735        _ => {
736            let mut builder =
737                get_list_builder(inner_type, capacity * 5, capacity, PlSmallStr::EMPTY);
738            for av in avs {
739                match av {
740                    AnyValue::List(b) => match b.cast(inner_type) {
741                        Ok(casted) => {
742                            if casted.null_count() != b.null_count() {
743                                valid = !strict;
744                            }
745                            builder.append_series(&casted)?;
746                        },
747                        Err(_) => {
748                            valid = false;
749                            for _ in 0..b.len() {
750                                builder.append_null();
751                            }
752                        },
753                    },
754                    AnyValue::Null => builder.append_null(),
755                    _ => {
756                        valid = false;
757                        builder.append_null()
758                    },
759                }
760            }
761
762            builder.finish()
763        },
764    };
765
766    if strict && !valid {
767        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", DataType::List(Box::new(inner_type.clone())));
768    }
769
770    Ok(ca)
771}
772
773#[cfg(feature = "dtype-array")]
774fn any_values_to_array(
775    avs: &[AnyValue],
776    inner_type: &DataType,
777    strict: bool,
778    width: usize,
779) -> PolarsResult<ArrayChunked> {
780    fn to_arr(s: &Series) -> Option<ArrayRef> {
781        if s.chunks().len() > 1 {
782            let s = s.rechunk();
783            Some(s.chunks()[0].clone())
784        } else {
785            Some(s.chunks()[0].clone())
786        }
787    }
788
789    let target_dtype = DataType::Array(Box::new(inner_type.clone()), width);
790
791    // This is handled downstream. The builder will choose the first non null type.
792    let mut valid = true;
793    #[allow(unused_mut)]
794    let mut out: ArrayChunked = if inner_type == &DataType::Null {
795        avs.iter()
796            .map(|av| match av {
797                AnyValue::List(b) | AnyValue::Array(b, _) => to_arr(b),
798                AnyValue::Null => None,
799                _ => {
800                    valid = false;
801                    None
802                },
803            })
804            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
805    }
806    // Make sure that wrongly inferred AnyValues don't deviate from the datatype.
807    else {
808        avs.iter()
809            .map(|av| match av {
810                AnyValue::List(b) | AnyValue::Array(b, _) => {
811                    if b.dtype() == inner_type {
812                        to_arr(b)
813                    } else {
814                        let s = match b.cast(inner_type) {
815                            Ok(out) => out,
816                            Err(_) => Series::full_null(b.name().clone(), b.len(), inner_type),
817                        };
818                        to_arr(&s)
819                    }
820                },
821                AnyValue::Null => None,
822                _ => {
823                    valid = false;
824                    None
825                },
826            })
827            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
828    };
829
830    if strict && !valid {
831        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", target_dtype);
832    }
833    polars_ensure!(
834        out.width() == width,
835        SchemaMismatch: "got mixed size array widths where width {} was expected", width
836    );
837
838    // Ensure the logical type is correct for nested types.
839    #[cfg(feature = "dtype-struct")]
840    if !matches!(inner_type, DataType::Null) && out.inner_dtype().is_nested() {
841        unsafe {
842            out.set_dtype(target_dtype);
843        };
844    }
845
846    Ok(out)
847}
848
849#[cfg(feature = "dtype-struct")]
850fn _any_values_to_struct<'a>(
851    av_fields: &[Field],
852    av_values: &[AnyValue<'a>],
853    field_index: usize,
854    field: &Field,
855    fields: &[Field],
856    field_avs: &mut Vec<AnyValue<'a>>,
857) {
858    // TODO: Optimize.
859
860    let mut append_by_search = || {
861        // Search for the name.
862        if let Some(i) = av_fields
863            .iter()
864            .position(|av_fld| av_fld.name == field.name)
865        {
866            field_avs.push(av_values[i].clone());
867            return;
868        }
869        field_avs.push(AnyValue::Null)
870    };
871
872    // All fields are available in this single value.
873    // We can use the index to get value.
874    if fields.len() == av_fields.len() {
875        if fields.iter().zip(av_fields.iter()).any(|(l, r)| l != r) {
876            append_by_search()
877        } else {
878            let av_val = av_values
879                .get(field_index)
880                .cloned()
881                .unwrap_or(AnyValue::Null);
882            field_avs.push(av_val)
883        }
884    }
885    // Not all fields are available, we search the proper field.
886    else {
887        // Search for the name.
888        append_by_search()
889    }
890}
891
892#[cfg(feature = "dtype-struct")]
893fn any_values_to_struct(
894    values: &[AnyValue],
895    fields: &[Field],
896    strict: bool,
897) -> PolarsResult<Series> {
898    // Fast path for structs with no fields.
899    if fields.is_empty() {
900        let mut out = StructChunked::from_series(PlSmallStr::EMPTY, values.len(), [].iter())?;
901        out.set_outer_validity(Bitmap::opt_from_iter(values.iter().map(|av| !av.is_null())));
902        return Ok(out.into_series());
903    }
904
905    // The physical series fields of the struct.
906    let mut series_fields = Vec::with_capacity(fields.len());
907    let mut has_outer_validity = false;
908    let mut field_avs = Vec::with_capacity(values.len());
909    for (i, field) in fields.iter().enumerate() {
910        field_avs.clear();
911
912        for av in values.iter() {
913            match av {
914                AnyValue::StructOwned(payload) => {
915                    let av_fields = &payload.1;
916                    let av_values = &payload.0;
917                    _any_values_to_struct(av_fields, av_values, i, field, fields, &mut field_avs);
918                },
919                AnyValue::Struct(_, _, av_fields) => {
920                    let av_values: Vec<_> = av._iter_struct_av().collect();
921                    _any_values_to_struct(av_fields, &av_values, i, field, fields, &mut field_avs);
922                },
923                AnyValue::List(s) if s.len() == fields.len() => {
924                    let av = unsafe { s.get_unchecked(i) };
925                    field_avs.push(av);
926                },
927                #[cfg(feature = "dtype-array")]
928                AnyValue::Array(s, _) if s.len() == fields.len() => {
929                    let av = unsafe { s.get_unchecked(i) };
930                    field_avs.push(av);
931                },
932                AnyValue::Null => {
933                    has_outer_validity = true;
934                    field_avs.push(AnyValue::Null)
935                },
936                _ => {
937                    if strict {
938                        return Err(invalid_value_error(&DataType::Struct(fields.to_vec()), av));
939                    } else {
940                        has_outer_validity = true;
941                        field_avs.push(AnyValue::Null)
942                    }
943                },
944            }
945        }
946        // If the inferred dtype is null, we let auto inference work.
947        let s = if matches!(field.dtype, DataType::Null) {
948            Series::from_any_values(field.name().clone(), &field_avs, strict)?
949        } else {
950            Series::from_any_values_and_dtype(
951                field.name().clone(),
952                &field_avs,
953                &field.dtype,
954                strict,
955            )?
956        };
957        series_fields.push(s)
958    }
959
960    let mut out =
961        StructChunked::from_series(PlSmallStr::EMPTY, values.len(), series_fields.iter())?;
962    if has_outer_validity {
963        out.set_outer_validity(Bitmap::opt_from_iter(values.iter().map(|av| !av.is_null())));
964    }
965    Ok(out.into_series())
966}
967
968#[cfg(feature = "object")]
969fn any_values_to_object(values: &[AnyValue]) -> PolarsResult<Series> {
970    use crate::chunked_array::object::registry;
971    let converter = registry::get_object_converter();
972    let mut builder = registry::get_object_builder(PlSmallStr::EMPTY, values.len());
973    for av in values {
974        match av {
975            AnyValue::Object(val) => builder.append_value(val.as_any()),
976            AnyValue::Null => builder.append_null(),
977            _ => {
978                // This is needed because in Python users can send mixed types.
979                // This only works if you set a global converter.
980                let any = converter(av.as_borrowed());
981                builder.append_value(&*any)
982            },
983        }
984    }
985
986    Ok(builder.to_series())
987}
988
989fn invalid_value_error(dtype: &DataType, value: &AnyValue) -> PolarsError {
990    polars_err!(
991        SchemaMismatch:
992        "unexpected value while building Series of type {:?}; found value of type {:?}: {}",
993        dtype,
994        value.dtype(),
995        value
996    )
997}
polars_core/series/any_value.rs

polars_core/series/
any_value.rs
