polars_core/series/
any_value.rs

1use std::fmt::Write;
2
3use arrow::bitmap::MutableBitmap;
4
5#[cfg(feature = "dtype-categorical")]
6use crate::chunked_array::builder::CategoricalChunkedBuilder;
7use crate::chunked_array::builder::{AnonymousOwnedListBuilder, get_list_builder};
8use crate::prelude::*;
9use crate::utils::any_values_to_supertype;
10
11impl<'a, T: AsRef<[AnyValue<'a>]>> NamedFrom<T, [AnyValue<'a>]> for Series {
12    /// Construct a new [`Series`] from a collection of [`AnyValue`].
13    ///
14    /// # Panics
15    ///
16    /// Panics if the values do not all share the same data type (with the exception
17    /// of [`DataType::Null`], which is always allowed).
18    ///
19    /// [`AnyValue`]: crate::datatypes::AnyValue
20    fn new(name: PlSmallStr, values: T) -> Self {
21        let values = values.as_ref();
22        Series::from_any_values(name, values, true).expect("data types of values should match")
23    }
24}
25
26impl Series {
27    /// Construct a new [`Series`] from a slice of AnyValues.
28    ///
29    /// The data type of the resulting Series is determined by the `values`
30    /// and the `strict` parameter:
31    /// - If `strict` is `true`, the data type is equal to the data type of the
32    ///   first non-null value. If any other non-null values do not match this
33    ///   data type, an error is raised. If the first non-null value is a
34    ///   decimal the slice is scanned for the maximum precision and scale possible.
35    /// - If `strict` is `false`, the data type is the supertype of the `values`.
36    ///   An error is returned if no supertype can be determined.
37    ///   **WARNING**: A full pass over the values is required to determine the supertype.
38    /// - If no values were passed, the resulting data type is `Null`.
39    pub fn from_any_values(
40        name: PlSmallStr,
41        values: &[AnyValue],
42        strict: bool,
43    ) -> PolarsResult<Self> {
44        fn get_first_non_null_dtype(values: &[AnyValue]) -> DataType {
45            let mut all_flat_null = true;
46            let first_non_null = values.iter().find(|av| {
47                if !av.is_null() {
48                    all_flat_null = false
49                };
50                !av.is_nested_null()
51            });
52            match first_non_null {
53                Some(av) => av.dtype(),
54                None => {
55                    if all_flat_null {
56                        DataType::Null
57                    } else {
58                        // Second pass to check for the nested null value that
59                        // toggled `all_flat_null` to false, e.g. a List(Null).
60                        let first_nested_null = values.iter().find(|av| !av.is_null()).unwrap();
61                        first_nested_null.dtype()
62                    }
63                },
64            }
65        }
66        let dtype = if strict {
67            match get_first_non_null_dtype(values) {
68                #[cfg(feature = "dtype-decimal")]
69                DataType::Decimal(mut prec, mut scale) => {
70                    for v in values {
71                        if let DataType::Decimal(p, s) = v.dtype() {
72                            prec = prec.max(p);
73                            scale = scale.max(s);
74                        }
75                    }
76                    DataType::Decimal(prec, scale)
77                },
78                dt => dt,
79            }
80        } else {
81            any_values_to_supertype(values)?
82        };
83
84        Self::from_any_values_and_dtype(name, values, &dtype, strict)
85    }
86
87    /// Construct a new [`Series`] with the given `dtype` from a slice of AnyValues.
88    ///
89    /// If `strict` is `true`, an error is returned if the values do not match the given
90    /// data type. If `strict` is `false`, values that do not match the given data type
91    /// are cast. If casting is not possible, the values are set to null instead.
92    pub fn from_any_values_and_dtype(
93        name: PlSmallStr,
94        values: &[AnyValue],
95        dtype: &DataType,
96        strict: bool,
97    ) -> PolarsResult<Self> {
98        if values.is_empty() {
99            return Ok(Self::new_empty(name, dtype));
100        }
101
102        let mut s = match dtype {
103            #[cfg(feature = "dtype-i8")]
104            DataType::Int8 => any_values_to_integer::<Int8Type>(values, strict)?.into_series(),
105            #[cfg(feature = "dtype-i16")]
106            DataType::Int16 => any_values_to_integer::<Int16Type>(values, strict)?.into_series(),
107            DataType::Int32 => any_values_to_integer::<Int32Type>(values, strict)?.into_series(),
108            DataType::Int64 => any_values_to_integer::<Int64Type>(values, strict)?.into_series(),
109            #[cfg(feature = "dtype-i128")]
110            DataType::Int128 => any_values_to_integer::<Int128Type>(values, strict)?.into_series(),
111            #[cfg(feature = "dtype-u8")]
112            DataType::UInt8 => any_values_to_integer::<UInt8Type>(values, strict)?.into_series(),
113            #[cfg(feature = "dtype-u16")]
114            DataType::UInt16 => any_values_to_integer::<UInt16Type>(values, strict)?.into_series(),
115            DataType::UInt32 => any_values_to_integer::<UInt32Type>(values, strict)?.into_series(),
116            DataType::UInt64 => any_values_to_integer::<UInt64Type>(values, strict)?.into_series(),
117            #[cfg(feature = "dtype-u128")]
118            DataType::UInt128 => {
119                any_values_to_integer::<UInt128Type>(values, strict)?.into_series()
120            },
121            DataType::Float32 => any_values_to_f32(values, strict)?.into_series(),
122            DataType::Float64 => any_values_to_f64(values, strict)?.into_series(),
123            DataType::Boolean => any_values_to_bool(values, strict)?.into_series(),
124            DataType::String => any_values_to_string(values, strict)?.into_series(),
125            DataType::Binary => any_values_to_binary(values, strict)?.into_series(),
126            DataType::BinaryOffset => any_values_to_binary_offset(values, strict)?.into_series(),
127            #[cfg(feature = "dtype-date")]
128            DataType::Date => any_values_to_date(values, strict)?.into_series(),
129            #[cfg(feature = "dtype-time")]
130            DataType::Time => any_values_to_time(values, strict)?.into_series(),
131            #[cfg(feature = "dtype-datetime")]
132            DataType::Datetime(tu, tz) => {
133                any_values_to_datetime(values, *tu, (*tz).clone(), strict)?.into_series()
134            },
135            #[cfg(feature = "dtype-duration")]
136            DataType::Duration(tu) => any_values_to_duration(values, *tu, strict)?.into_series(),
137            #[cfg(feature = "dtype-categorical")]
138            dt @ (DataType::Categorical(_, _) | DataType::Enum(_, _)) => {
139                any_values_to_categorical(values, dt, strict)?
140            },
141            #[cfg(feature = "dtype-decimal")]
142            DataType::Decimal(precision, scale) => {
143                any_values_to_decimal(values, *precision, *scale, strict)?.into_series()
144            },
145            #[cfg(feature = "dtype-extension")]
146            DataType::Extension(typ, storage) => {
147                Series::from_any_values_and_dtype(name.clone(), values, storage, strict)?
148                    .into_extension(typ.clone())
149            },
150            DataType::List(inner) => any_values_to_list(values, inner, strict)?.into_series(),
151            #[cfg(feature = "dtype-array")]
152            DataType::Array(inner, size) => any_values_to_array(values, inner, strict, *size)?
153                .into_series()
154                .cast(&DataType::Array(inner.clone(), *size))?,
155            #[cfg(feature = "dtype-struct")]
156            DataType::Struct(fields) => any_values_to_struct(values, fields, strict)?,
157            #[cfg(feature = "object")]
158            DataType::Object(_) => any_values_to_object(values)?,
159            DataType::Null => Series::new_null(PlSmallStr::EMPTY, values.len()),
160            dt => {
161                polars_bail!(
162                    InvalidOperation:
163                    "constructing a Series with data type {dt:?} from AnyValues is not supported"
164                )
165            },
166        };
167        s.rename(name);
168        Ok(s)
169    }
170}
171
172fn any_values_to_primitive_nonstrict<T: PolarsNumericType>(values: &[AnyValue]) -> ChunkedArray<T> {
173    values
174        .iter()
175        .map(|av| av.extract::<T::Native>())
176        .collect_trusted()
177}
178
179fn any_values_to_integer<T: PolarsIntegerType>(
180    values: &[AnyValue],
181    strict: bool,
182) -> PolarsResult<ChunkedArray<T>> {
183    fn any_values_to_integer_strict<T: PolarsIntegerType>(
184        values: &[AnyValue],
185    ) -> PolarsResult<ChunkedArray<T>> {
186        let mut builder = PrimitiveChunkedBuilder::<T>::new(PlSmallStr::EMPTY, values.len());
187        for av in values {
188            match &av {
189                av if av.is_integer() => {
190                    let opt_val = av.extract::<T::Native>();
191                    let val = match opt_val {
192                        Some(v) => v,
193                        None => return Err(invalid_value_error(&T::get_static_dtype(), av)),
194                    };
195                    builder.append_value(val)
196                },
197                AnyValue::Null => builder.append_null(),
198                av => return Err(invalid_value_error(&T::get_static_dtype(), av)),
199            }
200        }
201        Ok(builder.finish())
202    }
203
204    if strict {
205        any_values_to_integer_strict::<T>(values)
206    } else {
207        Ok(any_values_to_primitive_nonstrict::<T>(values))
208    }
209}
210
211fn any_values_to_f32(values: &[AnyValue], strict: bool) -> PolarsResult<Float32Chunked> {
212    fn any_values_to_f32_strict(values: &[AnyValue]) -> PolarsResult<Float32Chunked> {
213        let mut builder =
214            PrimitiveChunkedBuilder::<Float32Type>::new(PlSmallStr::EMPTY, values.len());
215        for av in values {
216            match av {
217                AnyValue::Float32(i) => builder.append_value(*i),
218                AnyValue::Null => builder.append_null(),
219                av => return Err(invalid_value_error(&DataType::Float32, av)),
220            }
221        }
222        Ok(builder.finish())
223    }
224    if strict {
225        any_values_to_f32_strict(values)
226    } else {
227        Ok(any_values_to_primitive_nonstrict::<Float32Type>(values))
228    }
229}
230fn any_values_to_f64(values: &[AnyValue], strict: bool) -> PolarsResult<Float64Chunked> {
231    fn any_values_to_f64_strict(values: &[AnyValue]) -> PolarsResult<Float64Chunked> {
232        let mut builder =
233            PrimitiveChunkedBuilder::<Float64Type>::new(PlSmallStr::EMPTY, values.len());
234        for av in values {
235            match av {
236                AnyValue::Float64(i) => builder.append_value(*i),
237                AnyValue::Float32(i) => builder.append_value(*i as f64),
238                AnyValue::Null => builder.append_null(),
239                av => return Err(invalid_value_error(&DataType::Float64, av)),
240            }
241        }
242        Ok(builder.finish())
243    }
244    if strict {
245        any_values_to_f64_strict(values)
246    } else {
247        Ok(any_values_to_primitive_nonstrict::<Float64Type>(values))
248    }
249}
250
251fn any_values_to_bool(values: &[AnyValue], strict: bool) -> PolarsResult<BooleanChunked> {
252    let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
253    for av in values {
254        match av {
255            AnyValue::Boolean(b) => builder.append_value(*b),
256            AnyValue::Null => builder.append_null(),
257            av => {
258                if strict {
259                    return Err(invalid_value_error(&DataType::Boolean, av));
260                }
261                match av.cast(&DataType::Boolean) {
262                    AnyValue::Boolean(b) => builder.append_value(b),
263                    _ => builder.append_null(),
264                }
265            },
266        }
267    }
268    Ok(builder.finish())
269}
270
271fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult<StringChunked> {
272    fn any_values_to_string_strict(values: &[AnyValue]) -> PolarsResult<StringChunked> {
273        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
274        for av in values {
275            match av {
276                AnyValue::String(s) => builder.append_value(s),
277                AnyValue::StringOwned(s) => builder.append_value(s),
278                AnyValue::Null => builder.append_null(),
279                av => return Err(invalid_value_error(&DataType::String, av)),
280            }
281        }
282        Ok(builder.finish())
283    }
284    fn any_values_to_string_nonstrict(values: &[AnyValue]) -> StringChunked {
285        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
286        let mut owned = String::new(); // Amortize allocations.
287        for av in values {
288            match av {
289                AnyValue::String(s) => builder.append_value(s),
290                AnyValue::StringOwned(s) => builder.append_value(s),
291                AnyValue::Null => builder.append_null(),
292                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(),
293                av => {
294                    owned.clear();
295                    write!(owned, "{av}").unwrap();
296                    builder.append_value(&owned);
297                },
298            }
299        }
300        builder.finish()
301    }
302    if strict {
303        any_values_to_string_strict(values)
304    } else {
305        Ok(any_values_to_string_nonstrict(values))
306    }
307}
308
309fn any_values_to_binary(values: &[AnyValue], strict: bool) -> PolarsResult<BinaryChunked> {
310    fn any_values_to_binary_strict(values: &[AnyValue]) -> PolarsResult<BinaryChunked> {
311        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
312        for av in values {
313            match av {
314                AnyValue::Binary(s) => builder.append_value(*s),
315                AnyValue::BinaryOwned(s) => builder.append_value(&**s),
316                AnyValue::Null => builder.append_null(),
317                av => return Err(invalid_value_error(&DataType::Binary, av)),
318            }
319        }
320        Ok(builder.finish())
321    }
322    fn any_values_to_binary_nonstrict(values: &[AnyValue]) -> BinaryChunked {
323        values
324            .iter()
325            .map(|av| match av {
326                AnyValue::Binary(b) => Some(*b),
327                AnyValue::BinaryOwned(b) => Some(&**b),
328                AnyValue::String(s) => Some(s.as_bytes()),
329                AnyValue::StringOwned(s) => Some(s.as_str().as_bytes()),
330                _ => None,
331            })
332            .collect_trusted()
333    }
334    if strict {
335        any_values_to_binary_strict(values)
336    } else {
337        Ok(any_values_to_binary_nonstrict(values))
338    }
339}
340
341fn any_values_to_binary_offset(
342    values: &[AnyValue],
343    strict: bool,
344) -> PolarsResult<BinaryOffsetChunked> {
345    let mut builder = MutableBinaryArray::<i64>::new();
346    for av in values {
347        match av {
348            AnyValue::Binary(s) => builder.push(Some(*s)),
349            AnyValue::BinaryOwned(s) => builder.push(Some(&**s)),
350            AnyValue::Null => builder.push_null(),
351            av => {
352                if strict {
353                    return Err(invalid_value_error(&DataType::Binary, av));
354                } else {
355                    builder.push_null();
356                };
357            },
358        }
359    }
360    Ok(BinaryOffsetChunked::with_chunk(
361        Default::default(),
362        builder.into(),
363    ))
364}
365
366#[cfg(feature = "dtype-date")]
367fn any_values_to_date(values: &[AnyValue], strict: bool) -> PolarsResult<DateChunked> {
368    let mut builder = PrimitiveChunkedBuilder::<Int32Type>::new(PlSmallStr::EMPTY, values.len());
369    for av in values {
370        match av {
371            AnyValue::Date(i) => builder.append_value(*i),
372            AnyValue::Null => builder.append_null(),
373            av => {
374                if strict {
375                    return Err(invalid_value_error(&DataType::Date, av));
376                }
377                match av.cast(&DataType::Date) {
378                    AnyValue::Date(i) => builder.append_value(i),
379                    _ => builder.append_null(),
380                }
381            },
382        }
383    }
384    Ok(builder.finish().into_date())
385}
386
387#[cfg(feature = "dtype-time")]
388fn any_values_to_time(values: &[AnyValue], strict: bool) -> PolarsResult<TimeChunked> {
389    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
390    for av in values {
391        match av {
392            AnyValue::Time(i) => builder.append_value(*i),
393            AnyValue::Null => builder.append_null(),
394            av => {
395                if strict {
396                    return Err(invalid_value_error(&DataType::Time, av));
397                }
398                match av.cast(&DataType::Time) {
399                    AnyValue::Time(i) => builder.append_value(i),
400                    _ => builder.append_null(),
401                }
402            },
403        }
404    }
405    Ok(builder.finish().into_time())
406}
407
408#[cfg(feature = "dtype-datetime")]
409fn any_values_to_datetime(
410    values: &[AnyValue],
411    time_unit: TimeUnit,
412    time_zone: Option<TimeZone>,
413    strict: bool,
414) -> PolarsResult<DatetimeChunked> {
415    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
416    let target_dtype = DataType::Datetime(time_unit, time_zone.clone());
417    for av in values {
418        match av {
419            AnyValue::Datetime(i, tu, _) if *tu == time_unit => builder.append_value(*i),
420            AnyValue::DatetimeOwned(i, tu, _) if *tu == time_unit => builder.append_value(*i),
421            AnyValue::Null => builder.append_null(),
422            av => {
423                if strict {
424                    return Err(invalid_value_error(&target_dtype, av));
425                }
426                match av.cast(&target_dtype) {
427                    AnyValue::Datetime(i, _, _) => builder.append_value(i),
428                    AnyValue::DatetimeOwned(i, _, _) => builder.append_value(i),
429                    _ => builder.append_null(),
430                }
431            },
432        }
433    }
434    Ok(builder.finish().into_datetime(time_unit, time_zone))
435}
436
437#[cfg(feature = "dtype-duration")]
438fn any_values_to_duration(
439    values: &[AnyValue],
440    time_unit: TimeUnit,
441    strict: bool,
442) -> PolarsResult<DurationChunked> {
443    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
444    let target_dtype = DataType::Duration(time_unit);
445    for av in values {
446        match av {
447            AnyValue::Duration(i, tu) if *tu == time_unit => builder.append_value(*i),
448            AnyValue::Null => builder.append_null(),
449            av => {
450                if strict {
451                    return Err(invalid_value_error(&target_dtype, av));
452                }
453                match av.cast(&target_dtype) {
454                    AnyValue::Duration(i, _) => builder.append_value(i),
455                    _ => builder.append_null(),
456                }
457            },
458        }
459    }
460    Ok(builder.finish().into_duration(time_unit))
461}
462
463#[cfg(feature = "dtype-categorical")]
464fn any_values_to_categorical(
465    values: &[AnyValue],
466    dtype: &DataType,
467    strict: bool,
468) -> PolarsResult<Series> {
469    with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
470        let mut builder = CategoricalChunkedBuilder::<$C>::new(PlSmallStr::EMPTY, dtype.clone());
471
472        let mut owned = String::new(); // Amortize allocations.
473        for av in values {
474            let ret = match av {
475                AnyValue::String(s) => builder.append_str(s),
476                AnyValue::StringOwned(s) => builder.append_str(s),
477
478                &AnyValue::Enum(cat, &ref map) |
479                &AnyValue::EnumOwned(cat, ref map) |
480                &AnyValue::Categorical(cat, &ref map) |
481                &AnyValue::CategoricalOwned(cat, ref map) => builder.append_cat(cat, map),
482
483                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) if !strict => {
484                    builder.append_null();
485                    Ok(())
486                },
487                AnyValue::Null => {
488                    builder.append_null();
489                    Ok(())
490                }
491
492                av => {
493                    if strict {
494                        return Err(invalid_value_error(&DataType::String, av));
495                    }
496
497                    owned.clear();
498                    write!(owned, "{av}").unwrap();
499                    builder.append_str(&owned)
500                },
501            };
502
503            if let Err(e) = ret {
504                if strict {
505                    return Err(e);
506                } else {
507                    builder.append_null();
508                }
509            }
510        }
511
512        let ca = builder.finish();
513        Ok(ca.into_series())
514    })
515}
516
517#[cfg(feature = "dtype-decimal")]
518fn any_values_to_decimal(
519    values: &[AnyValue],
520    precision: usize,
521    scale: usize,
522    strict: bool,
523) -> PolarsResult<DecimalChunked> {
524    let target_dtype = DataType::Decimal(precision, scale);
525
526    let mut builder = PrimitiveChunkedBuilder::<Int128Type>::new(PlSmallStr::EMPTY, values.len());
527    for av in values {
528        match av {
529            // Allow equal or less scale. We do want to support different scales even in 'strict' mode.
530            AnyValue::Decimal(v, p, s) if *s <= scale => {
531                if *p <= precision && *s == scale {
532                    builder.append_value(*v)
533                } else {
534                    match av.strict_cast(&target_dtype) {
535                        Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
536                        _ => builder.append_null(),
537                    }
538                }
539            },
540            AnyValue::Null => builder.append_null(),
541            av => {
542                if strict {
543                    return Err(invalid_value_error(&target_dtype, av));
544                }
545                match av.strict_cast(&target_dtype) {
546                    Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
547                    _ => builder.append_null(),
548                }
549            },
550        };
551    }
552
553    // Build the array and do a precision check if needed.
554    builder.finish().into_decimal(precision, scale)
555}
556
557fn any_values_to_list(
558    avs: &[AnyValue],
559    inner_type: &DataType,
560    strict: bool,
561) -> PolarsResult<ListChunked> {
562    // GB:
563    // Lord forgive for the sins I have committed in this function. The amount of strange
564    // exceptions that need to happen for this to work are insane and I feel like I am going crazy.
565    //
566    // This function is essentially a copy of the `<ListChunked as FromIterator>` where it does not
567    // sample the datatype from the first element and instead we give it explicitly. This allows
568    // this function to properly assign a datatype if `avs` starts with a `null` value. Previously,
569    // this was solved by assigning the `dtype` again afterwards, but why? We should not link the
570    // implementation of these functions. We still need to assign the dtype of the ListArray and
571    // such, anyways.
572    //
573    // Then, `collect_ca_with_dtype` does not possess the necessary exceptions shown in this
574    // function to use that. I have tried adding the exceptions there and it broke other things. I
575    // really do feel like this is the simplest solution.
576
577    let mut valid = true;
578    let capacity = avs.len();
579
580    let ca = match inner_type {
581        // AnyValues with empty lists in python can create
582        // Series of an unknown dtype.
583        // We use the anonymousbuilder without a dtype
584        // the empty arrays is then not added (we add an extra offset instead)
585        // the next non-empty series then must have the correct dtype.
586        DataType::Null => {
587            let mut builder = AnonymousOwnedListBuilder::new(PlSmallStr::EMPTY, capacity, None);
588            for av in avs {
589                match av {
590                    AnyValue::List(b) => builder.append_series(b)?,
591                    AnyValue::Null => builder.append_null(),
592                    _ => {
593                        valid = false;
594                        builder.append_null();
595                    },
596                }
597            }
598            builder.finish()
599        },
600
601        #[cfg(feature = "object")]
602        DataType::Object(_) => polars_bail!(nyi = "Nested object types"),
603
604        _ => {
605            let mut builder =
606                get_list_builder(inner_type, capacity * 5, capacity, PlSmallStr::EMPTY);
607            for av in avs {
608                match av {
609                    AnyValue::List(b) => match b.cast(inner_type) {
610                        Ok(casted) => {
611                            if casted.null_count() != b.null_count() {
612                                valid = !strict;
613                            }
614                            builder.append_series(&casted)?;
615                        },
616                        Err(_) => {
617                            valid = false;
618                            for _ in 0..b.len() {
619                                builder.append_null();
620                            }
621                        },
622                    },
623                    AnyValue::Null => builder.append_null(),
624                    _ => {
625                        valid = false;
626                        builder.append_null()
627                    },
628                }
629            }
630
631            builder.finish()
632        },
633    };
634
635    if strict && !valid {
636        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", DataType::List(Box::new(inner_type.clone())));
637    }
638
639    Ok(ca)
640}
641
642#[cfg(feature = "dtype-array")]
643fn any_values_to_array(
644    avs: &[AnyValue],
645    inner_type: &DataType,
646    strict: bool,
647    width: usize,
648) -> PolarsResult<ArrayChunked> {
649    fn to_arr(s: &Series) -> Option<ArrayRef> {
650        if s.chunks().len() > 1 {
651            let s = s.rechunk();
652            Some(s.chunks()[0].clone())
653        } else {
654            Some(s.chunks()[0].clone())
655        }
656    }
657
658    let target_dtype = DataType::Array(Box::new(inner_type.clone()), width);
659
660    // This is handled downstream. The builder will choose the first non null type.
661    let mut valid = true;
662    #[allow(unused_mut)]
663    let mut out: ArrayChunked = if inner_type == &DataType::Null {
664        avs.iter()
665            .map(|av| match av {
666                AnyValue::List(b) | AnyValue::Array(b, _) => to_arr(b),
667                AnyValue::Null => None,
668                _ => {
669                    valid = false;
670                    None
671                },
672            })
673            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
674    }
675    // Make sure that wrongly inferred AnyValues don't deviate from the datatype.
676    else {
677        avs.iter()
678            .map(|av| match av {
679                AnyValue::List(b) | AnyValue::Array(b, _) => {
680                    if b.dtype() == inner_type {
681                        to_arr(b)
682                    } else {
683                        let s = match b.cast(inner_type) {
684                            Ok(out) => out,
685                            Err(_) => Series::full_null(b.name().clone(), b.len(), inner_type),
686                        };
687                        to_arr(&s)
688                    }
689                },
690                AnyValue::Null => None,
691                _ => {
692                    valid = false;
693                    None
694                },
695            })
696            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
697    };
698
699    if strict && !valid {
700        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", target_dtype);
701    }
702    polars_ensure!(
703        out.width() == width,
704        SchemaMismatch: "got mixed size array widths where width {} was expected", width
705    );
706
707    // Ensure the logical type is correct for nested types.
708    #[cfg(feature = "dtype-struct")]
709    if !matches!(inner_type, DataType::Null) && out.inner_dtype().is_nested() {
710        unsafe {
711            out.set_dtype(target_dtype);
712        };
713    }
714
715    Ok(out)
716}
717
718#[cfg(feature = "dtype-struct")]
719fn _any_values_to_struct<'a>(
720    av_fields: &[Field],
721    av_values: &[AnyValue<'a>],
722    field_index: usize,
723    field: &Field,
724    fields: &[Field],
725    field_avs: &mut Vec<AnyValue<'a>>,
726) {
727    // TODO: Optimize.
728
729    let mut append_by_search = || {
730        // Search for the name.
731        if let Some(i) = av_fields
732            .iter()
733            .position(|av_fld| av_fld.name == field.name)
734        {
735            field_avs.push(av_values[i].clone());
736            return;
737        }
738        field_avs.push(AnyValue::Null)
739    };
740
741    // All fields are available in this single value.
742    // We can use the index to get value.
743    if fields.len() == av_fields.len() {
744        if fields.iter().zip(av_fields.iter()).any(|(l, r)| l != r) {
745            append_by_search()
746        } else {
747            let av_val = av_values
748                .get(field_index)
749                .cloned()
750                .unwrap_or(AnyValue::Null);
751            field_avs.push(av_val)
752        }
753    }
754    // Not all fields are available, we search the proper field.
755    else {
756        // Search for the name.
757        append_by_search()
758    }
759}
760
761#[cfg(feature = "dtype-struct")]
762fn any_values_to_struct(
763    values: &[AnyValue],
764    fields: &[Field],
765    strict: bool,
766) -> PolarsResult<Series> {
767    // Fast path for structs with no fields.
768    if fields.is_empty() {
769        return Ok(
770            StructChunked::from_series(PlSmallStr::EMPTY, values.len(), [].iter())?.into_series(),
771        );
772    }
773
774    // The physical series fields of the struct.
775    let mut series_fields = Vec::with_capacity(fields.len());
776    let mut has_outer_validity = false;
777    let mut field_avs = Vec::with_capacity(values.len());
778    for (i, field) in fields.iter().enumerate() {
779        field_avs.clear();
780
781        for av in values.iter() {
782            match av {
783                AnyValue::StructOwned(payload) => {
784                    let av_fields = &payload.1;
785                    let av_values = &payload.0;
786                    _any_values_to_struct(av_fields, av_values, i, field, fields, &mut field_avs);
787                },
788                AnyValue::Struct(_, _, av_fields) => {
789                    let av_values: Vec<_> = av._iter_struct_av().collect();
790                    _any_values_to_struct(av_fields, &av_values, i, field, fields, &mut field_avs);
791                },
792                _ => {
793                    has_outer_validity = true;
794                    field_avs.push(AnyValue::Null)
795                },
796            }
797        }
798        // If the inferred dtype is null, we let auto inference work.
799        let s = if matches!(field.dtype, DataType::Null) {
800            Series::from_any_values(field.name().clone(), &field_avs, strict)?
801        } else {
802            Series::from_any_values_and_dtype(
803                field.name().clone(),
804                &field_avs,
805                &field.dtype,
806                strict,
807            )?
808        };
809        series_fields.push(s)
810    }
811
812    let mut out =
813        StructChunked::from_series(PlSmallStr::EMPTY, values.len(), series_fields.iter())?;
814    if has_outer_validity {
815        let mut validity = MutableBitmap::new();
816        validity.extend_constant(values.len(), true);
817        for (i, v) in values.iter().enumerate() {
818            if matches!(v, AnyValue::Null) {
819                unsafe { validity.set_unchecked(i, false) }
820            }
821        }
822        out.set_outer_validity(Some(validity.freeze()))
823    }
824    Ok(out.into_series())
825}
826
827#[cfg(feature = "object")]
828fn any_values_to_object(values: &[AnyValue]) -> PolarsResult<Series> {
829    use crate::chunked_array::object::registry;
830    let converter = registry::get_object_converter();
831    let mut builder = registry::get_object_builder(PlSmallStr::EMPTY, values.len());
832    for av in values {
833        match av {
834            AnyValue::Object(val) => builder.append_value(val.as_any()),
835            AnyValue::Null => builder.append_null(),
836            _ => {
837                // This is needed because in Python users can send mixed types.
838                // This only works if you set a global converter.
839                let any = converter(av.as_borrowed());
840                builder.append_value(&*any)
841            },
842        }
843    }
844
845    Ok(builder.to_series())
846}
847
848fn invalid_value_error(dtype: &DataType, value: &AnyValue) -> PolarsError {
849    polars_err!(
850        SchemaMismatch:
851        "unexpected value while building Series of type {:?}; found value of type {:?}: {}",
852        dtype,
853        value.dtype(),
854        value
855    )
856}
polars_core/series/any_value.rs

polars_core/series/
any_value.rs
