polars_core/series/
any_value.rs

1use std::fmt::Write;
2
3use arrow::bitmap::MutableBitmap;
4
5#[cfg(feature = "dtype-categorical")]
6use crate::chunked_array::builder::CategoricalChunkedBuilder;
7use crate::chunked_array::builder::{AnonymousOwnedListBuilder, get_list_builder};
8use crate::prelude::*;
9use crate::utils::any_values_to_supertype;
10
11impl<'a, T: AsRef<[AnyValue<'a>]>> NamedFrom<T, [AnyValue<'a>]> for Series {
12    /// Construct a new [`Series`] from a collection of [`AnyValue`].
13    ///
14    /// # Panics
15    ///
16    /// Panics if the values do not all share the same data type (with the exception
17    /// of [`DataType::Null`], which is always allowed).
18    ///
19    /// [`AnyValue`]: crate::datatypes::AnyValue
20    fn new(name: PlSmallStr, values: T) -> Self {
21        let values = values.as_ref();
22        Series::from_any_values(name, values, true).expect("data types of values should match")
23    }
24}
25
26impl Series {
27    /// Construct a new [`Series`] from a slice of AnyValues.
28    ///
29    /// The data type of the resulting Series is determined by the `values`
30    /// and the `strict` parameter:
31    /// - If `strict` is `true`, the data type is equal to the data type of the
32    ///   first non-null value. If any other non-null values do not match this
33    ///   data type, an error is raised. If the first non-null value is a
34    ///   decimal the slice is scanned for the maximum precision and scale possible.
35    /// - If `strict` is `false`, the data type is the supertype of the `values`.
36    ///   An error is returned if no supertype can be determined.
37    ///   **WARNING**: A full pass over the values is required to determine the supertype.
38    /// - If no values were passed, the resulting data type is `Null`.
39    pub fn from_any_values(
40        name: PlSmallStr,
41        values: &[AnyValue],
42        strict: bool,
43    ) -> PolarsResult<Self> {
44        fn get_first_non_null_dtype(values: &[AnyValue]) -> DataType {
45            let mut all_flat_null = true;
46            let first_non_null = values.iter().find(|av| {
47                if !av.is_null() {
48                    all_flat_null = false
49                };
50                !av.is_nested_null()
51            });
52            match first_non_null {
53                Some(av) => av.dtype(),
54                None => {
55                    if all_flat_null {
56                        DataType::Null
57                    } else {
58                        // Second pass to check for the nested null value that
59                        // toggled `all_flat_null` to false, e.g. a List(Null).
60                        let first_nested_null = values.iter().find(|av| !av.is_null()).unwrap();
61                        first_nested_null.dtype()
62                    }
63                },
64            }
65        }
66        let dtype = if strict {
67            match get_first_non_null_dtype(values) {
68                #[cfg(feature = "dtype-decimal")]
69                DataType::Decimal(mut prec, mut scale) => {
70                    for v in values {
71                        if let DataType::Decimal(p, s) = v.dtype() {
72                            prec = prec.max(p);
73                            scale = scale.max(s);
74                        }
75                    }
76                    DataType::Decimal(prec, scale)
77                },
78                dt => dt,
79            }
80        } else {
81            any_values_to_supertype(values)?
82        };
83
84        Self::from_any_values_and_dtype(name, values, &dtype, strict)
85    }
86
87    /// Construct a new [`Series`] with the given `dtype` from a slice of AnyValues.
88    ///
89    /// If `strict` is `true`, an error is returned if the values do not match the given
90    /// data type. If `strict` is `false`, values that do not match the given data type
91    /// are cast. If casting is not possible, the values are set to null instead.
92    pub fn from_any_values_and_dtype(
93        name: PlSmallStr,
94        values: &[AnyValue],
95        dtype: &DataType,
96        strict: bool,
97    ) -> PolarsResult<Self> {
98        if values.is_empty() {
99            return Ok(Self::new_empty(name, dtype));
100        }
101
102        let mut s = match dtype {
103            #[cfg(feature = "dtype-i8")]
104            DataType::Int8 => any_values_to_integer::<Int8Type>(values, strict)?.into_series(),
105            #[cfg(feature = "dtype-i16")]
106            DataType::Int16 => any_values_to_integer::<Int16Type>(values, strict)?.into_series(),
107            DataType::Int32 => any_values_to_integer::<Int32Type>(values, strict)?.into_series(),
108            DataType::Int64 => any_values_to_integer::<Int64Type>(values, strict)?.into_series(),
109            #[cfg(feature = "dtype-i128")]
110            DataType::Int128 => any_values_to_integer::<Int128Type>(values, strict)?.into_series(),
111            #[cfg(feature = "dtype-u8")]
112            DataType::UInt8 => any_values_to_integer::<UInt8Type>(values, strict)?.into_series(),
113            #[cfg(feature = "dtype-u16")]
114            DataType::UInt16 => any_values_to_integer::<UInt16Type>(values, strict)?.into_series(),
115            DataType::UInt32 => any_values_to_integer::<UInt32Type>(values, strict)?.into_series(),
116            DataType::UInt64 => any_values_to_integer::<UInt64Type>(values, strict)?.into_series(),
117            #[cfg(feature = "dtype-u128")]
118            DataType::UInt128 => {
119                any_values_to_integer::<UInt128Type>(values, strict)?.into_series()
120            },
121            DataType::Float32 => any_values_to_f32(values, strict)?.into_series(),
122            DataType::Float64 => any_values_to_f64(values, strict)?.into_series(),
123            DataType::Boolean => any_values_to_bool(values, strict)?.into_series(),
124            DataType::String => any_values_to_string(values, strict)?.into_series(),
125            DataType::Binary => any_values_to_binary(values, strict)?.into_series(),
126            DataType::BinaryOffset => any_values_to_binary_offset(values, strict)?.into_series(),
127            #[cfg(feature = "dtype-date")]
128            DataType::Date => any_values_to_date(values, strict)?.into_series(),
129            #[cfg(feature = "dtype-time")]
130            DataType::Time => any_values_to_time(values, strict)?.into_series(),
131            #[cfg(feature = "dtype-datetime")]
132            DataType::Datetime(tu, tz) => {
133                any_values_to_datetime(values, *tu, (*tz).clone(), strict)?.into_series()
134            },
135            #[cfg(feature = "dtype-duration")]
136            DataType::Duration(tu) => any_values_to_duration(values, *tu, strict)?.into_series(),
137            #[cfg(feature = "dtype-categorical")]
138            dt @ (DataType::Categorical(_, _) | DataType::Enum(_, _)) => {
139                any_values_to_categorical(values, dt, strict)?
140            },
141            #[cfg(feature = "dtype-decimal")]
142            DataType::Decimal(precision, scale) => {
143                any_values_to_decimal(values, *precision, *scale, strict)?.into_series()
144            },
145            DataType::List(inner) => any_values_to_list(values, inner, strict)?.into_series(),
146            #[cfg(feature = "dtype-array")]
147            DataType::Array(inner, size) => any_values_to_array(values, inner, strict, *size)?
148                .into_series()
149                .cast(&DataType::Array(inner.clone(), *size))?,
150            #[cfg(feature = "dtype-struct")]
151            DataType::Struct(fields) => any_values_to_struct(values, fields, strict)?,
152            #[cfg(feature = "object")]
153            DataType::Object(_) => any_values_to_object(values)?,
154            DataType::Null => Series::new_null(PlSmallStr::EMPTY, values.len()),
155            dt => {
156                polars_bail!(
157                    InvalidOperation:
158                    "constructing a Series with data type {dt:?} from AnyValues is not supported"
159                )
160            },
161        };
162        s.rename(name);
163        Ok(s)
164    }
165}
166
167fn any_values_to_primitive_nonstrict<T: PolarsNumericType>(values: &[AnyValue]) -> ChunkedArray<T> {
168    values
169        .iter()
170        .map(|av| av.extract::<T::Native>())
171        .collect_trusted()
172}
173
174fn any_values_to_integer<T: PolarsIntegerType>(
175    values: &[AnyValue],
176    strict: bool,
177) -> PolarsResult<ChunkedArray<T>> {
178    fn any_values_to_integer_strict<T: PolarsIntegerType>(
179        values: &[AnyValue],
180    ) -> PolarsResult<ChunkedArray<T>> {
181        let mut builder = PrimitiveChunkedBuilder::<T>::new(PlSmallStr::EMPTY, values.len());
182        for av in values {
183            match &av {
184                av if av.is_integer() => {
185                    let opt_val = av.extract::<T::Native>();
186                    let val = match opt_val {
187                        Some(v) => v,
188                        None => return Err(invalid_value_error(&T::get_static_dtype(), av)),
189                    };
190                    builder.append_value(val)
191                },
192                AnyValue::Null => builder.append_null(),
193                av => return Err(invalid_value_error(&T::get_static_dtype(), av)),
194            }
195        }
196        Ok(builder.finish())
197    }
198
199    if strict {
200        any_values_to_integer_strict::<T>(values)
201    } else {
202        Ok(any_values_to_primitive_nonstrict::<T>(values))
203    }
204}
205
206fn any_values_to_f32(values: &[AnyValue], strict: bool) -> PolarsResult<Float32Chunked> {
207    fn any_values_to_f32_strict(values: &[AnyValue]) -> PolarsResult<Float32Chunked> {
208        let mut builder =
209            PrimitiveChunkedBuilder::<Float32Type>::new(PlSmallStr::EMPTY, values.len());
210        for av in values {
211            match av {
212                AnyValue::Float32(i) => builder.append_value(*i),
213                AnyValue::Null => builder.append_null(),
214                av => return Err(invalid_value_error(&DataType::Float32, av)),
215            }
216        }
217        Ok(builder.finish())
218    }
219    if strict {
220        any_values_to_f32_strict(values)
221    } else {
222        Ok(any_values_to_primitive_nonstrict::<Float32Type>(values))
223    }
224}
225fn any_values_to_f64(values: &[AnyValue], strict: bool) -> PolarsResult<Float64Chunked> {
226    fn any_values_to_f64_strict(values: &[AnyValue]) -> PolarsResult<Float64Chunked> {
227        let mut builder =
228            PrimitiveChunkedBuilder::<Float64Type>::new(PlSmallStr::EMPTY, values.len());
229        for av in values {
230            match av {
231                AnyValue::Float64(i) => builder.append_value(*i),
232                AnyValue::Float32(i) => builder.append_value(*i as f64),
233                AnyValue::Null => builder.append_null(),
234                av => return Err(invalid_value_error(&DataType::Float64, av)),
235            }
236        }
237        Ok(builder.finish())
238    }
239    if strict {
240        any_values_to_f64_strict(values)
241    } else {
242        Ok(any_values_to_primitive_nonstrict::<Float64Type>(values))
243    }
244}
245
246fn any_values_to_bool(values: &[AnyValue], strict: bool) -> PolarsResult<BooleanChunked> {
247    let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
248    for av in values {
249        match av {
250            AnyValue::Boolean(b) => builder.append_value(*b),
251            AnyValue::Null => builder.append_null(),
252            av => {
253                if strict {
254                    return Err(invalid_value_error(&DataType::Boolean, av));
255                }
256                match av.cast(&DataType::Boolean) {
257                    AnyValue::Boolean(b) => builder.append_value(b),
258                    _ => builder.append_null(),
259                }
260            },
261        }
262    }
263    Ok(builder.finish())
264}
265
266fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult<StringChunked> {
267    fn any_values_to_string_strict(values: &[AnyValue]) -> PolarsResult<StringChunked> {
268        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
269        for av in values {
270            match av {
271                AnyValue::String(s) => builder.append_value(s),
272                AnyValue::StringOwned(s) => builder.append_value(s),
273                AnyValue::Null => builder.append_null(),
274                av => return Err(invalid_value_error(&DataType::String, av)),
275            }
276        }
277        Ok(builder.finish())
278    }
279    fn any_values_to_string_nonstrict(values: &[AnyValue]) -> StringChunked {
280        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
281        let mut owned = String::new(); // Amortize allocations.
282        for av in values {
283            match av {
284                AnyValue::String(s) => builder.append_value(s),
285                AnyValue::StringOwned(s) => builder.append_value(s),
286                AnyValue::Null => builder.append_null(),
287                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(),
288                av => {
289                    owned.clear();
290                    write!(owned, "{av}").unwrap();
291                    builder.append_value(&owned);
292                },
293            }
294        }
295        builder.finish()
296    }
297    if strict {
298        any_values_to_string_strict(values)
299    } else {
300        Ok(any_values_to_string_nonstrict(values))
301    }
302}
303
304fn any_values_to_binary(values: &[AnyValue], strict: bool) -> PolarsResult<BinaryChunked> {
305    fn any_values_to_binary_strict(values: &[AnyValue]) -> PolarsResult<BinaryChunked> {
306        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
307        for av in values {
308            match av {
309                AnyValue::Binary(s) => builder.append_value(*s),
310                AnyValue::BinaryOwned(s) => builder.append_value(&**s),
311                AnyValue::Null => builder.append_null(),
312                av => return Err(invalid_value_error(&DataType::Binary, av)),
313            }
314        }
315        Ok(builder.finish())
316    }
317    fn any_values_to_binary_nonstrict(values: &[AnyValue]) -> BinaryChunked {
318        values
319            .iter()
320            .map(|av| match av {
321                AnyValue::Binary(b) => Some(*b),
322                AnyValue::BinaryOwned(b) => Some(&**b),
323                AnyValue::String(s) => Some(s.as_bytes()),
324                AnyValue::StringOwned(s) => Some(s.as_str().as_bytes()),
325                _ => None,
326            })
327            .collect_trusted()
328    }
329    if strict {
330        any_values_to_binary_strict(values)
331    } else {
332        Ok(any_values_to_binary_nonstrict(values))
333    }
334}
335
336fn any_values_to_binary_offset(
337    values: &[AnyValue],
338    strict: bool,
339) -> PolarsResult<BinaryOffsetChunked> {
340    let mut builder = MutableBinaryArray::<i64>::new();
341    for av in values {
342        match av {
343            AnyValue::Binary(s) => builder.push(Some(*s)),
344            AnyValue::BinaryOwned(s) => builder.push(Some(&**s)),
345            AnyValue::Null => builder.push_null(),
346            av => {
347                if strict {
348                    return Err(invalid_value_error(&DataType::Binary, av));
349                } else {
350                    builder.push_null();
351                };
352            },
353        }
354    }
355    Ok(BinaryOffsetChunked::with_chunk(
356        Default::default(),
357        builder.into(),
358    ))
359}
360
361#[cfg(feature = "dtype-date")]
362fn any_values_to_date(values: &[AnyValue], strict: bool) -> PolarsResult<DateChunked> {
363    let mut builder = PrimitiveChunkedBuilder::<Int32Type>::new(PlSmallStr::EMPTY, values.len());
364    for av in values {
365        match av {
366            AnyValue::Date(i) => builder.append_value(*i),
367            AnyValue::Null => builder.append_null(),
368            av => {
369                if strict {
370                    return Err(invalid_value_error(&DataType::Date, av));
371                }
372                match av.cast(&DataType::Date) {
373                    AnyValue::Date(i) => builder.append_value(i),
374                    _ => builder.append_null(),
375                }
376            },
377        }
378    }
379    Ok(builder.finish().into_date())
380}
381
382#[cfg(feature = "dtype-time")]
383fn any_values_to_time(values: &[AnyValue], strict: bool) -> PolarsResult<TimeChunked> {
384    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
385    for av in values {
386        match av {
387            AnyValue::Time(i) => builder.append_value(*i),
388            AnyValue::Null => builder.append_null(),
389            av => {
390                if strict {
391                    return Err(invalid_value_error(&DataType::Time, av));
392                }
393                match av.cast(&DataType::Time) {
394                    AnyValue::Time(i) => builder.append_value(i),
395                    _ => builder.append_null(),
396                }
397            },
398        }
399    }
400    Ok(builder.finish().into_time())
401}
402
403#[cfg(feature = "dtype-datetime")]
404fn any_values_to_datetime(
405    values: &[AnyValue],
406    time_unit: TimeUnit,
407    time_zone: Option<TimeZone>,
408    strict: bool,
409) -> PolarsResult<DatetimeChunked> {
410    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
411    let target_dtype = DataType::Datetime(time_unit, time_zone.clone());
412    for av in values {
413        match av {
414            AnyValue::Datetime(i, tu, _) if *tu == time_unit => builder.append_value(*i),
415            AnyValue::DatetimeOwned(i, tu, _) if *tu == time_unit => builder.append_value(*i),
416            AnyValue::Null => builder.append_null(),
417            av => {
418                if strict {
419                    return Err(invalid_value_error(&target_dtype, av));
420                }
421                match av.cast(&target_dtype) {
422                    AnyValue::Datetime(i, _, _) => builder.append_value(i),
423                    AnyValue::DatetimeOwned(i, _, _) => builder.append_value(i),
424                    _ => builder.append_null(),
425                }
426            },
427        }
428    }
429    Ok(builder.finish().into_datetime(time_unit, time_zone))
430}
431
432#[cfg(feature = "dtype-duration")]
433fn any_values_to_duration(
434    values: &[AnyValue],
435    time_unit: TimeUnit,
436    strict: bool,
437) -> PolarsResult<DurationChunked> {
438    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
439    let target_dtype = DataType::Duration(time_unit);
440    for av in values {
441        match av {
442            AnyValue::Duration(i, tu) if *tu == time_unit => builder.append_value(*i),
443            AnyValue::Null => builder.append_null(),
444            av => {
445                if strict {
446                    return Err(invalid_value_error(&target_dtype, av));
447                }
448                match av.cast(&target_dtype) {
449                    AnyValue::Duration(i, _) => builder.append_value(i),
450                    _ => builder.append_null(),
451                }
452            },
453        }
454    }
455    Ok(builder.finish().into_duration(time_unit))
456}
457
458#[cfg(feature = "dtype-categorical")]
459fn any_values_to_categorical(
460    values: &[AnyValue],
461    dtype: &DataType,
462    strict: bool,
463) -> PolarsResult<Series> {
464    with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
465        let mut builder = CategoricalChunkedBuilder::<$C>::new(PlSmallStr::EMPTY, dtype.clone());
466
467        let mut owned = String::new(); // Amortize allocations.
468        for av in values {
469            let ret = match av {
470                AnyValue::String(s) => builder.append_str(s),
471                AnyValue::StringOwned(s) => builder.append_str(s),
472
473                &AnyValue::Enum(cat, &ref map) |
474                &AnyValue::EnumOwned(cat, ref map) |
475                &AnyValue::Categorical(cat, &ref map) |
476                &AnyValue::CategoricalOwned(cat, ref map) => builder.append_cat(cat, map),
477
478                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) if !strict => {
479                    builder.append_null();
480                    Ok(())
481                },
482                AnyValue::Null => {
483                    builder.append_null();
484                    Ok(())
485                }
486
487                av => {
488                    if strict {
489                        return Err(invalid_value_error(&DataType::String, av));
490                    }
491
492                    owned.clear();
493                    write!(owned, "{av}").unwrap();
494                    builder.append_str(&owned)
495                },
496            };
497
498            if let Err(e) = ret {
499                if strict {
500                    return Err(e);
501                } else {
502                    builder.append_null();
503                }
504            }
505        }
506
507        let ca = builder.finish();
508        Ok(ca.into_series())
509    })
510}
511
512#[cfg(feature = "dtype-decimal")]
513fn any_values_to_decimal(
514    values: &[AnyValue],
515    precision: usize,
516    scale: usize,
517    strict: bool,
518) -> PolarsResult<DecimalChunked> {
519    let target_dtype = DataType::Decimal(precision, scale);
520
521    let mut builder = PrimitiveChunkedBuilder::<Int128Type>::new(PlSmallStr::EMPTY, values.len());
522    for av in values {
523        match av {
524            // Allow equal or less scale. We do want to support different scales even in 'strict' mode.
525            AnyValue::Decimal(v, p, s) if *s <= scale => {
526                if *p <= precision && *s == scale {
527                    builder.append_value(*v)
528                } else {
529                    match av.strict_cast(&target_dtype) {
530                        Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
531                        _ => builder.append_null(),
532                    }
533                }
534            },
535            AnyValue::Null => builder.append_null(),
536            av => {
537                if strict {
538                    return Err(invalid_value_error(&target_dtype, av));
539                }
540                match av.strict_cast(&target_dtype) {
541                    Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
542                    _ => builder.append_null(),
543                }
544            },
545        };
546    }
547
548    // Build the array and do a precision check if needed.
549    builder.finish().into_decimal(precision, scale)
550}
551
552fn any_values_to_list(
553    avs: &[AnyValue],
554    inner_type: &DataType,
555    strict: bool,
556) -> PolarsResult<ListChunked> {
557    // GB:
558    // Lord forgive for the sins I have committed in this function. The amount of strange
559    // exceptions that need to happen for this to work are insane and I feel like I am going crazy.
560    //
561    // This function is essentially a copy of the `<ListChunked as FromIterator>` where it does not
562    // sample the datatype from the first element and instead we give it explicitly. This allows
563    // this function to properly assign a datatype if `avs` starts with a `null` value. Previously,
564    // this was solved by assigning the `dtype` again afterwards, but why? We should not link the
565    // implementation of these functions. We still need to assign the dtype of the ListArray and
566    // such, anyways.
567    //
568    // Then, `collect_ca_with_dtype` does not possess the necessary exceptions shown in this
569    // function to use that. I have tried adding the exceptions there and it broke other things. I
570    // really do feel like this is the simplest solution.
571
572    let mut valid = true;
573    let capacity = avs.len();
574
575    let ca = match inner_type {
576        // AnyValues with empty lists in python can create
577        // Series of an unknown dtype.
578        // We use the anonymousbuilder without a dtype
579        // the empty arrays is then not added (we add an extra offset instead)
580        // the next non-empty series then must have the correct dtype.
581        DataType::Null => {
582            let mut builder = AnonymousOwnedListBuilder::new(PlSmallStr::EMPTY, capacity, None);
583            for av in avs {
584                match av {
585                    AnyValue::List(b) => builder.append_series(b)?,
586                    AnyValue::Null => builder.append_null(),
587                    _ => {
588                        valid = false;
589                        builder.append_null();
590                    },
591                }
592            }
593            builder.finish()
594        },
595
596        #[cfg(feature = "object")]
597        DataType::Object(_) => polars_bail!(nyi = "Nested object types"),
598
599        _ => {
600            let mut builder =
601                get_list_builder(inner_type, capacity * 5, capacity, PlSmallStr::EMPTY);
602            for av in avs {
603                match av {
604                    AnyValue::List(b) => match b.cast(inner_type) {
605                        Ok(casted) => {
606                            if casted.null_count() != b.null_count() {
607                                valid = !strict;
608                            }
609                            builder.append_series(&casted)?;
610                        },
611                        Err(_) => {
612                            valid = false;
613                            for _ in 0..b.len() {
614                                builder.append_null();
615                            }
616                        },
617                    },
618                    AnyValue::Null => builder.append_null(),
619                    _ => {
620                        valid = false;
621                        builder.append_null()
622                    },
623                }
624            }
625
626            builder.finish()
627        },
628    };
629
630    if strict && !valid {
631        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", DataType::List(Box::new(inner_type.clone())));
632    }
633
634    Ok(ca)
635}
636
637#[cfg(feature = "dtype-array")]
638fn any_values_to_array(
639    avs: &[AnyValue],
640    inner_type: &DataType,
641    strict: bool,
642    width: usize,
643) -> PolarsResult<ArrayChunked> {
644    fn to_arr(s: &Series) -> Option<ArrayRef> {
645        if s.chunks().len() > 1 {
646            let s = s.rechunk();
647            Some(s.chunks()[0].clone())
648        } else {
649            Some(s.chunks()[0].clone())
650        }
651    }
652
653    let target_dtype = DataType::Array(Box::new(inner_type.clone()), width);
654
655    // This is handled downstream. The builder will choose the first non null type.
656    let mut valid = true;
657    #[allow(unused_mut)]
658    let mut out: ArrayChunked = if inner_type == &DataType::Null {
659        avs.iter()
660            .map(|av| match av {
661                AnyValue::List(b) | AnyValue::Array(b, _) => to_arr(b),
662                AnyValue::Null => None,
663                _ => {
664                    valid = false;
665                    None
666                },
667            })
668            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
669    }
670    // Make sure that wrongly inferred AnyValues don't deviate from the datatype.
671    else {
672        avs.iter()
673            .map(|av| match av {
674                AnyValue::List(b) | AnyValue::Array(b, _) => {
675                    if b.dtype() == inner_type {
676                        to_arr(b)
677                    } else {
678                        let s = match b.cast(inner_type) {
679                            Ok(out) => out,
680                            Err(_) => Series::full_null(b.name().clone(), b.len(), inner_type),
681                        };
682                        to_arr(&s)
683                    }
684                },
685                AnyValue::Null => None,
686                _ => {
687                    valid = false;
688                    None
689                },
690            })
691            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
692    };
693
694    if strict && !valid {
695        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", target_dtype);
696    }
697    polars_ensure!(
698        out.width() == width,
699        SchemaMismatch: "got mixed size array widths where width {} was expected", width
700    );
701
702    // Ensure the logical type is correct for nested types.
703    #[cfg(feature = "dtype-struct")]
704    if !matches!(inner_type, DataType::Null) && out.inner_dtype().is_nested() {
705        unsafe {
706            out.set_dtype(target_dtype);
707        };
708    }
709
710    Ok(out)
711}
712
713#[cfg(feature = "dtype-struct")]
714fn _any_values_to_struct<'a>(
715    av_fields: &[Field],
716    av_values: &[AnyValue<'a>],
717    field_index: usize,
718    field: &Field,
719    fields: &[Field],
720    field_avs: &mut Vec<AnyValue<'a>>,
721) {
722    // TODO: Optimize.
723
724    let mut append_by_search = || {
725        // Search for the name.
726        if let Some(i) = av_fields
727            .iter()
728            .position(|av_fld| av_fld.name == field.name)
729        {
730            field_avs.push(av_values[i].clone());
731            return;
732        }
733        field_avs.push(AnyValue::Null)
734    };
735
736    // All fields are available in this single value.
737    // We can use the index to get value.
738    if fields.len() == av_fields.len() {
739        if fields.iter().zip(av_fields.iter()).any(|(l, r)| l != r) {
740            append_by_search()
741        } else {
742            let av_val = av_values
743                .get(field_index)
744                .cloned()
745                .unwrap_or(AnyValue::Null);
746            field_avs.push(av_val)
747        }
748    }
749    // Not all fields are available, we search the proper field.
750    else {
751        // Search for the name.
752        append_by_search()
753    }
754}
755
756#[cfg(feature = "dtype-struct")]
757fn any_values_to_struct(
758    values: &[AnyValue],
759    fields: &[Field],
760    strict: bool,
761) -> PolarsResult<Series> {
762    // Fast path for structs with no fields.
763    if fields.is_empty() {
764        return Ok(
765            StructChunked::from_series(PlSmallStr::EMPTY, values.len(), [].iter())?.into_series(),
766        );
767    }
768
769    // The physical series fields of the struct.
770    let mut series_fields = Vec::with_capacity(fields.len());
771    let mut has_outer_validity = false;
772    let mut field_avs = Vec::with_capacity(values.len());
773    for (i, field) in fields.iter().enumerate() {
774        field_avs.clear();
775
776        for av in values.iter() {
777            match av {
778                AnyValue::StructOwned(payload) => {
779                    let av_fields = &payload.1;
780                    let av_values = &payload.0;
781                    _any_values_to_struct(av_fields, av_values, i, field, fields, &mut field_avs);
782                },
783                AnyValue::Struct(_, _, av_fields) => {
784                    let av_values: Vec<_> = av._iter_struct_av().collect();
785                    _any_values_to_struct(av_fields, &av_values, i, field, fields, &mut field_avs);
786                },
787                _ => {
788                    has_outer_validity = true;
789                    field_avs.push(AnyValue::Null)
790                },
791            }
792        }
793        // If the inferred dtype is null, we let auto inference work.
794        let s = if matches!(field.dtype, DataType::Null) {
795            Series::from_any_values(field.name().clone(), &field_avs, strict)?
796        } else {
797            Series::from_any_values_and_dtype(
798                field.name().clone(),
799                &field_avs,
800                &field.dtype,
801                strict,
802            )?
803        };
804        series_fields.push(s)
805    }
806
807    let mut out =
808        StructChunked::from_series(PlSmallStr::EMPTY, values.len(), series_fields.iter())?;
809    if has_outer_validity {
810        let mut validity = MutableBitmap::new();
811        validity.extend_constant(values.len(), true);
812        for (i, v) in values.iter().enumerate() {
813            if matches!(v, AnyValue::Null) {
814                unsafe { validity.set_unchecked(i, false) }
815            }
816        }
817        out.set_outer_validity(Some(validity.freeze()))
818    }
819    Ok(out.into_series())
820}
821
822#[cfg(feature = "object")]
823fn any_values_to_object(values: &[AnyValue]) -> PolarsResult<Series> {
824    use crate::chunked_array::object::registry;
825    let converter = registry::get_object_converter();
826    let mut builder = registry::get_object_builder(PlSmallStr::EMPTY, values.len());
827    for av in values {
828        match av {
829            AnyValue::Object(val) => builder.append_value(val.as_any()),
830            AnyValue::Null => builder.append_null(),
831            _ => {
832                // This is needed because in Python users can send mixed types.
833                // This only works if you set a global converter.
834                let any = converter(av.as_borrowed());
835                builder.append_value(&*any)
836            },
837        }
838    }
839
840    Ok(builder.to_series())
841}
842
843fn invalid_value_error(dtype: &DataType, value: &AnyValue) -> PolarsError {
844    polars_err!(
845        SchemaMismatch:
846        "unexpected value while building Series of type {:?}; found value of type {:?}: {}",
847        dtype,
848        value.dtype(),
849        value
850    )
851}
polars_core/series/any_value.rs

polars_core/series/
any_value.rs
