polars_core/series/
any_value.rs

1use std::fmt::Write;
2
3use arrow::bitmap::Bitmap;
4use num_traits::AsPrimitive;
5use polars_compute::cast::SerPrimitive;
6
7#[cfg(feature = "dtype-categorical")]
8use crate::chunked_array::builder::CategoricalChunkedBuilder;
9use crate::chunked_array::builder::{AnonymousOwnedListBuilder, get_list_builder};
10use crate::prelude::*;
11use crate::utils::any_values_to_supertype;
12
13impl<'a, T: AsRef<[AnyValue<'a>]>> NamedFrom<T, [AnyValue<'a>]> for Series {
14    /// Construct a new [`Series`] from a collection of [`AnyValue`].
15    ///
16    /// # Panics
17    ///
18    /// Panics if the values do not all share the same data type (with the exception
19    /// of [`DataType::Null`], which is always allowed).
20    ///
21    /// [`AnyValue`]: crate::datatypes::AnyValue
22    fn new(name: PlSmallStr, values: T) -> Self {
23        let values = values.as_ref();
24        Series::from_any_values(name, values, true).expect("data types of values should match")
25    }
26}
27
28impl Series {
29    /// Construct a new [`Series`] from a slice of AnyValues.
30    ///
31    /// The data type of the resulting Series is determined by the `values`
32    /// and the `strict` parameter:
33    /// - If `strict` is `true`, the data type is equal to the data type of the
34    ///   first non-null value. If any other non-null values do not match this
35    ///   data type, an error is raised. If the first non-null value is a
36    ///   decimal the slice is scanned for the maximum precision and scale possible.
37    /// - If `strict` is `false`, the data type is the supertype of the `values`.
38    ///   An error is returned if no supertype can be determined.
39    ///   **WARNING**: A full pass over the values is required to determine the supertype.
40    /// - If no values were passed, the resulting data type is `Null`.
41    pub fn from_any_values(
42        name: PlSmallStr,
43        values: &[AnyValue],
44        strict: bool,
45    ) -> PolarsResult<Self> {
46        fn get_first_non_null_dtype(values: &[AnyValue]) -> DataType {
47            let mut all_flat_null = true;
48            let first_non_null = values.iter().find(|av| {
49                if !av.is_null() {
50                    all_flat_null = false
51                };
52                !av.is_nested_null()
53            });
54            match first_non_null {
55                Some(av) => av.dtype(),
56                None => {
57                    if all_flat_null {
58                        DataType::Null
59                    } else {
60                        // Second pass to check for the nested null value that
61                        // toggled `all_flat_null` to false, e.g. a List(Null).
62                        let first_nested_null = values.iter().find(|av| !av.is_null()).unwrap();
63                        first_nested_null.dtype()
64                    }
65                },
66            }
67        }
68        let dtype = if strict {
69            match get_first_non_null_dtype(values) {
70                #[cfg(feature = "dtype-decimal")]
71                DataType::Decimal(mut prec, mut scale) => {
72                    for v in values {
73                        if let DataType::Decimal(p, s) = v.dtype() {
74                            prec = prec.max(p);
75                            scale = scale.max(s);
76                        }
77                    }
78                    DataType::Decimal(prec, scale)
79                },
80                dt => dt,
81            }
82        } else {
83            any_values_to_supertype(values)?
84        };
85
86        Self::from_any_values_and_dtype(name, values, &dtype, strict)
87    }
88
89    /// Construct a new [`Series`] with the given `dtype` from a slice of AnyValues.
90    ///
91    /// If `strict` is `true`, an error is returned if the values do not match the given
92    /// data type. If `strict` is `false`, values that do not match the given data type
93    /// are cast. If casting is not possible, the values are set to null instead.
94    pub fn from_any_values_and_dtype(
95        name: PlSmallStr,
96        values: &[AnyValue],
97        dtype: &DataType,
98        strict: bool,
99    ) -> PolarsResult<Self> {
100        Self::from_any_values_and_dtype_unnamed(values, dtype, strict)
101            .with_context(|| format!("while constructing Series '{name}'"))
102            .map(|s| s.with_name(name))
103    }
104
105    fn from_any_values_and_dtype_unnamed(
106        values: &[AnyValue],
107        dtype: &DataType,
108        strict: bool,
109    ) -> PolarsResult<Self> {
110        if values.is_empty() {
111            return Ok(Self::new_empty(PlSmallStr::EMPTY, dtype));
112        }
113
114        Ok(match dtype {
115            #[cfg(feature = "dtype-i8")]
116            DataType::Int8 => any_values_to_integer::<Int8Type>(values, strict)?.into_series(),
117            #[cfg(feature = "dtype-i16")]
118            DataType::Int16 => any_values_to_integer::<Int16Type>(values, strict)?.into_series(),
119            DataType::Int32 => any_values_to_integer::<Int32Type>(values, strict)?.into_series(),
120            DataType::Int64 => any_values_to_integer::<Int64Type>(values, strict)?.into_series(),
121            #[cfg(feature = "dtype-i128")]
122            DataType::Int128 => any_values_to_integer::<Int128Type>(values, strict)?.into_series(),
123            #[cfg(feature = "dtype-u8")]
124            DataType::UInt8 => any_values_to_integer::<UInt8Type>(values, strict)?.into_series(),
125            #[cfg(feature = "dtype-u16")]
126            DataType::UInt16 => any_values_to_integer::<UInt16Type>(values, strict)?.into_series(),
127            DataType::UInt32 => any_values_to_integer::<UInt32Type>(values, strict)?.into_series(),
128            DataType::UInt64 => any_values_to_integer::<UInt64Type>(values, strict)?.into_series(),
129            #[cfg(feature = "dtype-u128")]
130            DataType::UInt128 => {
131                any_values_to_integer::<UInt128Type>(values, strict)?.into_series()
132            },
133            #[cfg(feature = "dtype-f16")]
134            DataType::Float16 => any_values_to_f16(values, strict)?.into_series(),
135            DataType::Float32 => any_values_to_f32(values, strict)?.into_series(),
136            DataType::Float64 => any_values_to_f64(values, strict)?.into_series(),
137            DataType::Boolean => any_values_to_bool(values, strict)?.into_series(),
138            DataType::String => any_values_to_string(values, strict)?.into_series(),
139            DataType::Binary => any_values_to_binary(values, strict)?.into_series(),
140            DataType::BinaryOffset => any_values_to_binary_offset(values, strict)?.into_series(),
141            #[cfg(feature = "dtype-date")]
142            DataType::Date => any_values_to_date(values, strict)?.into_series(),
143            #[cfg(feature = "dtype-time")]
144            DataType::Time => any_values_to_time(values, strict)?.into_series(),
145            #[cfg(feature = "dtype-datetime")]
146            DataType::Datetime(tu, tz) => {
147                any_values_to_datetime(values, *tu, (*tz).clone(), strict)?.into_series()
148            },
149            #[cfg(feature = "dtype-duration")]
150            DataType::Duration(tu) => any_values_to_duration(values, *tu, strict)?.into_series(),
151            #[cfg(feature = "dtype-categorical")]
152            dt @ (DataType::Categorical(_, _) | DataType::Enum(_, _)) => {
153                any_values_to_categorical(values, dt, strict)?
154            },
155            #[cfg(feature = "dtype-decimal")]
156            DataType::Decimal(precision, scale) => {
157                any_values_to_decimal(values, *precision, *scale, strict)?.into_series()
158            },
159            #[cfg(feature = "dtype-extension")]
160            DataType::Extension(typ, storage) => {
161                Series::from_any_values_and_dtype_unnamed(values, storage, strict)?
162                    .into_extension(typ.clone())
163            },
164            DataType::List(inner) => any_values_to_list(values, inner, strict)?.into_series(),
165            #[cfg(feature = "dtype-array")]
166            DataType::Array(inner, size) => any_values_to_array(values, inner, strict, *size)?
167                .into_series()
168                .cast(&DataType::Array(inner.clone(), *size))?,
169            #[cfg(feature = "dtype-struct")]
170            DataType::Struct(fields) => any_values_to_struct(values, fields, strict)?,
171            #[cfg(feature = "object")]
172            DataType::Object(_) => any_values_to_object(values)?,
173            DataType::Null => Series::new_null(PlSmallStr::EMPTY, values.len()),
174            dt => {
175                polars_bail!(
176                    InvalidOperation:
177                    "constructing a Series with data type {dt:?} from AnyValues is not supported"
178                )
179            },
180        })
181    }
182}
183
184fn any_values_to_primitive_nonstrict<T: PolarsNumericType>(values: &[AnyValue]) -> ChunkedArray<T> {
185    values
186        .iter()
187        .map(|av| av.extract::<T::Native>())
188        .collect_trusted()
189}
190
191fn any_values_to_integer<T: PolarsIntegerType>(
192    values: &[AnyValue],
193    strict: bool,
194) -> PolarsResult<ChunkedArray<T>> {
195    fn any_values_to_integer_strict<T: PolarsIntegerType>(
196        values: &[AnyValue],
197    ) -> PolarsResult<ChunkedArray<T>> {
198        let mut builder = PrimitiveChunkedBuilder::<T>::new(PlSmallStr::EMPTY, values.len());
199        for av in values {
200            match &av {
201                av if av.is_integer() => {
202                    let opt_val = av.extract::<T::Native>();
203                    let val = match opt_val {
204                        Some(v) => v,
205                        None => return Err(invalid_value_error(&T::get_static_dtype(), av)),
206                    };
207                    builder.append_value(val)
208                },
209                AnyValue::Null => builder.append_null(),
210                av => return Err(invalid_value_error(&T::get_static_dtype(), av)),
211            }
212        }
213        Ok(builder.finish())
214    }
215
216    if strict {
217        any_values_to_integer_strict::<T>(values)
218    } else {
219        Ok(any_values_to_primitive_nonstrict::<T>(values))
220    }
221}
222
223#[cfg(feature = "dtype-f16")]
224fn any_values_to_f16(values: &[AnyValue], strict: bool) -> PolarsResult<Float16Chunked> {
225    fn any_values_to_f16_strict(values: &[AnyValue]) -> PolarsResult<Float16Chunked> {
226        let mut builder =
227            PrimitiveChunkedBuilder::<Float16Type>::new(PlSmallStr::EMPTY, values.len());
228        for av in values {
229            match av {
230                AnyValue::Float16(i) => builder.append_value(*i),
231                AnyValue::Null => builder.append_null(),
232                av => return Err(invalid_value_error(&DataType::Float16, av)),
233            }
234        }
235        Ok(builder.finish())
236    }
237    if strict {
238        any_values_to_f16_strict(values)
239    } else {
240        Ok(any_values_to_primitive_nonstrict::<Float16Type>(values))
241    }
242}
243
244fn any_values_to_f32(values: &[AnyValue], strict: bool) -> PolarsResult<Float32Chunked> {
245    fn any_values_to_f32_strict(values: &[AnyValue]) -> PolarsResult<Float32Chunked> {
246        let mut builder =
247            PrimitiveChunkedBuilder::<Float32Type>::new(PlSmallStr::EMPTY, values.len());
248        for av in values {
249            match av {
250                AnyValue::Float32(i) => builder.append_value(*i),
251                AnyValue::Float16(i) => builder.append_value(i.as_()),
252                AnyValue::Null => builder.append_null(),
253                av => return Err(invalid_value_error(&DataType::Float32, av)),
254            }
255        }
256        Ok(builder.finish())
257    }
258    if strict {
259        any_values_to_f32_strict(values)
260    } else {
261        Ok(any_values_to_primitive_nonstrict::<Float32Type>(values))
262    }
263}
264fn any_values_to_f64(values: &[AnyValue], strict: bool) -> PolarsResult<Float64Chunked> {
265    fn any_values_to_f64_strict(values: &[AnyValue]) -> PolarsResult<Float64Chunked> {
266        let mut builder =
267            PrimitiveChunkedBuilder::<Float64Type>::new(PlSmallStr::EMPTY, values.len());
268        for av in values {
269            match av {
270                AnyValue::Float64(i) => builder.append_value(*i),
271                AnyValue::Float32(i) => builder.append_value(*i as f64),
272                AnyValue::Float16(i) => builder.append_value(i.as_()),
273                AnyValue::Null => builder.append_null(),
274                av => return Err(invalid_value_error(&DataType::Float64, av)),
275            }
276        }
277        Ok(builder.finish())
278    }
279    if strict {
280        any_values_to_f64_strict(values)
281    } else {
282        Ok(any_values_to_primitive_nonstrict::<Float64Type>(values))
283    }
284}
285
286fn any_values_to_bool(values: &[AnyValue], strict: bool) -> PolarsResult<BooleanChunked> {
287    let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
288    for av in values {
289        match av {
290            AnyValue::Boolean(b) => builder.append_value(*b),
291            AnyValue::Null => builder.append_null(),
292            av => {
293                if strict {
294                    return Err(invalid_value_error(&DataType::Boolean, av));
295                }
296                match av.cast(&DataType::Boolean) {
297                    AnyValue::Boolean(b) => builder.append_value(b),
298                    _ => builder.append_null(),
299                }
300            },
301        }
302    }
303    Ok(builder.finish())
304}
305
306fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult<StringChunked> {
307    fn any_values_to_string_strict(values: &[AnyValue]) -> PolarsResult<StringChunked> {
308        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
309        for av in values {
310            match av {
311                AnyValue::String(s) => builder.append_value(s),
312                AnyValue::StringOwned(s) => builder.append_value(s),
313                AnyValue::Null => builder.append_null(),
314                av => return Err(invalid_value_error(&DataType::String, av)),
315            }
316        }
317        Ok(builder.finish())
318    }
319    fn any_values_to_string_nonstrict(values: &[AnyValue]) -> StringChunked {
320        fn _write_any_value(av: &AnyValue<'_>, buffer: &mut String) {
321            match av {
322                AnyValue::String(s) => buffer.push_str(s),
323                AnyValue::Float64(f) => {
324                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
325                },
326                AnyValue::Float32(f) => {
327                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
328                },
329                #[cfg(feature = "dtype-f16")]
330                AnyValue::Float16(f) => {
331                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
332                },
333                #[cfg(feature = "dtype-struct")]
334                AnyValue::StructOwned(payload) => {
335                    buffer.push('{');
336                    let mut iter = payload.0.iter().peekable();
337                    while let Some(child) = iter.next() {
338                        _write_any_value(child, buffer);
339                        if iter.peek().is_some() {
340                            buffer.push(',')
341                        }
342                    }
343                    buffer.push('}');
344                },
345                #[cfg(feature = "dtype-struct")]
346                AnyValue::Struct(_, _, flds) => {
347                    let mut vals = Vec::with_capacity(flds.len());
348                    av._materialize_struct_av(&mut vals);
349
350                    buffer.push('{');
351                    let mut iter = vals.iter().peekable();
352                    while let Some(child) = iter.next() {
353                        _write_any_value(child, buffer);
354                        if iter.peek().is_some() {
355                            buffer.push(',')
356                        }
357                    }
358                    buffer.push('}');
359                },
360                #[cfg(feature = "dtype-array")]
361                AnyValue::Array(vals, _) => {
362                    buffer.push('[');
363                    let mut iter = vals.iter().peekable();
364                    while let Some(child) = iter.next() {
365                        _write_any_value(&child, buffer);
366                        if iter.peek().is_some() {
367                            buffer.push(',');
368                        }
369                    }
370                    buffer.push(']');
371                },
372                AnyValue::List(vals) => {
373                    buffer.push('[');
374                    let mut iter = vals.iter().peekable();
375                    while let Some(child) = iter.next() {
376                        _write_any_value(&child, buffer);
377                        if iter.peek().is_some() {
378                            buffer.push(',');
379                        }
380                    }
381                    buffer.push(']');
382                },
383                av => {
384                    write!(buffer, "{av}").unwrap();
385                },
386            }
387        }
388
389        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
390        let mut owned = String::new(); // Amortize allocations.
391        for av in values {
392            owned.clear();
393
394            match av {
395                AnyValue::String(s) => builder.append_value(s),
396                AnyValue::StringOwned(s) => builder.append_value(s),
397                AnyValue::Null => builder.append_null(),
398                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(),
399
400                // Explicitly convert and dump floating-point values to strings
401                // to preserve as much precision as possible.
402                // Using write!(..., "{av}") steps through Display formatting
403                // which rounds to an arbitrary precision thus losing information.
404                av => {
405                    _write_any_value(av, &mut owned);
406                    builder.append_value(&owned);
407                },
408            }
409        }
410        builder.finish()
411    }
412    if strict {
413        any_values_to_string_strict(values)
414    } else {
415        Ok(any_values_to_string_nonstrict(values))
416    }
417}
418
419fn any_values_to_binary(values: &[AnyValue], strict: bool) -> PolarsResult<BinaryChunked> {
420    fn any_values_to_binary_strict(values: &[AnyValue]) -> PolarsResult<BinaryChunked> {
421        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
422        for av in values {
423            match av {
424                AnyValue::Binary(s) => builder.append_value(*s),
425                AnyValue::BinaryOwned(s) => builder.append_value(&**s),
426                AnyValue::Null => builder.append_null(),
427                av => return Err(invalid_value_error(&DataType::Binary, av)),
428            }
429        }
430        Ok(builder.finish())
431    }
432    fn any_values_to_binary_nonstrict(values: &[AnyValue]) -> BinaryChunked {
433        values
434            .iter()
435            .map(|av| match av {
436                AnyValue::Binary(b) => Some(*b),
437                AnyValue::BinaryOwned(b) => Some(&**b),
438                AnyValue::String(s) => Some(s.as_bytes()),
439                AnyValue::StringOwned(s) => Some(s.as_bytes()),
440                _ => None,
441            })
442            .collect_trusted()
443    }
444    if strict {
445        any_values_to_binary_strict(values)
446    } else {
447        Ok(any_values_to_binary_nonstrict(values))
448    }
449}
450
451fn any_values_to_binary_offset(
452    values: &[AnyValue],
453    strict: bool,
454) -> PolarsResult<BinaryOffsetChunked> {
455    let mut builder = MutableBinaryArray::<i64>::new();
456    for av in values {
457        match av {
458            AnyValue::Binary(s) => builder.push(Some(*s)),
459            AnyValue::BinaryOwned(s) => builder.push(Some(&**s)),
460            AnyValue::Null => builder.push_null(),
461            av => {
462                if strict {
463                    return Err(invalid_value_error(&DataType::Binary, av));
464                } else {
465                    builder.push_null();
466                };
467            },
468        }
469    }
470    Ok(BinaryOffsetChunked::with_chunk(
471        Default::default(),
472        builder.into(),
473    ))
474}
475
476#[cfg(feature = "dtype-date")]
477fn any_values_to_date(values: &[AnyValue], strict: bool) -> PolarsResult<DateChunked> {
478    let mut builder = PrimitiveChunkedBuilder::<Int32Type>::new(PlSmallStr::EMPTY, values.len());
479    for av in values {
480        match av {
481            AnyValue::Date(i) => builder.append_value(*i),
482            AnyValue::Null => builder.append_null(),
483            av => {
484                if strict {
485                    return Err(invalid_value_error(&DataType::Date, av));
486                }
487                match av.cast(&DataType::Date) {
488                    AnyValue::Date(i) => builder.append_value(i),
489                    _ => builder.append_null(),
490                }
491            },
492        }
493    }
494    Ok(builder.finish().into_date())
495}
496
497#[cfg(feature = "dtype-time")]
498fn any_values_to_time(values: &[AnyValue], strict: bool) -> PolarsResult<TimeChunked> {
499    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
500    for av in values {
501        match av {
502            AnyValue::Time(i) => builder.append_value(*i),
503            AnyValue::Null => builder.append_null(),
504            av => {
505                if strict {
506                    return Err(invalid_value_error(&DataType::Time, av));
507                }
508                match av.cast(&DataType::Time) {
509                    AnyValue::Time(i) => builder.append_value(i),
510                    _ => builder.append_null(),
511                }
512            },
513        }
514    }
515    Ok(builder.finish().into_time())
516}
517
518#[cfg(feature = "dtype-datetime")]
519fn any_values_to_datetime(
520    values: &[AnyValue],
521    time_unit: TimeUnit,
522    time_zone: Option<TimeZone>,
523    strict: bool,
524) -> PolarsResult<DatetimeChunked> {
525    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
526    let target_dtype = DataType::Datetime(time_unit, time_zone.clone());
527    for av in values {
528        match av {
529            AnyValue::Datetime(i, tu, _) if *tu == time_unit => builder.append_value(*i),
530            AnyValue::DatetimeOwned(i, tu, _) if *tu == time_unit => builder.append_value(*i),
531            AnyValue::Null => builder.append_null(),
532            av => {
533                if strict {
534                    return Err(invalid_value_error(&target_dtype, av));
535                }
536                match av.cast(&target_dtype) {
537                    AnyValue::Datetime(i, _, _) => builder.append_value(i),
538                    AnyValue::DatetimeOwned(i, _, _) => builder.append_value(i),
539                    _ => builder.append_null(),
540                }
541            },
542        }
543    }
544    Ok(builder.finish().into_datetime(time_unit, time_zone))
545}
546
547#[cfg(feature = "dtype-duration")]
548fn any_values_to_duration(
549    values: &[AnyValue],
550    time_unit: TimeUnit,
551    strict: bool,
552) -> PolarsResult<DurationChunked> {
553    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
554    let target_dtype = DataType::Duration(time_unit);
555    for av in values {
556        match av {
557            AnyValue::Duration(i, tu) if *tu == time_unit => builder.append_value(*i),
558            AnyValue::Null => builder.append_null(),
559            av => {
560                if strict {
561                    return Err(invalid_value_error(&target_dtype, av));
562                }
563                match av.cast(&target_dtype) {
564                    AnyValue::Duration(i, _) => builder.append_value(i),
565                    _ => builder.append_null(),
566                }
567            },
568        }
569    }
570    Ok(builder.finish().into_duration(time_unit))
571}
572
573#[cfg(feature = "dtype-categorical")]
574fn any_values_to_categorical(
575    values: &[AnyValue],
576    dtype: &DataType,
577    strict: bool,
578) -> PolarsResult<Series> {
579    with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
580        let mut builder = CategoricalChunkedBuilder::<$C>::new(PlSmallStr::EMPTY, dtype.clone());
581
582        let mut owned = String::new(); // Amortize allocations.
583        for av in values {
584            let ret = match av {
585                AnyValue::String(s) => builder.append_str(s),
586                AnyValue::StringOwned(s) => builder.append_str(s),
587
588                &AnyValue::Enum(cat, &ref map) |
589                &AnyValue::EnumOwned(cat, ref map) |
590                &AnyValue::Categorical(cat, &ref map) |
591                &AnyValue::CategoricalOwned(cat, ref map) => builder.append_cat(cat, map),
592
593                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) if !strict => {
594                    builder.append_null();
595                    Ok(())
596                },
597                AnyValue::Null => {
598                    builder.append_null();
599                    Ok(())
600                }
601
602                av => {
603                    if strict {
604                        return Err(invalid_value_error(&DataType::String, av));
605                    }
606
607                    owned.clear();
608                    write!(owned, "{av}").unwrap();
609                    builder.append_str(&owned)
610                },
611            };
612
613            if let Err(e) = ret {
614                if strict {
615                    return Err(e);
616                } else {
617                    builder.append_null();
618                }
619            }
620        }
621
622        let ca = builder.finish();
623        Ok(ca.into_series())
624    })
625}
626
627#[cfg(feature = "dtype-decimal")]
628fn any_values_to_decimal(
629    values: &[AnyValue],
630    precision: usize,
631    scale: usize,
632    strict: bool,
633) -> PolarsResult<DecimalChunked> {
634    let target_dtype = DataType::Decimal(precision, scale);
635
636    let mut builder = PrimitiveChunkedBuilder::<Int128Type>::new(PlSmallStr::EMPTY, values.len());
637    for av in values {
638        match av {
639            // Allow equal or less scale. We do want to support different scales even in 'strict' mode.
640            AnyValue::Decimal(v, p, s) if *s <= scale => {
641                if *p <= precision && *s == scale {
642                    builder.append_value(*v)
643                } else {
644                    match av.strict_cast(&target_dtype) {
645                        Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
646                        _ => builder.append_null(),
647                    }
648                }
649            },
650            AnyValue::Null => builder.append_null(),
651            av => {
652                if strict {
653                    return Err(invalid_value_error(&target_dtype, av));
654                }
655                match av.strict_cast(&target_dtype) {
656                    Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
657                    _ => builder.append_null(),
658                }
659            },
660        };
661    }
662
663    // Build the array and do a precision check if needed.
664    builder.finish().into_decimal(precision, scale)
665}
666
667fn any_values_to_list(
668    avs: &[AnyValue],
669    inner_type: &DataType,
670    strict: bool,
671) -> PolarsResult<ListChunked> {
672    // GB:
673    // Lord forgive for the sins I have committed in this function. The amount of strange
674    // exceptions that need to happen for this to work are insane and I feel like I am going crazy.
675    //
676    // This function is essentially a copy of the `<ListChunked as FromIterator>` where it does not
677    // sample the datatype from the first element and instead we give it explicitly. This allows
678    // this function to properly assign a datatype if `avs` starts with a `null` value. Previously,
679    // this was solved by assigning the `dtype` again afterwards, but why? We should not link the
680    // implementation of these functions. We still need to assign the dtype of the ListArray and
681    // such, anyways.
682    //
683    // Then, `collect_ca_with_dtype` does not possess the necessary exceptions shown in this
684    // function to use that. I have tried adding the exceptions there and it broke other things. I
685    // really do feel like this is the simplest solution.
686
687    let mut valid = true;
688    let capacity = avs.len();
689
690    let ca = match inner_type {
691        // AnyValues with empty lists in python can create
692        // Series of an unknown dtype.
693        // We use the anonymousbuilder without a dtype
694        // the empty arrays is then not added (we add an extra offset instead)
695        // the next non-empty series then must have the correct dtype.
696        DataType::Null => {
697            let mut builder = AnonymousOwnedListBuilder::new(PlSmallStr::EMPTY, capacity, None);
698            for av in avs {
699                match av {
700                    AnyValue::List(b) => builder.append_series(b)?,
701                    AnyValue::Null => builder.append_null(),
702                    _ => {
703                        valid = false;
704                        builder.append_null();
705                    },
706                }
707            }
708            builder.finish()
709        },
710
711        #[cfg(feature = "object")]
712        DataType::Object(_) => polars_bail!(nyi = "Nested object types"),
713
714        _ => {
715            let mut builder =
716                get_list_builder(inner_type, capacity * 5, capacity, PlSmallStr::EMPTY);
717            for av in avs {
718                match av {
719                    AnyValue::List(b) => match b.cast(inner_type) {
720                        Ok(casted) => {
721                            if casted.null_count() != b.null_count() {
722                                valid = !strict;
723                            }
724                            builder.append_series(&casted)?;
725                        },
726                        Err(_) => {
727                            valid = false;
728                            for _ in 0..b.len() {
729                                builder.append_null();
730                            }
731                        },
732                    },
733                    AnyValue::Null => builder.append_null(),
734                    _ => {
735                        valid = false;
736                        builder.append_null()
737                    },
738                }
739            }
740
741            builder.finish()
742        },
743    };
744
745    if strict && !valid {
746        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", DataType::List(Box::new(inner_type.clone())));
747    }
748
749    Ok(ca)
750}
751
752#[cfg(feature = "dtype-array")]
753fn any_values_to_array(
754    avs: &[AnyValue],
755    inner_type: &DataType,
756    strict: bool,
757    width: usize,
758) -> PolarsResult<ArrayChunked> {
759    fn to_arr(s: &Series) -> Option<ArrayRef> {
760        if s.chunks().len() > 1 {
761            let s = s.rechunk();
762            Some(s.chunks()[0].clone())
763        } else {
764            Some(s.chunks()[0].clone())
765        }
766    }
767
768    let target_dtype = DataType::Array(Box::new(inner_type.clone()), width);
769
770    // This is handled downstream. The builder will choose the first non null type.
771    let mut valid = true;
772    #[allow(unused_mut)]
773    let mut out: ArrayChunked = if inner_type == &DataType::Null {
774        avs.iter()
775            .map(|av| match av {
776                AnyValue::List(b) | AnyValue::Array(b, _) => to_arr(b),
777                AnyValue::Null => None,
778                _ => {
779                    valid = false;
780                    None
781                },
782            })
783            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
784    }
785    // Make sure that wrongly inferred AnyValues don't deviate from the datatype.
786    else {
787        avs.iter()
788            .map(|av| match av {
789                AnyValue::List(b) | AnyValue::Array(b, _) => {
790                    if b.dtype() == inner_type {
791                        to_arr(b)
792                    } else {
793                        let s = match b.cast(inner_type) {
794                            Ok(out) => out,
795                            Err(_) => Series::full_null(b.name().clone(), b.len(), inner_type),
796                        };
797                        to_arr(&s)
798                    }
799                },
800                AnyValue::Null => None,
801                _ => {
802                    valid = false;
803                    None
804                },
805            })
806            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
807    };
808
809    if strict && !valid {
810        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", target_dtype);
811    }
812    polars_ensure!(
813        out.width() == width,
814        SchemaMismatch: "got mixed size array widths where width {} was expected", width
815    );
816
817    // Ensure the logical type is correct for nested types.
818    #[cfg(feature = "dtype-struct")]
819    if !matches!(inner_type, DataType::Null) && out.inner_dtype().is_nested() {
820        unsafe {
821            out.set_dtype(target_dtype);
822        };
823    }
824
825    Ok(out)
826}
827
828#[cfg(feature = "dtype-struct")]
829fn _any_values_to_struct<'a>(
830    av_fields: &[Field],
831    av_values: &[AnyValue<'a>],
832    field_index: usize,
833    field: &Field,
834    fields: &[Field],
835    field_avs: &mut Vec<AnyValue<'a>>,
836) {
837    // TODO: Optimize.
838
839    let mut append_by_search = || {
840        // Search for the name.
841        if let Some(i) = av_fields
842            .iter()
843            .position(|av_fld| av_fld.name == field.name)
844        {
845            field_avs.push(av_values[i].clone());
846            return;
847        }
848        field_avs.push(AnyValue::Null)
849    };
850
851    // All fields are available in this single value.
852    // We can use the index to get value.
853    if fields.len() == av_fields.len() {
854        if fields.iter().zip(av_fields.iter()).any(|(l, r)| l != r) {
855            append_by_search()
856        } else {
857            let av_val = av_values
858                .get(field_index)
859                .cloned()
860                .unwrap_or(AnyValue::Null);
861            field_avs.push(av_val)
862        }
863    }
864    // Not all fields are available, we search the proper field.
865    else {
866        // Search for the name.
867        append_by_search()
868    }
869}
870
871#[cfg(feature = "dtype-struct")]
872fn any_values_to_struct(
873    values: &[AnyValue],
874    fields: &[Field],
875    strict: bool,
876) -> PolarsResult<Series> {
877    // Fast path for structs with no fields.
878    if fields.is_empty() {
879        let mut out = StructChunked::from_series(PlSmallStr::EMPTY, values.len(), [].iter())?;
880        out.set_outer_validity(Bitmap::opt_from_iter(values.iter().map(|av| !av.is_null())));
881        return Ok(out.into_series());
882    }
883
884    // The physical series fields of the struct.
885    let mut series_fields = Vec::with_capacity(fields.len());
886    let mut has_outer_validity = false;
887    let mut field_avs = Vec::with_capacity(values.len());
888    for (i, field) in fields.iter().enumerate() {
889        field_avs.clear();
890
891        for av in values.iter() {
892            match av {
893                AnyValue::StructOwned(payload) => {
894                    let av_fields = &payload.1;
895                    let av_values = &payload.0;
896                    _any_values_to_struct(av_fields, av_values, i, field, fields, &mut field_avs);
897                },
898                AnyValue::Struct(_, _, av_fields) => {
899                    let av_values: Vec<_> = av._iter_struct_av().collect();
900                    _any_values_to_struct(av_fields, &av_values, i, field, fields, &mut field_avs);
901                },
902                AnyValue::List(s) if s.len() == fields.len() => {
903                    let av = unsafe { s.get_unchecked(i) };
904                    field_avs.push(av);
905                },
906                #[cfg(feature = "dtype-array")]
907                AnyValue::Array(s, _) if s.len() == fields.len() => {
908                    let av = unsafe { s.get_unchecked(i) };
909                    field_avs.push(av);
910                },
911                AnyValue::Null => {
912                    has_outer_validity = true;
913                    field_avs.push(AnyValue::Null)
914                },
915                _ => {
916                    if strict {
917                        return Err(invalid_value_error(&DataType::Struct(fields.to_vec()), av));
918                    } else {
919                        has_outer_validity = true;
920                        field_avs.push(AnyValue::Null)
921                    }
922                },
923            }
924        }
925        // If the inferred dtype is null, we let auto inference work.
926        let s = if matches!(field.dtype, DataType::Null) {
927            Series::from_any_values(field.name().clone(), &field_avs, strict)?
928        } else {
929            Series::from_any_values_and_dtype(
930                field.name().clone(),
931                &field_avs,
932                &field.dtype,
933                strict,
934            )?
935        };
936        series_fields.push(s)
937    }
938
939    let mut out =
940        StructChunked::from_series(PlSmallStr::EMPTY, values.len(), series_fields.iter())?;
941    if has_outer_validity {
942        out.set_outer_validity(Bitmap::opt_from_iter(values.iter().map(|av| !av.is_null())));
943    }
944    Ok(out.into_series())
945}
946
947#[cfg(feature = "object")]
948fn any_values_to_object(values: &[AnyValue]) -> PolarsResult<Series> {
949    use crate::chunked_array::object::registry;
950    let converter = registry::get_object_converter();
951    let mut builder = registry::get_object_builder(PlSmallStr::EMPTY, values.len());
952    for av in values {
953        match av {
954            AnyValue::Object(val) => builder.append_value(val.as_any()),
955            AnyValue::Null => builder.append_null(),
956            _ => {
957                // This is needed because in Python users can send mixed types.
958                // This only works if you set a global converter.
959                let any = converter(av.as_borrowed());
960                builder.append_value(&*any)
961            },
962        }
963    }
964
965    Ok(builder.to_series())
966}
967
968fn invalid_value_error(dtype: &DataType, value: &AnyValue) -> PolarsError {
969    polars_err!(
970        SchemaMismatch:
971        "unexpected value while building Series of type {:?}; found value of type {:?}: {}",
972        dtype,
973        value.dtype(),
974        value
975    )
976}
polars_core/series/any_value.rs

polars_core/series/
any_value.rs
