polars_core/series/
any_value.rs

1#[cfg(feature = "dtype-struct")]
2use std::borrow::Cow;
3use std::fmt::Write;
4
5use num_traits::AsPrimitive;
6use polars_arrow::bitmap::Bitmap;
7use polars_compute::cast::SerPrimitive;
8
9#[cfg(feature = "dtype-categorical")]
10use crate::chunked_array::builder::CategoricalChunkedBuilder;
11use crate::chunked_array::builder::{AnonymousOwnedListBuilder, get_list_builder};
12#[cfg(feature = "dtype-struct")]
13use crate::prelude::any_value::arr_to_any_value;
14use crate::prelude::*;
15use crate::utils::any_values_to_supertype;
16
17impl<'a, T: AsRef<[AnyValue<'a>]>> NamedFrom<T, [AnyValue<'a>]> for Series {
18    /// Construct a new [`Series`] from a collection of [`AnyValue`].
19    ///
20    /// # Panics
21    ///
22    /// Panics if the values do not all share the same data type (with the exception
23    /// of [`DataType::Null`], which is always allowed).
24    ///
25    /// [`AnyValue`]: crate::datatypes::AnyValue
26    fn new(name: PlSmallStr, values: T) -> Self {
27        let values = values.as_ref();
28        Series::from_any_values(name, values, true).expect("data types of values should match")
29    }
30}
31
32impl Series {
33    /// Construct a new [`Series`] from a slice of AnyValues.
34    ///
35    /// The data type of the resulting Series is determined by the `values`
36    /// and the `strict` parameter:
37    /// - If `strict` is `true`, the data type is equal to the data type of the
38    ///   first non-null value. If any other non-null values do not match this
39    ///   data type, an error is raised. If the first non-null value is a
40    ///   decimal the slice is scanned for the maximum precision and scale possible.
41    ///   If it is a struct, fields of all struct values are merged (in first-seen order).
42    /// - If `strict` is `false`, the data type is the supertype of the `values`.
43    ///   An error is returned if no supertype can be determined.
44    ///   **WARNING**: A full pass over the values is required to determine the supertype.
45    /// - If no values were passed, the resulting data type is `Null`.
46    pub fn from_any_values(
47        name: PlSmallStr,
48        values: &[AnyValue],
49        strict: bool,
50    ) -> PolarsResult<Self> {
51        fn get_first_non_null_dtype(values: &[AnyValue]) -> DataType {
52            let mut all_flat_null = true;
53            let first_non_null = values.iter().find(|av| {
54                if !av.is_null() {
55                    all_flat_null = false
56                };
57                !av.is_nested_null()
58            });
59            match first_non_null {
60                Some(av) => av.dtype(),
61                None => {
62                    if all_flat_null {
63                        DataType::Null
64                    } else {
65                        // Second pass to check for the nested null value that
66                        // toggled `all_flat_null` to false, e.g. a List(Null).
67                        let first_nested_null = values.iter().find(|av| !av.is_null()).unwrap();
68                        first_nested_null.dtype()
69                    }
70                },
71            }
72        }
73        let dtype = if strict {
74            match get_first_non_null_dtype(values) {
75                #[cfg(feature = "dtype-decimal")]
76                DataType::Decimal(mut prec, mut scale) => {
77                    for v in values {
78                        if let DataType::Decimal(p, s) = v.dtype() {
79                            prec = prec.max(p);
80                            scale = scale.max(s);
81                        }
82                    }
83                    DataType::Decimal(prec, scale)
84                },
85                #[cfg(feature = "dtype-struct")]
86                DataType::Struct(_) => {
87                    return any_values_to_struct(values, None, strict)
88                        .with_context(|| format!("while constructing Series '{name}'"))
89                        .map(|s| s.with_name(name));
90                },
91                dt => dt,
92            }
93        } else {
94            any_values_to_supertype(values)?
95        };
96
97        Self::from_any_values_and_dtype(name, values, &dtype, strict)
98    }
99
100    /// Construct a new [`Series`] with the given `dtype` from a slice of AnyValues.
101    ///
102    /// If `strict` is `true`, an error is returned if the values do not match the given
103    /// data type. If `strict` is `false`, values that do not match the given data type
104    /// are cast. If casting is not possible, the values are set to null instead.
105    pub fn from_any_values_and_dtype(
106        name: PlSmallStr,
107        values: &[AnyValue],
108        dtype: &DataType,
109        strict: bool,
110    ) -> PolarsResult<Self> {
111        Self::from_any_values_and_dtype_unnamed(values, dtype, strict)
112            .with_context(|| format!("while constructing Series '{name}'"))
113            .map(|s| s.with_name(name))
114    }
115
116    fn from_any_values_and_dtype_unnamed(
117        values: &[AnyValue],
118        dtype: &DataType,
119        strict: bool,
120    ) -> PolarsResult<Self> {
121        if values.is_empty() {
122            return Ok(Self::new_empty(PlSmallStr::EMPTY, dtype));
123        }
124
125        Ok(match dtype {
126            #[cfg(feature = "dtype-i8")]
127            DataType::Int8 => any_values_to_integer::<Int8Type>(values, strict)?.into_series(),
128            #[cfg(feature = "dtype-i16")]
129            DataType::Int16 => any_values_to_integer::<Int16Type>(values, strict)?.into_series(),
130            DataType::Int32 => any_values_to_integer::<Int32Type>(values, strict)?.into_series(),
131            DataType::Int64 => any_values_to_integer::<Int64Type>(values, strict)?.into_series(),
132            #[cfg(feature = "dtype-i128")]
133            DataType::Int128 => any_values_to_integer::<Int128Type>(values, strict)?.into_series(),
134            #[cfg(feature = "dtype-u8")]
135            DataType::UInt8 => any_values_to_integer::<UInt8Type>(values, strict)?.into_series(),
136            #[cfg(feature = "dtype-u16")]
137            DataType::UInt16 => any_values_to_integer::<UInt16Type>(values, strict)?.into_series(),
138            DataType::UInt32 => any_values_to_integer::<UInt32Type>(values, strict)?.into_series(),
139            DataType::UInt64 => any_values_to_integer::<UInt64Type>(values, strict)?.into_series(),
140            #[cfg(feature = "dtype-u128")]
141            DataType::UInt128 => {
142                any_values_to_integer::<UInt128Type>(values, strict)?.into_series()
143            },
144            #[cfg(feature = "dtype-f16")]
145            DataType::Float16 => any_values_to_f16(values, strict)?.into_series(),
146            DataType::Float32 => any_values_to_f32(values, strict)?.into_series(),
147            DataType::Float64 => any_values_to_f64(values, strict)?.into_series(),
148            DataType::Boolean => any_values_to_bool(values, strict)?.into_series(),
149            DataType::String => any_values_to_string(values, strict)?.into_series(),
150            DataType::Binary => any_values_to_binary(values, strict)?.into_series(),
151            DataType::BinaryOffset => any_values_to_binary_offset(values, strict)?.into_series(),
152            #[cfg(feature = "dtype-date")]
153            DataType::Date => any_values_to_date(values, strict)?.into_series(),
154            #[cfg(feature = "dtype-time")]
155            DataType::Time => any_values_to_time(values, strict)?.into_series(),
156            #[cfg(feature = "dtype-datetime")]
157            DataType::Datetime(tu, tz) => {
158                any_values_to_datetime(values, *tu, (*tz).clone(), strict)?.into_series()
159            },
160            #[cfg(feature = "dtype-duration")]
161            DataType::Duration(tu) => any_values_to_duration(values, *tu, strict)?.into_series(),
162            #[cfg(feature = "dtype-categorical")]
163            dt @ (DataType::Categorical(_, _) | DataType::Enum(_, _)) => {
164                any_values_to_categorical(values, dt, strict)?
165            },
166            #[cfg(feature = "dtype-decimal")]
167            DataType::Decimal(precision, scale) => {
168                any_values_to_decimal(values, *precision, *scale, strict)?.into_series()
169            },
170            #[cfg(feature = "dtype-map")]
171            DataType::Map(_, _) => {
172                let entries_dtype = dtype.map_entries_dtype().unwrap();
173                let entries = values
174                    .iter()
175                    .map(|av| match av {
176                        AnyValue::Map(entries) => Ok(AnyValue::List(entries.clone())),
177                        // An empty list of entries carries no field information.
178                        AnyValue::List(entries) if entries.dtype().is_nested_null() => {
179                            Ok(AnyValue::List(entries.clone()))
180                        },
181                        AnyValue::List(entries) => {
182                            ensure_map_entries_dtype(entries.dtype())?;
183                            Ok(AnyValue::List(entries.clone()))
184                        },
185                        av => Ok(av.clone()),
186                    })
187                    .collect::<PolarsResult<Vec<AnyValue>>>()?;
188                let storage = any_values_to_list(&entries, &entries_dtype, strict)?.into_series();
189                MapChunked::try_from_storage(dtype.clone(), storage)?.into_series()
190            },
191            #[cfg(feature = "dtype-extension")]
192            DataType::Extension(typ, storage) => {
193                Series::from_any_values_and_dtype_unnamed(values, storage, strict)?
194                    .into_extension(typ.clone())
195            },
196            DataType::List(inner) => any_values_to_list(values, inner, strict)?.into_series(),
197            #[cfg(feature = "dtype-array")]
198            DataType::Array(inner, size) => any_values_to_array(values, inner, strict, *size)?
199                .into_series()
200                .cast(&DataType::Array(inner.clone(), *size))?,
201            #[cfg(feature = "dtype-struct")]
202            DataType::Struct(fields) => any_values_to_struct(values, Some(fields), strict)?,
203            #[cfg(feature = "object")]
204            DataType::Object(_) => any_values_to_object(values)?,
205            DataType::Null => Series::new_null(PlSmallStr::EMPTY, values.len()),
206            dt => {
207                polars_bail!(
208                    InvalidOperation:
209                    "constructing a Series with data type {dt:?} from AnyValues is not supported"
210                )
211            },
212        })
213    }
214}
215
216fn any_values_to_primitive_nonstrict<T: PolarsNumericType>(values: &[AnyValue]) -> ChunkedArray<T> {
217    values
218        .iter()
219        .map(|av| av.extract::<T::Native>())
220        .collect_trusted()
221}
222
223fn any_values_to_integer<T: PolarsIntegerType>(
224    values: &[AnyValue],
225    strict: bool,
226) -> PolarsResult<ChunkedArray<T>> {
227    fn any_values_to_integer_strict<T: PolarsIntegerType>(
228        values: &[AnyValue],
229    ) -> PolarsResult<ChunkedArray<T>> {
230        let mut builder = PrimitiveChunkedBuilder::<T>::new(PlSmallStr::EMPTY, values.len());
231        for av in values {
232            match &av {
233                av if av.is_integer() => {
234                    let opt_val = av.extract::<T::Native>();
235                    let val = match opt_val {
236                        Some(v) => v,
237                        None => return Err(invalid_value_error(&T::get_static_dtype(), av)),
238                    };
239                    builder.append_value(val)
240                },
241                AnyValue::Null => builder.append_null(),
242                av => return Err(invalid_value_error(&T::get_static_dtype(), av)),
243            }
244        }
245        Ok(builder.finish())
246    }
247
248    if strict {
249        any_values_to_integer_strict::<T>(values)
250    } else {
251        Ok(any_values_to_primitive_nonstrict::<T>(values))
252    }
253}
254
255#[cfg(feature = "dtype-f16")]
256fn any_values_to_f16(values: &[AnyValue], strict: bool) -> PolarsResult<Float16Chunked> {
257    fn any_values_to_f16_strict(values: &[AnyValue]) -> PolarsResult<Float16Chunked> {
258        let mut builder =
259            PrimitiveChunkedBuilder::<Float16Type>::new(PlSmallStr::EMPTY, values.len());
260        for av in values {
261            match av {
262                AnyValue::Float16(i) => builder.append_value(*i),
263                AnyValue::Null => builder.append_null(),
264                av => return Err(invalid_value_error(&DataType::Float16, av)),
265            }
266        }
267        Ok(builder.finish())
268    }
269    if strict {
270        any_values_to_f16_strict(values)
271    } else {
272        Ok(any_values_to_primitive_nonstrict::<Float16Type>(values))
273    }
274}
275
276fn any_values_to_f32(values: &[AnyValue], strict: bool) -> PolarsResult<Float32Chunked> {
277    fn any_values_to_f32_strict(values: &[AnyValue]) -> PolarsResult<Float32Chunked> {
278        let mut builder =
279            PrimitiveChunkedBuilder::<Float32Type>::new(PlSmallStr::EMPTY, values.len());
280        for av in values {
281            match av {
282                AnyValue::Float32(i) => builder.append_value(*i),
283                AnyValue::Float16(i) => builder.append_value(i.as_()),
284                AnyValue::Null => builder.append_null(),
285                av => return Err(invalid_value_error(&DataType::Float32, av)),
286            }
287        }
288        Ok(builder.finish())
289    }
290    if strict {
291        any_values_to_f32_strict(values)
292    } else {
293        Ok(any_values_to_primitive_nonstrict::<Float32Type>(values))
294    }
295}
296fn any_values_to_f64(values: &[AnyValue], strict: bool) -> PolarsResult<Float64Chunked> {
297    fn any_values_to_f64_strict(values: &[AnyValue]) -> PolarsResult<Float64Chunked> {
298        let mut builder =
299            PrimitiveChunkedBuilder::<Float64Type>::new(PlSmallStr::EMPTY, values.len());
300        for av in values {
301            match av {
302                AnyValue::Float64(i) => builder.append_value(*i),
303                AnyValue::Float32(i) => builder.append_value(*i as f64),
304                AnyValue::Float16(i) => builder.append_value(i.as_()),
305                AnyValue::Null => builder.append_null(),
306                av => return Err(invalid_value_error(&DataType::Float64, av)),
307            }
308        }
309        Ok(builder.finish())
310    }
311    if strict {
312        any_values_to_f64_strict(values)
313    } else {
314        Ok(any_values_to_primitive_nonstrict::<Float64Type>(values))
315    }
316}
317
318fn any_values_to_bool(values: &[AnyValue], strict: bool) -> PolarsResult<BooleanChunked> {
319    let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
320    for av in values {
321        match av {
322            AnyValue::Boolean(b) => builder.append_value(*b),
323            AnyValue::Null => builder.append_null(),
324            av => {
325                if strict {
326                    return Err(invalid_value_error(&DataType::Boolean, av));
327                }
328                match av.cast(&DataType::Boolean) {
329                    AnyValue::Boolean(b) => builder.append_value(b),
330                    _ => builder.append_null(),
331                }
332            },
333        }
334    }
335    Ok(builder.finish())
336}
337
338fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult<StringChunked> {
339    fn any_values_to_string_strict(values: &[AnyValue]) -> PolarsResult<StringChunked> {
340        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
341        for av in values {
342            match av {
343                AnyValue::String(s) => builder.append_value(s),
344                AnyValue::StringOwned(s) => builder.append_value(s),
345                AnyValue::Null => builder.append_null(),
346                av => return Err(invalid_value_error(&DataType::String, av)),
347            }
348        }
349        Ok(builder.finish())
350    }
351    fn any_values_to_string_nonstrict(values: &[AnyValue]) -> StringChunked {
352        fn _write_any_value(av: &AnyValue<'_>, buffer: &mut String) {
353            match av {
354                AnyValue::String(s) => buffer.push_str(s),
355                AnyValue::Float64(f) => {
356                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
357                },
358                AnyValue::Float32(f) => {
359                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
360                },
361                #[cfg(feature = "dtype-f16")]
362                AnyValue::Float16(f) => {
363                    SerPrimitive::write(unsafe { buffer.as_mut_vec() }, *f);
364                },
365                #[cfg(feature = "dtype-struct")]
366                AnyValue::StructOwned(payload) => {
367                    buffer.push('{');
368                    let mut iter = payload.0.iter().peekable();
369                    while let Some(child) = iter.next() {
370                        _write_any_value(child, buffer);
371                        if iter.peek().is_some() {
372                            buffer.push(',')
373                        }
374                    }
375                    buffer.push('}');
376                },
377                #[cfg(feature = "dtype-struct")]
378                AnyValue::Struct(_, _, flds) => {
379                    let mut vals = Vec::with_capacity(flds.len());
380                    av._materialize_struct_av(&mut vals);
381
382                    buffer.push('{');
383                    let mut iter = vals.iter().peekable();
384                    while let Some(child) = iter.next() {
385                        _write_any_value(child, buffer);
386                        if iter.peek().is_some() {
387                            buffer.push(',')
388                        }
389                    }
390                    buffer.push('}');
391                },
392                #[cfg(feature = "dtype-array")]
393                AnyValue::Array(vals, _) => {
394                    buffer.push('[');
395                    let mut iter = vals.iter().peekable();
396                    while let Some(child) = iter.next() {
397                        _write_any_value(&child, buffer);
398                        if iter.peek().is_some() {
399                            buffer.push(',');
400                        }
401                    }
402                    buffer.push(']');
403                },
404                AnyValue::List(vals) => {
405                    buffer.push('[');
406                    let mut iter = vals.iter().peekable();
407                    while let Some(child) = iter.next() {
408                        _write_any_value(&child, buffer);
409                        if iter.peek().is_some() {
410                            buffer.push(',');
411                        }
412                    }
413                    buffer.push(']');
414                },
415                av => {
416                    write!(buffer, "{av}").unwrap();
417                },
418            }
419        }
420
421        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
422        let mut owned = String::new(); // Amortize allocations.
423        for av in values {
424            owned.clear();
425
426            match av {
427                AnyValue::String(s) => builder.append_value(s),
428                AnyValue::StringOwned(s) => builder.append_value(s),
429                AnyValue::Null => builder.append_null(),
430                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(),
431
432                // Explicitly convert and dump floating-point values to strings
433                // to preserve as much precision as possible.
434                // Using write!(..., "{av}") steps through Display formatting
435                // which rounds to an arbitrary precision thus losing information.
436                av => {
437                    _write_any_value(av, &mut owned);
438                    builder.append_value(&owned);
439                },
440            }
441        }
442        builder.finish()
443    }
444    if strict {
445        any_values_to_string_strict(values)
446    } else {
447        Ok(any_values_to_string_nonstrict(values))
448    }
449}
450
451fn any_values_to_binary(values: &[AnyValue], strict: bool) -> PolarsResult<BinaryChunked> {
452    fn any_values_to_binary_strict(values: &[AnyValue]) -> PolarsResult<BinaryChunked> {
453        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, values.len());
454        for av in values {
455            match av {
456                AnyValue::Binary(s) => builder.append_value(*s),
457                AnyValue::BinaryOwned(s) => builder.append_value(&**s),
458                AnyValue::Null => builder.append_null(),
459                av => return Err(invalid_value_error(&DataType::Binary, av)),
460            }
461        }
462        Ok(builder.finish())
463    }
464    fn any_values_to_binary_nonstrict(values: &[AnyValue]) -> BinaryChunked {
465        values
466            .iter()
467            .map(|av| match av {
468                AnyValue::Binary(b) => Some(*b),
469                AnyValue::BinaryOwned(b) => Some(&**b),
470                AnyValue::String(s) => Some(s.as_bytes()),
471                AnyValue::StringOwned(s) => Some(s.as_bytes()),
472                _ => None,
473            })
474            .collect_trusted()
475    }
476    if strict {
477        any_values_to_binary_strict(values)
478    } else {
479        Ok(any_values_to_binary_nonstrict(values))
480    }
481}
482
483fn any_values_to_binary_offset(
484    values: &[AnyValue],
485    strict: bool,
486) -> PolarsResult<BinaryOffsetChunked> {
487    let mut builder = MutableBinaryArray::<i64>::new();
488    for av in values {
489        match av {
490            AnyValue::Binary(s) => builder.push(Some(*s)),
491            AnyValue::BinaryOwned(s) => builder.push(Some(&**s)),
492            AnyValue::Null => builder.push_null(),
493            av => {
494                if strict {
495                    return Err(invalid_value_error(&DataType::Binary, av));
496                } else {
497                    builder.push_null();
498                };
499            },
500        }
501    }
502    Ok(BinaryOffsetChunked::with_chunk(
503        Default::default(),
504        builder.into(),
505    ))
506}
507
508#[cfg(feature = "dtype-date")]
509fn any_values_to_date(values: &[AnyValue], strict: bool) -> PolarsResult<DateChunked> {
510    let mut builder = PrimitiveChunkedBuilder::<Int32Type>::new(PlSmallStr::EMPTY, values.len());
511    for av in values {
512        match av {
513            AnyValue::Date(i) => builder.append_value(*i),
514            AnyValue::Null => builder.append_null(),
515            av => {
516                if strict {
517                    return Err(invalid_value_error(&DataType::Date, av));
518                }
519                match av.cast(&DataType::Date) {
520                    AnyValue::Date(i) => builder.append_value(i),
521                    _ => builder.append_null(),
522                }
523            },
524        }
525    }
526    Ok(builder.finish().into_date())
527}
528
529#[cfg(feature = "dtype-time")]
530fn any_values_to_time(values: &[AnyValue], strict: bool) -> PolarsResult<TimeChunked> {
531    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
532    for av in values {
533        match av {
534            AnyValue::Time(i) => builder.append_value(*i),
535            AnyValue::Null => builder.append_null(),
536            av => {
537                if strict {
538                    return Err(invalid_value_error(&DataType::Time, av));
539                }
540                match av.cast(&DataType::Time) {
541                    AnyValue::Time(i) => builder.append_value(i),
542                    _ => builder.append_null(),
543                }
544            },
545        }
546    }
547    Ok(builder.finish().into_time())
548}
549
550#[cfg(feature = "dtype-datetime")]
551fn any_values_to_datetime(
552    values: &[AnyValue],
553    time_unit: TimeUnit,
554    time_zone: Option<TimeZone>,
555    strict: bool,
556) -> PolarsResult<DatetimeChunked> {
557    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
558    let target_dtype = DataType::Datetime(time_unit, time_zone.clone());
559    for av in values {
560        match av {
561            AnyValue::Datetime(i, tu, _) if *tu == time_unit => builder.append_value(*i),
562            AnyValue::DatetimeOwned(i, tu, _) if *tu == time_unit => builder.append_value(*i),
563            AnyValue::Null => builder.append_null(),
564            av => {
565                if strict {
566                    return Err(invalid_value_error(&target_dtype, av));
567                }
568                match av.cast(&target_dtype) {
569                    AnyValue::Datetime(i, _, _) => builder.append_value(i),
570                    AnyValue::DatetimeOwned(i, _, _) => builder.append_value(i),
571                    _ => builder.append_null(),
572                }
573            },
574        }
575    }
576    Ok(builder.finish().into_datetime(time_unit, time_zone))
577}
578
579#[cfg(feature = "dtype-duration")]
580fn any_values_to_duration(
581    values: &[AnyValue],
582    time_unit: TimeUnit,
583    strict: bool,
584) -> PolarsResult<DurationChunked> {
585    let mut builder = PrimitiveChunkedBuilder::<Int64Type>::new(PlSmallStr::EMPTY, values.len());
586    let target_dtype = DataType::Duration(time_unit);
587    for av in values {
588        match av {
589            AnyValue::Duration(i, tu) if *tu == time_unit => builder.append_value(*i),
590            AnyValue::Null => builder.append_null(),
591            av => {
592                if strict {
593                    return Err(invalid_value_error(&target_dtype, av));
594                }
595                match av.cast(&target_dtype) {
596                    AnyValue::Duration(i, _) => builder.append_value(i),
597                    _ => builder.append_null(),
598                }
599            },
600        }
601    }
602    Ok(builder.finish().into_duration(time_unit))
603}
604
605#[cfg(feature = "dtype-categorical")]
606fn any_values_to_categorical(
607    values: &[AnyValue],
608    dtype: &DataType,
609    strict: bool,
610) -> PolarsResult<Series> {
611    with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
612        let mut builder = CategoricalChunkedBuilder::<$C>::new(PlSmallStr::EMPTY, dtype.clone());
613
614        let mut owned = String::new(); // Amortize allocations.
615        for av in values {
616            let ret = match av {
617                AnyValue::String(s) => builder.append_str(s),
618                AnyValue::StringOwned(s) => builder.append_str(s),
619
620                &AnyValue::Enum(cat, &ref map) |
621                &AnyValue::EnumOwned(cat, ref map) |
622                &AnyValue::Categorical(cat, &ref map) |
623                &AnyValue::CategoricalOwned(cat, ref map) => builder.append_cat(cat, map),
624
625                AnyValue::Binary(_) | AnyValue::BinaryOwned(_) if !strict => {
626                    builder.append_null();
627                    Ok(())
628                },
629                AnyValue::Null => {
630                    builder.append_null();
631                    Ok(())
632                }
633
634                av => {
635                    if strict {
636                        return Err(invalid_value_error(&DataType::String, av));
637                    }
638
639                    owned.clear();
640                    write!(owned, "{av}").unwrap();
641                    builder.append_str(&owned)
642                },
643            };
644
645            if let Err(e) = ret {
646                if strict {
647                    return Err(e);
648                } else {
649                    builder.append_null();
650                }
651            }
652        }
653
654        let ca = builder.finish();
655        Ok(ca.into_series())
656    })
657}
658
659#[cfg(feature = "dtype-decimal")]
660fn any_values_to_decimal(
661    values: &[AnyValue],
662    precision: usize,
663    scale: usize,
664    strict: bool,
665) -> PolarsResult<DecimalChunked> {
666    let target_dtype = DataType::Decimal(precision, scale);
667
668    let mut builder = PrimitiveChunkedBuilder::<Int128Type>::new(PlSmallStr::EMPTY, values.len());
669    for av in values {
670        match av {
671            // Allow equal or less scale. We do want to support different scales even in 'strict' mode.
672            AnyValue::Decimal(v, p, s) if *s <= scale => {
673                if *p <= precision && *s == scale {
674                    builder.append_value(*v)
675                } else {
676                    match av.strict_cast(&target_dtype) {
677                        Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
678                        _ => builder.append_null(),
679                    }
680                }
681            },
682            AnyValue::Null => builder.append_null(),
683            av => {
684                if strict {
685                    return Err(invalid_value_error(&target_dtype, av));
686                }
687                match av.strict_cast(&target_dtype) {
688                    Some(AnyValue::Decimal(i, _, _)) => builder.append_value(i),
689                    _ => builder.append_null(),
690                }
691            },
692        };
693    }
694
695    // Build the array and do a precision check if needed.
696    builder.finish().into_decimal(precision, scale)
697}
698
699fn any_values_to_list(
700    avs: &[AnyValue],
701    inner_type: &DataType,
702    strict: bool,
703) -> PolarsResult<ListChunked> {
704    // GB:
705    // Lord forgive for the sins I have committed in this function. The amount of strange
706    // exceptions that need to happen for this to work are insane and I feel like I am going crazy.
707    //
708    // This function is essentially a copy of the `<ListChunked as FromIterator>` where it does not
709    // sample the datatype from the first element and instead we give it explicitly. This allows
710    // this function to properly assign a datatype if `avs` starts with a `null` value. Previously,
711    // this was solved by assigning the `dtype` again afterwards, but why? We should not link the
712    // implementation of these functions. We still need to assign the dtype of the ListArray and
713    // such, anyways.
714    //
715    // Then, `collect_ca_with_dtype` does not possess the necessary exceptions shown in this
716    // function to use that. I have tried adding the exceptions there and it broke other things. I
717    // really do feel like this is the simplest solution.
718
719    let mut valid = true;
720    let capacity = avs.len();
721
722    let ca = match inner_type {
723        // AnyValues with empty lists in python can create
724        // Series of an unknown dtype.
725        // We use the anonymousbuilder without a dtype
726        // the empty arrays is then not added (we add an extra offset instead)
727        // the next non-empty series then must have the correct dtype.
728        DataType::Null => {
729            let mut builder = AnonymousOwnedListBuilder::new(PlSmallStr::EMPTY, capacity, None);
730            for av in avs {
731                match av {
732                    AnyValue::List(b) => builder.append_series(b)?,
733                    AnyValue::Null => builder.append_null(),
734                    _ => {
735                        valid = false;
736                        builder.append_null();
737                    },
738                }
739            }
740            builder.finish()
741        },
742
743        #[cfg(feature = "object")]
744        DataType::Object(_) => polars_bail!(nyi = "Nested object types"),
745
746        _ => {
747            let mut builder =
748                get_list_builder(inner_type, capacity * 5, capacity, PlSmallStr::EMPTY);
749            for av in avs {
750                match av {
751                    AnyValue::List(b) => match b.cast(inner_type) {
752                        Ok(casted) => {
753                            if casted.null_count() != b.null_count() {
754                                valid = !strict;
755                            }
756                            builder.append_series(&casted)?;
757                        },
758                        Err(_) => {
759                            valid = false;
760                            for _ in 0..b.len() {
761                                builder.append_null();
762                            }
763                        },
764                    },
765                    AnyValue::Null => builder.append_null(),
766                    _ => {
767                        valid = false;
768                        builder.append_null()
769                    },
770                }
771            }
772
773            builder.finish()
774        },
775    };
776
777    if strict && !valid {
778        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", DataType::List(Box::new(inner_type.clone())));
779    }
780
781    Ok(ca)
782}
783
784#[cfg(feature = "dtype-array")]
785fn any_values_to_array(
786    avs: &[AnyValue],
787    inner_type: &DataType,
788    strict: bool,
789    width: usize,
790) -> PolarsResult<ArrayChunked> {
791    fn to_arr(s: &Series) -> Option<ArrayRef> {
792        if s.chunks().len() > 1 {
793            let s = s.rechunk();
794            Some(s.chunks()[0].clone())
795        } else {
796            Some(s.chunks()[0].clone())
797        }
798    }
799
800    let target_dtype = DataType::Array(Box::new(inner_type.clone()), width);
801
802    // This is handled downstream. The builder will choose the first non null type.
803    let mut valid = true;
804    #[allow(unused_mut)]
805    let mut out: ArrayChunked = if inner_type == &DataType::Null {
806        avs.iter()
807            .map(|av| match av {
808                AnyValue::List(b) | AnyValue::Array(b, _) => to_arr(b),
809                AnyValue::Null => None,
810                _ => {
811                    valid = false;
812                    None
813                },
814            })
815            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
816    }
817    // Make sure that wrongly inferred AnyValues don't deviate from the datatype.
818    else {
819        avs.iter()
820            .map(|av| match av {
821                AnyValue::List(b) | AnyValue::Array(b, _) => {
822                    if b.dtype() == inner_type {
823                        to_arr(b)
824                    } else {
825                        let s = match b.cast(inner_type) {
826                            Ok(out) => out,
827                            Err(_) => Series::full_null(b.name().clone(), b.len(), inner_type),
828                        };
829                        to_arr(&s)
830                    }
831                },
832                AnyValue::Null => None,
833                _ => {
834                    valid = false;
835                    None
836                },
837            })
838            .collect_ca_with_dtype(PlSmallStr::EMPTY, target_dtype.clone())
839    };
840
841    if strict && !valid {
842        polars_bail!(SchemaMismatch: "unexpected value while building Series of type {:?}", target_dtype);
843    }
844    polars_ensure!(
845        out.width() == width,
846        SchemaMismatch: "got mixed size array widths where width {} was expected", width
847    );
848
849    // Ensure the logical type is correct for nested types.
850    #[cfg(feature = "dtype-struct")]
851    if !matches!(inner_type, DataType::Null) && out.inner_dtype().is_nested() {
852        unsafe {
853            out.set_dtype(target_dtype);
854        };
855    }
856
857    Ok(out)
858}
859
860/// A Struct's Fields, compared and hashed by name.
861#[cfg(feature = "dtype-struct")]
862#[derive(Clone, Copy)]
863struct FieldsByName<'a>(&'a [Field]);
864
865#[cfg(feature = "dtype-struct")]
866impl Eq for FieldsByName<'_> {}
867
868#[cfg(feature = "dtype-struct")]
869impl PartialEq for FieldsByName<'_> {
870    fn eq(&self, other: &Self) -> bool {
871        let names_eq = |(l, r): (&Field, &Field)| l.name == r.name;
872        self.0.len() == other.0.len() && self.0.iter().zip(other.0).all(names_eq)
873    }
874}
875
876#[cfg(feature = "dtype-struct")]
877impl std::hash::Hash for FieldsByName<'_> {
878    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
879        self.0.iter().for_each(|fld| fld.name.hash(state));
880    }
881}
882
883/// Resolves the fields of struct values (by name) against those of the struct
884/// being built, once per distinct ordered sequence of field names.
885#[cfg(feature = "dtype-struct")]
886struct StructFieldResolver<'a> {
887    target: Option<&'a [Field]>,
888    fields: PlIndexMap<&'a str, Option<&'a DataType>>,
889    ordering_ids: PlHashMap<FieldsByName<'a>, usize>,
890    last: (FieldsByName<'a>, usize),
891    indexes: Vec<usize>,
892    offsets: Vec<usize>,
893}
894
895#[cfg(feature = "dtype-struct")]
896impl<'a> StructFieldResolver<'a> {
897    fn new(target: Option<&'a [Field]>) -> Self {
898        // Values without fields (or that aren't structs) have ordering id 0.
899        let no_fields = FieldsByName(&[]);
900        let to_entry = |fld: &'a Field| (fld.name.as_str(), Some(&fld.dtype));
901        Self {
902            target,
903            fields: target.unwrap_or_default().iter().map(to_entry).collect(),
904            ordering_ids: PlHashMap::from_iter([(no_fields, 0)]),
905            last: (no_fields, 0),
906            indexes: Vec::new(),
907            offsets: vec![0],
908        }
909    }
910
911    /// Returns the ordering id of `av_fields` (potentially merging dtypes).
912    #[inline]
913    fn resolve(&mut self, av_fields: &'a [Field]) -> usize {
914        // Values from the same struct array share (already resolved) fields.
915        if std::ptr::eq(av_fields, self.last.0.0) {
916            return self.last.1;
917        }
918        let av_fields = FieldsByName(av_fields);
919        if self.last.0 != av_fields {
920            let next_id = self.ordering_ids.len();
921            let id = *self.ordering_ids.entry(av_fields).or_insert_with(|| {
922                self.offsets.push(self.indexes.len());
923                for fld in av_fields.0 {
924                    let entry = self.fields.entry(fld.name.as_str());
925                    self.indexes.push(entry.index());
926                    entry.or_insert(Some(&fld.dtype));
927                }
928                next_id
929            });
930            self.last = (av_fields, id);
931        }
932        let id = self.last.1;
933        if self.target.is_none() {
934            let indexes = &self.indexes[self.offsets[id]..];
935            for (fld, &idx) in av_fields.0.iter().zip(indexes) {
936                let dtype = &mut self.fields[idx];
937                if !fld.dtype.is_null()
938                    && let Some(dt) = *dtype
939                    && *dt != fld.dtype
940                {
941                    // `Null` data type gives way to the first non-null one
942                    *dtype = dt.is_null().then_some(&fld.dtype);
943                }
944            }
945        }
946        id
947    }
948
949    /// Returns the struct's fields, and the position (if any) of each in each ordering.
950    fn finish(mut self) -> (Cow<'a, [Field]>, Vec<Option<usize>>) {
951        let fields = match self.target {
952            Some(fields) => Cow::Borrowed(fields),
953            None => {
954                let to_field = |(name, dtype): (&str, Option<&DataType>)| {
955                    Field::new(name.into(), dtype.cloned().unwrap_or(DataType::Null))
956                };
957                Cow::Owned(self.fields.into_iter().map(to_field).collect())
958            },
959        };
960        let n_fields = fields.len();
961        self.offsets.push(self.indexes.len());
962        let mut positions = vec![None; (self.offsets.len() - 1) * n_fields];
963        for (id, range) in self.offsets.windows(2).enumerate() {
964            for (j, &idx) in self.indexes[range[0]..range[1]].iter().enumerate() {
965                // With a `target`, `fields` also has the values' other fields (after its own).
966                if idx < n_fields {
967                    positions[id * n_fields + idx].get_or_insert(j);
968                }
969            }
970        }
971        (fields, positions)
972    }
973}
974
975/// Builds a struct Series from `values` with the given `fields` or, if `None`,
976/// with the fields of all struct `values` (see [`StructFieldResolver`]).
977#[cfg(feature = "dtype-struct")]
978fn any_values_to_struct(
979    values: &[AnyValue],
980    fields: Option<&[Field]>,
981    strict: bool,
982) -> PolarsResult<Series> {
983    let mut field_resolver = StructFieldResolver::new(fields);
984    let av_field_order_ids: Vec<usize> = values
985        .iter()
986        .map(|av| match av {
987            AnyValue::StructOwned(payload) => field_resolver.resolve(&payload.1),
988            AnyValue::Struct(_, _, av_fields) => field_resolver.resolve(av_fields),
989            _ => 0,
990        })
991        .collect();
992    let (fields, field_positions) = field_resolver.finish();
993
994    // Values other than structs here (or lists/arrays of their fields)
995    // are null (or invalid if `strict`).
996    let n_fields = fields.len();
997    let is_valid = |av: &AnyValue| match av {
998        AnyValue::StructOwned(_) | AnyValue::Struct(..) => true,
999        AnyValue::List(s) => s.len() == n_fields,
1000        #[cfg(feature = "dtype-array")]
1001        AnyValue::Array(s, _) => s.len() == n_fields,
1002        _ => false,
1003    };
1004    if strict && let Some(av) = values.iter().find(|av| !av.is_null() && !is_valid(av)) {
1005        return Err(invalid_value_error(&DataType::Struct(fields.to_vec()), av));
1006    }
1007
1008    // Empty struct fast path (no fields, but outer validity still required).
1009    if n_fields == 0 {
1010        let mut out = StructChunked::from_series(PlSmallStr::EMPTY, values.len(), [].iter())?;
1011        out.set_outer_validity(Bitmap::opt_from_iter(values.iter().map(is_valid)));
1012        return Ok(out.into_series());
1013    }
1014
1015    // The physical series fields of the struct.
1016    let mut series_fields = Vec::with_capacity(n_fields);
1017    let mut field_avs = Vec::with_capacity(values.len());
1018    for (i, field) in fields.iter().enumerate() {
1019        field_avs.clear();
1020
1021        for (av, &field_order_id) in values.iter().zip(&av_field_order_ids) {
1022            let position = field_positions[field_order_id * n_fields + i];
1023            match av {
1024                AnyValue::StructOwned(payload) => {
1025                    let av_values = &payload.0;
1026                    field_avs.push(position.map_or(AnyValue::Null, |j| av_values[j].as_borrowed()));
1027                },
1028                AnyValue::Struct(idx, arr, av_fields) => {
1029                    field_avs.push(position.map_or(AnyValue::Null, |j| {
1030                        // SAFETY: `idx` and `j` are in bounds; `av_fields[j]` describes this child.
1031                        unsafe { arr_to_any_value(&*arr.values()[j], *idx, av_fields[j].dtype()) }
1032                    }));
1033                },
1034                AnyValue::List(s) if s.len() == n_fields => {
1035                    let av = unsafe { s.get_unchecked(i) };
1036                    field_avs.push(av);
1037                },
1038                #[cfg(feature = "dtype-array")]
1039                AnyValue::Array(s, _) if s.len() == n_fields => {
1040                    let av = unsafe { s.get_unchecked(i) };
1041                    field_avs.push(av);
1042                },
1043                _ => field_avs.push(AnyValue::Null),
1044            }
1045        }
1046        // If the inferred dtype is null, we let auto inference work.
1047        let s = if matches!(field.dtype, DataType::Null) {
1048            Series::from_any_values(field.name().clone(), &field_avs, strict)?
1049        } else {
1050            Series::from_any_values_and_dtype(
1051                field.name().clone(),
1052                &field_avs,
1053                &field.dtype,
1054                strict,
1055            )?
1056        };
1057        series_fields.push(s)
1058    }
1059
1060    let mut out =
1061        StructChunked::from_series(PlSmallStr::EMPTY, values.len(), series_fields.iter())?;
1062    let validity = Bitmap::opt_from_iter(values.iter().map(is_valid));
1063    if validity.is_some() {
1064        out.set_outer_validity(validity);
1065    }
1066    Ok(out.into_series())
1067}
1068
1069#[cfg(feature = "object")]
1070fn any_values_to_object(values: &[AnyValue]) -> PolarsResult<Series> {
1071    use crate::chunked_array::object::registry;
1072    let converter = registry::get_object_converter();
1073    let mut builder = registry::get_object_builder(PlSmallStr::EMPTY, values.len());
1074    for av in values {
1075        match av {
1076            AnyValue::Object(val) => builder.append_value(val.as_any()),
1077            AnyValue::Null => builder.append_null(),
1078            _ => {
1079                // This is needed because in Python users can send mixed types.
1080                // This only works if you set a global converter.
1081                let any = converter(av.as_borrowed());
1082                builder.append_value(&*any)
1083            },
1084        }
1085    }
1086
1087    Ok(builder.to_series())
1088}
1089
1090fn invalid_value_error(dtype: &DataType, value: &AnyValue) -> PolarsError {
1091    polars_err!(
1092        SchemaMismatch:
1093        "unexpected value while building Series of type {:?}; found value of type {:?}: {}",
1094        dtype,
1095        value.dtype(),
1096        value
1097    )
1098}
polars_core/series/any_value.rs

polars_core/series/
any_value.rs
