polars_core/series/
from.rs

1#[cfg(feature = "dtype-categorical")]
2use arrow::compute::concatenate::concatenate_unchecked;
3use arrow::datatypes::Metadata;
4#[cfg(any(
5    feature = "dtype-date",
6    feature = "dtype-datetime",
7    feature = "dtype-time",
8    feature = "dtype-duration"
9))]
10use arrow::temporal_conversions::*;
11use polars_compute::cast::cast_unchecked as cast;
12use polars_error::feature_gated;
13use polars_utils::itertools::Itertools;
14
15use crate::chunked_array::cast::{CastOptions, cast_chunks};
16#[cfg(feature = "object")]
17use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
18#[cfg(feature = "object")]
19use crate::chunked_array::object::registry::get_object_builder;
20#[cfg(feature = "timezones")]
21use crate::chunked_array::temporal::parse_fixed_offset;
22#[cfg(feature = "timezones")]
23use crate::chunked_array::temporal::validate_time_zone;
24use crate::prelude::*;
25
26impl Series {
27    /// Takes chunks and a polars datatype and constructs the Series
28    /// This is faster than creating from chunks and an arrow datatype because there is no
29    /// casting involved
30    ///
31    /// # Safety
32    ///
33    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
34    pub unsafe fn from_chunks_and_dtype_unchecked(
35        name: PlSmallStr,
36        chunks: Vec<ArrayRef>,
37        dtype: &DataType,
38    ) -> Self {
39        use DataType::*;
40        match dtype {
41            #[cfg(feature = "dtype-i8")]
42            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
43            #[cfg(feature = "dtype-i16")]
44            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
45            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
46            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
47            #[cfg(feature = "dtype-u8")]
48            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
49            #[cfg(feature = "dtype-u16")]
50            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
51            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
52            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
53            #[cfg(feature = "dtype-i128")]
54            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
55            #[cfg(feature = "dtype-date")]
56            Date => Int32Chunked::from_chunks(name, chunks)
57                .into_date()
58                .into_series(),
59            #[cfg(feature = "dtype-time")]
60            Time => Int64Chunked::from_chunks(name, chunks)
61                .into_time()
62                .into_series(),
63            #[cfg(feature = "dtype-duration")]
64            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
65                .into_duration(*tu)
66                .into_series(),
67            #[cfg(feature = "dtype-datetime")]
68            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
69                .into_datetime(*tu, tz.clone())
70                .into_series(),
71            #[cfg(feature = "dtype-decimal")]
72            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
73                .into_decimal_unchecked(
74                    *precision,
75                    scale.unwrap_or_else(|| unreachable!("scale should be set")),
76                )
77                .into_series(),
78            #[cfg(feature = "dtype-array")]
79            Array(_, _) => {
80                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
81                    .into_series()
82            },
83            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
84                .into_series(),
85            String => StringChunked::from_chunks(name, chunks).into_series(),
86            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
87            #[cfg(feature = "dtype-categorical")]
88            dt @ (Categorical(rev_map, ordering) | Enum(rev_map, ordering)) => {
89                let cats = UInt32Chunked::from_chunks(name, chunks);
90                let rev_map = rev_map.clone().unwrap_or_else(|| {
91                    assert!(cats.is_empty());
92                    Arc::new(RevMapping::default())
93                });
94                let mut ca = CategoricalChunked::from_cats_and_rev_map_unchecked(
95                    cats,
96                    rev_map,
97                    matches!(dt, Enum(_, _)),
98                    *ordering,
99                );
100                ca.set_fast_unique(false);
101                ca.into_series()
102            },
103            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
104            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
105            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
106            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
107            #[cfg(feature = "dtype-struct")]
108            Struct(_) => {
109                let mut ca =
110                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
111                ca.propagate_nulls();
112                ca.into_series()
113            },
114            #[cfg(feature = "object")]
115            Object(_) => {
116                if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
117                    assert_eq!(chunks.len(), 1);
118                    // SAFETY:
119                    // this is highly unsafe. it will dereference a raw ptr on the heap
120                    // make sure the ptr is allocated and from this pid
121                    // (the pid is checked before dereference)
122                    {
123                        let pe = PolarsExtension::new(arr.clone());
124                        let s = pe.get_series(&name);
125                        pe.take_and_forget();
126                        s
127                    }
128                } else {
129                    unsafe { get_object_builder(name, 0).from_chunks(chunks) }
130                }
131            },
132            Null => new_null(name, &chunks),
133            Unknown(_) => {
134                panic!("dtype is unknown; consider supplying data-types for all operations")
135            },
136            #[allow(unreachable_patterns)]
137            _ => unreachable!(),
138        }
139    }
140
141    /// # Safety
142    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
143    pub unsafe fn _try_from_arrow_unchecked(
144        name: PlSmallStr,
145        chunks: Vec<ArrayRef>,
146        dtype: &ArrowDataType,
147    ) -> PolarsResult<Self> {
148        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
149    }
150
151    /// Create a new Series without checking if the inner dtype of the chunks is correct
152    ///
153    /// # Safety
154    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
155    pub unsafe fn _try_from_arrow_unchecked_with_md(
156        name: PlSmallStr,
157        chunks: Vec<ArrayRef>,
158        dtype: &ArrowDataType,
159        md: Option<&Metadata>,
160    ) -> PolarsResult<Self> {
161        match dtype {
162            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
163            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
164                let chunks =
165                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
166                Ok(StringChunked::from_chunks(name, chunks).into_series())
167            },
168            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
169            ArrowDataType::LargeBinary => {
170                if let Some(md) = md {
171                    if md.maintain_type() {
172                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
173                    }
174                }
175                let chunks =
176                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
177                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
178            },
179            ArrowDataType::Binary => {
180                let chunks =
181                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
182                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
183            },
184            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
185                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
186                unsafe {
187                    Ok(
188                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
189                            .into_series(),
190                    )
191                }
192            },
193            #[cfg(feature = "dtype-array")]
194            ArrowDataType::FixedSizeList(_, _) => {
195                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
196                unsafe {
197                    Ok(
198                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
199                            .into_series(),
200                    )
201                }
202            },
203            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
204            #[cfg(feature = "dtype-u8")]
205            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
206            #[cfg(feature = "dtype-u16")]
207            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
208            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
209            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
210            #[cfg(feature = "dtype-i8")]
211            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
212            #[cfg(feature = "dtype-i16")]
213            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
214            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
215            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
216            ArrowDataType::Int128 => feature_gated!(
217                "dtype-i128",
218                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
219            ),
220            ArrowDataType::Float16 => {
221                let chunks =
222                    cast_chunks(&chunks, &DataType::Float32, CastOptions::NonStrict).unwrap();
223                Ok(Float32Chunked::from_chunks(name, chunks).into_series())
224            },
225            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
226            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
227            #[cfg(feature = "dtype-date")]
228            ArrowDataType::Date32 => {
229                let chunks =
230                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
231                Ok(Int32Chunked::from_chunks(name, chunks)
232                    .into_date()
233                    .into_series())
234            },
235            #[cfg(feature = "dtype-datetime")]
236            ArrowDataType::Date64 => {
237                let chunks =
238                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
239                let ca = Int64Chunked::from_chunks(name, chunks);
240                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
241            },
242            #[cfg(feature = "dtype-datetime")]
243            ArrowDataType::Timestamp(tu, tz) => {
244                let canonical_tz = DataType::canonical_timezone(tz);
245                let tz = match canonical_tz.as_deref() {
246                    #[cfg(feature = "timezones")]
247                    Some(tz_str) => match validate_time_zone(tz_str) {
248                        Ok(_) => canonical_tz,
249                        Err(_) => Some(parse_fixed_offset(tz_str)?),
250                    },
251                    _ => canonical_tz,
252                };
253                let chunks =
254                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
255                let s = Int64Chunked::from_chunks(name, chunks)
256                    .into_datetime(tu.into(), tz)
257                    .into_series();
258                Ok(match tu {
259                    ArrowTimeUnit::Second => &s * MILLISECONDS,
260                    ArrowTimeUnit::Millisecond => s,
261                    ArrowTimeUnit::Microsecond => s,
262                    ArrowTimeUnit::Nanosecond => s,
263                })
264            },
265            #[cfg(feature = "dtype-duration")]
266            ArrowDataType::Duration(tu) => {
267                let chunks =
268                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
269                let s = Int64Chunked::from_chunks(name, chunks)
270                    .into_duration(tu.into())
271                    .into_series();
272                Ok(match tu {
273                    ArrowTimeUnit::Second => &s * MILLISECONDS,
274                    ArrowTimeUnit::Millisecond => s,
275                    ArrowTimeUnit::Microsecond => s,
276                    ArrowTimeUnit::Nanosecond => s,
277                })
278            },
279            #[cfg(feature = "dtype-time")]
280            ArrowDataType::Time64(tu) | ArrowDataType::Time32(tu) => {
281                let mut chunks = chunks;
282                if matches!(dtype, ArrowDataType::Time32(_)) {
283                    chunks =
284                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
285                }
286                let chunks =
287                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
288                let s = Int64Chunked::from_chunks(name, chunks)
289                    .into_time()
290                    .into_series();
291                Ok(match tu {
292                    ArrowTimeUnit::Second => &s * NANOSECONDS,
293                    ArrowTimeUnit::Millisecond => &s * 1_000_000,
294                    ArrowTimeUnit::Microsecond => &s * 1_000,
295                    ArrowTimeUnit::Nanosecond => s,
296                })
297            },
298            ArrowDataType::Decimal(precision, scale)
299            | ArrowDataType::Decimal256(precision, scale) => {
300                feature_gated!("dtype-decimal", {
301                    polars_ensure!(*scale <= *precision, InvalidOperation: "invalid decimal precision and scale (prec={precision}, scale={scale})");
302                    polars_ensure!(*precision <= 38, InvalidOperation: "polars does not support decimals about 38 precision");
303
304                    let mut chunks = chunks;
305                    // @NOTE: We cannot cast here as that will lower the scale.
306                    for chunk in chunks.iter_mut() {
307                        *chunk = std::mem::take(
308                            chunk
309                                .as_any_mut()
310                                .downcast_mut::<PrimitiveArray<i128>>()
311                                .unwrap(),
312                        )
313                        .to(ArrowDataType::Int128)
314                        .to_boxed();
315                    }
316                    let s = Int128Chunked::from_chunks(name, chunks)
317                        .into_decimal_unchecked(Some(*precision), *scale)
318                        .into_series();
319                    Ok(s)
320                })
321            },
322            ArrowDataType::Null => Ok(new_null(name, &chunks)),
323            #[cfg(not(feature = "dtype-categorical"))]
324            ArrowDataType::Dictionary(_, _, _) => {
325                panic!("activate dtype-categorical to convert dictionary arrays")
326            },
327            #[cfg(feature = "dtype-categorical")]
328            ArrowDataType::Dictionary(key_type, value_type, _) => {
329                use arrow::datatypes::IntegerType;
330                // don't spuriously call this; triggers a read on mmapped data
331                let arr = if chunks.len() > 1 {
332                    concatenate_unchecked(&chunks)?
333                } else {
334                    chunks[0].clone()
335                };
336
337                // If the value type is a string, they are converted to Categoricals or Enums
338                if matches!(
339                    value_type.as_ref(),
340                    ArrowDataType::Utf8
341                        | ArrowDataType::LargeUtf8
342                        | ArrowDataType::Utf8View
343                        | ArrowDataType::Null
344                ) {
345                    macro_rules! unpack_keys_values {
346                        ($dt:ty) => {{
347                            let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
348                            let keys = arr.keys();
349                            let keys = cast(keys, &ArrowDataType::UInt32).unwrap();
350                            let values = arr.values();
351                            let values = cast(&**values, &ArrowDataType::Utf8View)?;
352                            (keys, values)
353                        }};
354                    }
355
356                    use IntegerType as I;
357                    let (keys, values) = match key_type {
358                        I::Int8 => unpack_keys_values!(i8),
359                        I::UInt8 => unpack_keys_values!(u8),
360                        I::Int16 => unpack_keys_values!(i16),
361                        I::UInt16 => unpack_keys_values!(u16),
362                        I::Int32 => unpack_keys_values!(i32),
363                        I::UInt32 => unpack_keys_values!(u32),
364                        I::Int64 => unpack_keys_values!(i64),
365                        _ => polars_bail!(
366                            ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
367                        ),
368                    };
369
370                    let keys = keys.as_any().downcast_ref::<PrimitiveArray<u32>>().unwrap();
371                    let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
372
373                    // Categoricals and Enums expect the RevMap values to not contain any nulls
374                    let (keys, values) =
375                        polars_compute::propagate_dictionary::propagate_dictionary_value_nulls(
376                            keys, values,
377                        );
378
379                    let mut ordering = CategoricalOrdering::default();
380                    if let Some(metadata) = md {
381                        if metadata.is_enum() {
382                            // SAFETY:
383                            // the invariants of an Arrow Dictionary guarantee the keys are in bounds
384                            return Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
385                                UInt32Chunked::with_chunk(name, keys),
386                                Arc::new(RevMapping::build_local(values)),
387                                true,
388                                CategoricalOrdering::Physical, // Enum always uses physical ordering
389                            )
390                            .into_series());
391                        } else if let Some(o) = metadata.categorical() {
392                            ordering = o;
393                        }
394                    }
395
396                    return Ok(CategoricalChunked::from_keys_and_values(
397                        name, &keys, &values, ordering,
398                    )
399                    .into_series());
400                }
401
402                macro_rules! unpack_keys_values {
403                    ($dt:ty) => {{
404                        let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
405                        let keys = arr.keys();
406                        let keys = polars_compute::cast::primitive_as_primitive::<
407                            $dt,
408                            <IdxType as PolarsNumericType>::Native,
409                        >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
410                        (arr.values(), keys)
411                    }};
412                }
413
414                use IntegerType as I;
415                let (values, keys) = match key_type {
416                    I::Int8 => unpack_keys_values!(i8),
417                    I::UInt8 => unpack_keys_values!(u8),
418                    I::Int16 => unpack_keys_values!(i16),
419                    I::UInt16 => unpack_keys_values!(u16),
420                    I::Int32 => unpack_keys_values!(i32),
421                    I::UInt32 => unpack_keys_values!(u32),
422                    I::Int64 => unpack_keys_values!(i64),
423                    _ => polars_bail!(
424                        ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
425                    ),
426                };
427
428                // Convert the dictionary to a flat array
429                let values = Series::_try_from_arrow_unchecked_with_md(
430                    name,
431                    vec![values.clone()],
432                    values.dtype(),
433                    None,
434                )?;
435                let values = values.take_unchecked(&IdxCa::from_chunks_and_dtype(
436                    PlSmallStr::EMPTY,
437                    vec![keys.to_boxed()],
438                    IDX_DTYPE,
439                ));
440
441                Ok(values)
442            },
443            #[cfg(feature = "object")]
444            ArrowDataType::Extension(ext)
445                if ext.name == EXTENSION_NAME && ext.metadata.is_some() =>
446            {
447                assert_eq!(chunks.len(), 1);
448                let arr = chunks[0]
449                    .as_any()
450                    .downcast_ref::<FixedSizeBinaryArray>()
451                    .unwrap();
452                // SAFETY:
453                // this is highly unsafe. it will dereference a raw ptr on the heap
454                // make sure the ptr is allocated and from this pid
455                // (the pid is checked before dereference)
456                let s = {
457                    let pe = PolarsExtension::new(arr.clone());
458                    let s = pe.get_series(&name);
459                    pe.take_and_forget();
460                    s
461                };
462                Ok(s)
463            },
464            #[cfg(feature = "dtype-struct")]
465            ArrowDataType::Struct(_) => {
466                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
467
468                unsafe {
469                    let mut ca =
470                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
471                    ca.propagate_nulls();
472                    Ok(ca.into_series())
473                }
474            },
475            ArrowDataType::FixedSizeBinary(_) => {
476                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
477                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
478            },
479            ArrowDataType::Map(_, _) => map_arrays_to_series(name, chunks),
480            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
481        }
482    }
483}
484
485fn map_arrays_to_series(name: PlSmallStr, chunks: Vec<ArrayRef>) -> PolarsResult<Series> {
486    let chunks = chunks
487        .iter()
488        .map(|arr| {
489            // we convert the map to the logical type: List<struct<key, value>>
490            let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
491            let inner = arr.field().clone();
492
493            // map has i32 offsets
494            let dtype = ListArray::<i32>::default_datatype(inner.dtype().clone());
495            Box::new(ListArray::<i32>::new(
496                dtype,
497                arr.offsets().clone(),
498                inner,
499                arr.validity().cloned(),
500            )) as ArrayRef
501        })
502        .collect::<Vec<_>>();
503    Series::try_from((name, chunks))
504}
505
506fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
507    arr.iter().map(|arr| f(&**arr)).collect()
508}
509
510/// Converts to physical types and bubbles up the correct [`DataType`].
511#[allow(clippy::only_used_in_recursion)]
512unsafe fn to_physical_and_dtype(
513    arrays: Vec<ArrayRef>,
514    md: Option<&Metadata>,
515) -> (Vec<ArrayRef>, DataType) {
516    match arrays[0].dtype() {
517        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
518            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
519            (chunks, DataType::String)
520        },
521        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
522            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
523            (chunks, DataType::Binary)
524        },
525        #[allow(unused_variables)]
526        dt @ ArrowDataType::Dictionary(_, _, _) => {
527            feature_gated!("dtype-categorical", {
528                let s = unsafe {
529                    let dt = dt.clone();
530                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
531                }
532                .unwrap();
533                (s.chunks().clone(), s.dtype().clone())
534            })
535        },
536        ArrowDataType::List(field) => {
537            let out = convert(&arrays, |arr| {
538                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
539            });
540            to_physical_and_dtype(out, md)
541        },
542        #[cfg(feature = "dtype-array")]
543        ArrowDataType::FixedSizeList(field, size) => {
544            let values = arrays
545                .iter()
546                .map(|arr| {
547                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
548                    arr.values().clone()
549                })
550                .collect::<Vec<_>>();
551
552            let (converted_values, dtype) =
553                to_physical_and_dtype(values, field.metadata.as_deref());
554
555            let arrays = arrays
556                .iter()
557                .zip(converted_values)
558                .map(|(arr, values)| {
559                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
560
561                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
562                    Box::from(FixedSizeListArray::new(
563                        dtype,
564                        arr.len(),
565                        values,
566                        arr.validity().cloned(),
567                    )) as ArrayRef
568                })
569                .collect();
570            (arrays, DataType::Array(Box::new(dtype), *size))
571        },
572        ArrowDataType::LargeList(field) => {
573            let values = arrays
574                .iter()
575                .map(|arr| {
576                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
577                    arr.values().clone()
578                })
579                .collect::<Vec<_>>();
580
581            let (converted_values, dtype) =
582                to_physical_and_dtype(values, field.metadata.as_deref());
583
584            let arrays = arrays
585                .iter()
586                .zip(converted_values)
587                .map(|(arr, values)| {
588                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
589
590                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
591                    Box::from(ListArray::<i64>::new(
592                        dtype,
593                        arr.offsets().clone(),
594                        values,
595                        arr.validity().cloned(),
596                    )) as ArrayRef
597                })
598                .collect();
599            (arrays, DataType::List(Box::new(dtype)))
600        },
601        ArrowDataType::Struct(_fields) => {
602            feature_gated!("dtype-struct", {
603                let mut pl_fields = None;
604                let arrays = arrays
605                    .iter()
606                    .map(|arr| {
607                        let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
608                        let (values, dtypes): (Vec<_>, Vec<_>) = arr
609                            .values()
610                            .iter()
611                            .zip(_fields.iter())
612                            .map(|(value, field)| {
613                                let mut out = to_physical_and_dtype(
614                                    vec![value.clone()],
615                                    field.metadata.as_deref(),
616                                );
617                                (out.0.pop().unwrap(), out.1)
618                            })
619                            .unzip();
620
621                        let arrow_fields = values
622                            .iter()
623                            .zip(_fields.iter())
624                            .map(|(arr, field)| {
625                                ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
626                            })
627                            .collect();
628                        let arrow_array = Box::new(StructArray::new(
629                            ArrowDataType::Struct(arrow_fields),
630                            arr.len(),
631                            values,
632                            arr.validity().cloned(),
633                        )) as ArrayRef;
634
635                        if pl_fields.is_none() {
636                            pl_fields = Some(
637                                _fields
638                                    .iter()
639                                    .zip(dtypes)
640                                    .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
641                                    .collect_vec(),
642                            )
643                        }
644
645                        arrow_array
646                    })
647                    .collect_vec();
648
649                (arrays, DataType::Struct(pl_fields.unwrap()))
650            })
651        },
652        // Use Series architecture to convert nested logical types to physical.
653        dt @ (ArrowDataType::Duration(_)
654        | ArrowDataType::Time32(_)
655        | ArrowDataType::Time64(_)
656        | ArrowDataType::Timestamp(_, _)
657        | ArrowDataType::Date32
658        | ArrowDataType::Decimal(_, _)
659        | ArrowDataType::Date64) => {
660            let dt = dt.clone();
661            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt).unwrap();
662            let dtype = s.dtype().clone();
663            (std::mem::take(s.chunks_mut()), dtype)
664        },
665        dt => {
666            let dtype = DataType::from_arrow(dt, true, md);
667            (arrays, dtype)
668        },
669    }
670}
671
672fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
673    let mut chunks_iter = chunks.iter();
674    let dtype: ArrowDataType = chunks_iter
675        .next()
676        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
677        .dtype()
678        .clone();
679
680    for chunk in chunks_iter {
681        if chunk.dtype() != &dtype {
682            polars_bail!(
683                ComputeError: "cannot create series from multiple arrays with different types"
684            );
685        }
686    }
687    Ok(dtype)
688}
689
690impl Series {
691    pub fn try_new<T>(
692        name: PlSmallStr,
693        data: T,
694    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
695    where
696        (PlSmallStr, T): TryInto<Self>,
697    {
698        // # TODO
699        // * Remove the TryFrom<tuple> impls in favor of this
700        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
701    }
702}
703
704impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
705    type Error = PolarsError;
706
707    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
708        let (name, chunks) = name_arr;
709
710        let dtype = check_types(&chunks)?;
711        // SAFETY:
712        // dtype is checked
713        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
714    }
715}
716
717impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
718    type Error = PolarsError;
719
720    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
721        let (name, arr) = name_arr;
722        Series::try_from((name, vec![arr]))
723    }
724}
725
726impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
727    type Error = PolarsError;
728
729    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
730        let (field, chunks) = field_arr;
731
732        let dtype = check_types(&chunks)?;
733
734        // SAFETY:
735        // dtype is checked
736        unsafe {
737            Series::_try_from_arrow_unchecked_with_md(
738                field.name.clone(),
739                chunks,
740                &dtype,
741                field.metadata.as_deref(),
742            )
743        }
744    }
745}
746
747impl TryFrom<(&ArrowField, ArrayRef)> for Series {
748    type Error = PolarsError;
749
750    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
751        let (field, arr) = field_arr;
752        Series::try_from((field, vec![arr]))
753    }
754}
755
756/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
757/// into a [`Series`].
758/// # Safety
759///
760/// This trait is marked `unsafe` as the `is_series` return is used
761/// to transmute to `Series`. This must always return `false` except
762/// for `Series` structs.
763pub unsafe trait IntoSeries {
764    fn is_series() -> bool {
765        false
766    }
767
768    fn into_series(self) -> Series
769    where
770        Self: Sized;
771}
772
773impl<T> From<ChunkedArray<T>> for Series
774where
775    T: PolarsDataType,
776    ChunkedArray<T>: IntoSeries,
777{
778    fn from(ca: ChunkedArray<T>) -> Self {
779        ca.into_series()
780    }
781}
782
783#[cfg(feature = "dtype-date")]
784impl From<DateChunked> for Series {
785    fn from(a: DateChunked) -> Self {
786        a.into_series()
787    }
788}
789
790#[cfg(feature = "dtype-datetime")]
791impl From<DatetimeChunked> for Series {
792    fn from(a: DatetimeChunked) -> Self {
793        a.into_series()
794    }
795}
796
797#[cfg(feature = "dtype-duration")]
798impl From<DurationChunked> for Series {
799    fn from(a: DurationChunked) -> Self {
800        a.into_series()
801    }
802}
803
804#[cfg(feature = "dtype-time")]
805impl From<TimeChunked> for Series {
806    fn from(a: TimeChunked) -> Self {
807        a.into_series()
808    }
809}
810
811unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
812    fn into_series(self) -> Series {
813        Series(self)
814    }
815}
816
817unsafe impl IntoSeries for Series {
818    fn is_series() -> bool {
819        true
820    }
821
822    fn into_series(self) -> Series {
823        self
824    }
825}
826
827fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
828    let len = chunks.iter().map(|arr| arr.len()).sum();
829    Series::new_null(name, len)
830}
polars_core/series/from.rs

polars_core/series/
from.rs
