polars_core/series/
from.rs

1use arrow::datatypes::{IntervalUnit, Metadata};
2use arrow::offset::OffsetsBuffer;
3#[cfg(any(
4    feature = "dtype-date",
5    feature = "dtype-datetime",
6    feature = "dtype-time",
7    feature = "dtype-duration"
8))]
9use arrow::temporal_conversions::*;
10use arrow::types::months_days_ns;
11use polars_compute::cast::cast_unchecked as cast;
12#[cfg(feature = "dtype-decimal")]
13use polars_compute::decimal::dec128_fits;
14use polars_error::feature_gated;
15use polars_utils::check_allow_importing_interval_as_struct;
16use polars_utils::itertools::Itertools;
17
18use crate::chunked_array::cast::{CastOptions, cast_chunks};
19#[cfg(feature = "object")]
20use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
21#[cfg(feature = "object")]
22use crate::chunked_array::object::registry::get_object_builder;
23use crate::prelude::*;
24
25impl Series {
26    pub fn from_array<A: ParameterFreeDtypeStaticArray>(name: PlSmallStr, array: A) -> Self {
27        unsafe {
28            Self::from_chunks_and_dtype_unchecked(
29                name,
30                vec![Box::new(array)],
31                &DataType::from_arrow_dtype(&A::get_dtype()),
32            )
33        }
34    }
35
36    pub fn from_chunk_and_dtype(
37        name: PlSmallStr,
38        chunk: ArrayRef,
39        dtype: &DataType,
40    ) -> PolarsResult<Self> {
41        if &dtype.to_physical().to_arrow(CompatLevel::newest()) != chunk.dtype() {
42            polars_bail!(
43                InvalidOperation: "cannot create a series of type '{dtype}' of arrow chunk with type '{:?}'",
44                chunk.dtype()
45            );
46        }
47
48        // SAFETY: We check that the datatype matches.
49        let series = unsafe { Self::from_chunks_and_dtype_unchecked(name, vec![chunk], dtype) };
50        Ok(series)
51    }
52
53    /// Takes chunks and a polars datatype and constructs the Series.
54    /// This is faster than creating from chunks and an arrow datatype because there is no
55    /// casting involved.
56    ///
57    /// # Safety
58    ///
59    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
60    pub unsafe fn from_chunks_and_dtype_unchecked(
61        name: PlSmallStr,
62        chunks: Vec<ArrayRef>,
63        dtype: &DataType,
64    ) -> Self {
65        use DataType::*;
66        match dtype {
67            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
68            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
69            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
70            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
71            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
72            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
73            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
74            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
75            #[cfg(feature = "dtype-i128")]
76            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
77            #[cfg(feature = "dtype-u128")]
78            UInt128 => UInt128Chunked::from_chunks(name, chunks).into_series(),
79            #[cfg(feature = "dtype-date")]
80            Date => Int32Chunked::from_chunks(name, chunks)
81                .into_date()
82                .into_series(),
83            #[cfg(feature = "dtype-time")]
84            Time => Int64Chunked::from_chunks(name, chunks)
85                .into_time()
86                .into_series(),
87            #[cfg(feature = "dtype-duration")]
88            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
89                .into_duration(*tu)
90                .into_series(),
91            #[cfg(feature = "dtype-datetime")]
92            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
93                .into_datetime(*tu, tz.clone())
94                .into_series(),
95            #[cfg(feature = "dtype-decimal")]
96            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
97                .into_decimal_unchecked(*precision, *scale)
98                .into_series(),
99            #[cfg(feature = "dtype-array")]
100            Array(_, _) => {
101                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
102                    .into_series()
103            },
104            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
105                .into_series(),
106            String => StringChunked::from_chunks(name, chunks).into_series(),
107            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
108            #[cfg(feature = "dtype-categorical")]
109            dt @ (Categorical(_, _) | Enum(_, _)) => {
110                with_match_categorical_physical_type!(dt.cat_physical().unwrap(), |$C| {
111                    let phys = ChunkedArray::from_chunks(name, chunks);
112                    CategoricalChunked::<$C>::from_cats_and_dtype_unchecked(phys, dt.clone()).into_series()
113                })
114            },
115            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
116            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
117            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
118            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
119            #[cfg(feature = "dtype-struct")]
120            Struct(_) => {
121                let mut ca =
122                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
123                StructChunked::propagate_nulls_mut(&mut ca);
124                ca.into_series()
125            },
126            #[cfg(feature = "object")]
127            Object(_) => {
128                if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
129                    assert_eq!(chunks.len(), 1);
130                    // SAFETY:
131                    // this is highly unsafe. it will dereference a raw ptr on the heap
132                    // make sure the ptr is allocated and from this pid
133                    // (the pid is checked before dereference)
134                    {
135                        let pe = PolarsExtension::new(arr.clone());
136                        let s = pe.get_series(&name);
137                        pe.take_and_forget();
138                        s
139                    }
140                } else {
141                    unsafe { get_object_builder(name, 0).from_chunks(chunks) }
142                }
143            },
144            Null => new_null(name, &chunks),
145            Unknown(_) => {
146                panic!("dtype is unknown; consider supplying data-types for all operations")
147            },
148            #[allow(unreachable_patterns)]
149            _ => unreachable!(),
150        }
151    }
152
153    /// # Safety
154    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
155    pub unsafe fn _try_from_arrow_unchecked(
156        name: PlSmallStr,
157        chunks: Vec<ArrayRef>,
158        dtype: &ArrowDataType,
159    ) -> PolarsResult<Self> {
160        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
161    }
162
163    /// Create a new Series without checking if the inner dtype of the chunks is correct
164    ///
165    /// # Safety
166    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
167    pub unsafe fn _try_from_arrow_unchecked_with_md(
168        name: PlSmallStr,
169        chunks: Vec<ArrayRef>,
170        dtype: &ArrowDataType,
171        md: Option<&Metadata>,
172    ) -> PolarsResult<Self> {
173        match dtype {
174            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
175            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
176                let chunks =
177                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
178                Ok(StringChunked::from_chunks(name, chunks).into_series())
179            },
180            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
181            ArrowDataType::LargeBinary => {
182                if let Some(md) = md {
183                    if md.maintain_type() {
184                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
185                    }
186                }
187                let chunks =
188                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
189                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
190            },
191            ArrowDataType::Binary => {
192                let chunks =
193                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
194                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
195            },
196            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
197                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
198                unsafe {
199                    Ok(
200                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
201                            .into_series(),
202                    )
203                }
204            },
205            #[cfg(feature = "dtype-array")]
206            ArrowDataType::FixedSizeList(_, _) => {
207                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
208                unsafe {
209                    Ok(
210                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
211                            .into_series(),
212                    )
213                }
214            },
215            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
216            #[cfg(feature = "dtype-u8")]
217            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
218            #[cfg(feature = "dtype-u16")]
219            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
220            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
221            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
222            ArrowDataType::UInt128 => feature_gated!(
223                "dtype-u128",
224                Ok(UInt128Chunked::from_chunks(name, chunks).into_series())
225            ),
226            #[cfg(feature = "dtype-i8")]
227            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
228            #[cfg(feature = "dtype-i16")]
229            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
230            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
231            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
232            ArrowDataType::Int128 => feature_gated!(
233                "dtype-i128",
234                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
235            ),
236            ArrowDataType::Float16 => {
237                let chunks =
238                    cast_chunks(&chunks, &DataType::Float32, CastOptions::NonStrict).unwrap();
239                Ok(Float32Chunked::from_chunks(name, chunks).into_series())
240            },
241            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
242            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
243            #[cfg(feature = "dtype-date")]
244            ArrowDataType::Date32 => {
245                let chunks =
246                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
247                Ok(Int32Chunked::from_chunks(name, chunks)
248                    .into_date()
249                    .into_series())
250            },
251            #[cfg(feature = "dtype-datetime")]
252            ArrowDataType::Date64 => {
253                let chunks =
254                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
255                let ca = Int64Chunked::from_chunks(name, chunks);
256                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
257            },
258            #[cfg(feature = "dtype-datetime")]
259            ArrowDataType::Timestamp(tu, tz) => {
260                let tz = TimeZone::opt_try_new(tz.clone())?;
261                let chunks =
262                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
263                let s = Int64Chunked::from_chunks(name, chunks)
264                    .into_datetime(tu.into(), tz)
265                    .into_series();
266                Ok(match tu {
267                    ArrowTimeUnit::Second => &s * MILLISECONDS,
268                    ArrowTimeUnit::Millisecond => s,
269                    ArrowTimeUnit::Microsecond => s,
270                    ArrowTimeUnit::Nanosecond => s,
271                })
272            },
273            #[cfg(feature = "dtype-duration")]
274            ArrowDataType::Duration(tu) => {
275                let chunks =
276                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
277                let s = Int64Chunked::from_chunks(name, chunks)
278                    .into_duration(tu.into())
279                    .into_series();
280                Ok(match tu {
281                    ArrowTimeUnit::Second => &s * MILLISECONDS,
282                    ArrowTimeUnit::Millisecond => s,
283                    ArrowTimeUnit::Microsecond => s,
284                    ArrowTimeUnit::Nanosecond => s,
285                })
286            },
287            #[cfg(feature = "dtype-time")]
288            ArrowDataType::Time64(tu) | ArrowDataType::Time32(tu) => {
289                let mut chunks = chunks;
290                if matches!(dtype, ArrowDataType::Time32(_)) {
291                    chunks =
292                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
293                }
294                let chunks =
295                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
296                let s = Int64Chunked::from_chunks(name, chunks)
297                    .into_time()
298                    .into_series();
299                Ok(match tu {
300                    ArrowTimeUnit::Second => &s * NANOSECONDS,
301                    ArrowTimeUnit::Millisecond => &s * 1_000_000,
302                    ArrowTimeUnit::Microsecond => &s * 1_000,
303                    ArrowTimeUnit::Nanosecond => s,
304                })
305            },
306            ArrowDataType::Decimal32(precision, scale) => {
307                feature_gated!("dtype-decimal", {
308                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
309
310                    let mut chunks = chunks;
311                    for chunk in chunks.iter_mut() {
312                        let old_chunk = chunk
313                            .as_any_mut()
314                            .downcast_mut::<PrimitiveArray<i32>>()
315                            .unwrap();
316
317                        // For now, we just cast the whole data to i128.
318                        let (_, values, validity) = std::mem::take(old_chunk).into_inner();
319                        *chunk = PrimitiveArray::new(
320                            ArrowDataType::Int128,
321                            values.iter().map(|&v| v as i128).collect(),
322                            validity,
323                        )
324                        .to_boxed();
325                    }
326
327                    let s = Int128Chunked::from_chunks(name, chunks)
328                        .into_decimal_unchecked(*precision, *scale)
329                        .into_series();
330                    Ok(s)
331                })
332            },
333            ArrowDataType::Decimal64(precision, scale) => {
334                feature_gated!("dtype-decimal", {
335                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
336
337                    let mut chunks = chunks;
338                    for chunk in chunks.iter_mut() {
339                        let old_chunk = chunk
340                            .as_any_mut()
341                            .downcast_mut::<PrimitiveArray<i64>>()
342                            .unwrap();
343
344                        // For now, we just cast the whole data to i128.
345                        let (_, values, validity) = std::mem::take(old_chunk).into_inner();
346                        *chunk = PrimitiveArray::new(
347                            ArrowDataType::Int128,
348                            values.iter().map(|&v| v as i128).collect(),
349                            validity,
350                        )
351                        .to_boxed();
352                    }
353
354                    let s = Int128Chunked::from_chunks(name, chunks)
355                        .into_decimal_unchecked(*precision, *scale)
356                        .into_series();
357                    Ok(s)
358                })
359            },
360            ArrowDataType::Decimal(precision, scale) => {
361                feature_gated!("dtype-decimal", {
362                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
363
364                    let mut chunks = chunks;
365                    for chunk in chunks.iter_mut() {
366                        *chunk = std::mem::take(
367                            chunk
368                                .as_any_mut()
369                                .downcast_mut::<PrimitiveArray<i128>>()
370                                .unwrap(),
371                        )
372                        .to(ArrowDataType::Int128)
373                        .to_boxed();
374                    }
375
376                    let s = Int128Chunked::from_chunks(name, chunks)
377                        .into_decimal_unchecked(*precision, *scale)
378                        .into_series();
379                    Ok(s)
380                })
381            },
382            ArrowDataType::Decimal256(precision, scale) => {
383                feature_gated!("dtype-decimal", {
384                    use arrow::types::i256;
385
386                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
387
388                    let mut chunks = chunks;
389                    for chunk in chunks.iter_mut() {
390                        let arr = std::mem::take(
391                            chunk
392                                .as_any_mut()
393                                .downcast_mut::<PrimitiveArray<i256>>()
394                                .unwrap(),
395                        );
396                        let arr_128: PrimitiveArray<i128> = arr.iter().map(|opt_v| {
397                            if let Some(v) = opt_v {
398                                let smaller: Option<i128> = (*v).try_into().ok();
399                                let smaller = smaller.filter(|v| dec128_fits(*v, *precision));
400                                smaller.ok_or_else(|| {
401                                    polars_err!(ComputeError: "Decimal256 to Decimal128 conversion overflowed, Decimal256 is not (yet) supported in Polars")
402                                }).map(Some)
403                            } else {
404                                Ok(None)
405                            }
406                        }).try_collect_arr_trusted()?;
407
408                        *chunk = arr_128.to(ArrowDataType::Int128).to_boxed();
409                    }
410
411                    let s = Int128Chunked::from_chunks(name, chunks)
412                        .into_decimal_unchecked(*precision, *scale)
413                        .into_series();
414                    Ok(s)
415                })
416            },
417            ArrowDataType::Null => Ok(new_null(name, &chunks)),
418            #[cfg(not(feature = "dtype-categorical"))]
419            ArrowDataType::Dictionary(_, _, _) => {
420                panic!("activate dtype-categorical to convert dictionary arrays")
421            },
422            #[cfg(feature = "dtype-categorical")]
423            ArrowDataType::Dictionary(key_type, _, _) => {
424                let polars_dtype = DataType::from_arrow(chunks[0].dtype(), md);
425
426                let mut series_iter = chunks.into_iter().map(|arr| {
427                    import_arrow_dictionary_array(name.clone(), arr, key_type, &polars_dtype)
428                });
429
430                let mut first = series_iter.next().unwrap()?;
431
432                for s in series_iter {
433                    first.append_owned(s?)?;
434                }
435
436                Ok(first)
437            },
438            #[cfg(feature = "object")]
439            ArrowDataType::Extension(ext)
440                if ext.name == EXTENSION_NAME && ext.metadata.is_some() =>
441            {
442                assert_eq!(chunks.len(), 1);
443                let arr = chunks[0]
444                    .as_any()
445                    .downcast_ref::<FixedSizeBinaryArray>()
446                    .unwrap();
447                // SAFETY:
448                // this is highly unsafe. it will dereference a raw ptr on the heap
449                // make sure the ptr is allocated and from this pid
450                // (the pid is checked before dereference)
451                let s = {
452                    let pe = PolarsExtension::new(arr.clone());
453                    let s = pe.get_series(&name);
454                    pe.take_and_forget();
455                    s
456                };
457                Ok(s)
458            },
459            #[cfg(feature = "dtype-struct")]
460            ArrowDataType::Struct(_) => {
461                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
462
463                unsafe {
464                    let mut ca =
465                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
466                    StructChunked::propagate_nulls_mut(&mut ca);
467                    Ok(ca.into_series())
468                }
469            },
470            ArrowDataType::FixedSizeBinary(_) => {
471                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
472                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
473            },
474            ArrowDataType::Map(field, _is_ordered) => {
475                let struct_arrays = chunks
476                    .iter()
477                    .map(|arr| {
478                        let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
479                        arr.field().clone()
480                    })
481                    .collect::<Vec<_>>();
482
483                let (phys_struct_arrays, dtype) =
484                    to_physical_and_dtype(struct_arrays, field.metadata.as_deref());
485
486                let chunks = chunks
487                    .iter()
488                    .zip(phys_struct_arrays)
489                    .map(|(arr, values)| {
490                        let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
491                        let offsets: &OffsetsBuffer<i32> = arr.offsets();
492
493                        let validity = values.validity().cloned();
494
495                        Box::from(ListArray::<i64>::new(
496                            ListArray::<i64>::default_datatype(values.dtype().clone()),
497                            OffsetsBuffer::<i64>::from(offsets),
498                            values,
499                            validity,
500                        )) as ArrayRef
501                    })
502                    .collect();
503
504                unsafe {
505                    let out = ListChunked::from_chunks_and_dtype_unchecked(
506                        name,
507                        chunks,
508                        DataType::List(Box::new(dtype)),
509                    );
510
511                    Ok(out.into_series())
512                }
513            },
514            ArrowDataType::Interval(IntervalUnit::MonthDayNano) => {
515                check_allow_importing_interval_as_struct("month_day_nano_interval")?;
516
517                feature_gated!("dtype-struct", {
518                    let chunks = chunks
519                        .into_iter()
520                        .map(convert_month_day_nano_to_struct)
521                        .collect::<PolarsResult<Vec<_>>>()?;
522
523                    Ok(StructChunked::from_chunks_and_dtype_unchecked(
524                        name,
525                        chunks,
526                        DataType::_month_days_ns_struct_type(),
527                    )
528                    .into_series())
529                })
530            },
531            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
532        }
533    }
534}
535
536fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
537    arr.iter().map(|arr| f(&**arr)).collect()
538}
539
540/// Converts to physical types and bubbles up the correct [`DataType`].
541#[allow(clippy::only_used_in_recursion)]
542unsafe fn to_physical_and_dtype(
543    arrays: Vec<ArrayRef>,
544    md: Option<&Metadata>,
545) -> (Vec<ArrayRef>, DataType) {
546    match arrays[0].dtype() {
547        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
548            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
549            (chunks, DataType::String)
550        },
551        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
552            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
553            (chunks, DataType::Binary)
554        },
555        #[allow(unused_variables)]
556        dt @ ArrowDataType::Dictionary(_, _, _) => {
557            feature_gated!("dtype-categorical", {
558                let s = unsafe {
559                    let dt = dt.clone();
560                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
561                }
562                .unwrap();
563                (s.chunks().clone(), s.dtype().clone())
564            })
565        },
566        ArrowDataType::List(field) => {
567            let out = convert(&arrays, |arr| {
568                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
569            });
570            to_physical_and_dtype(out, md)
571        },
572        #[cfg(feature = "dtype-array")]
573        ArrowDataType::FixedSizeList(field, size) => {
574            let values = arrays
575                .iter()
576                .map(|arr| {
577                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
578                    arr.values().clone()
579                })
580                .collect::<Vec<_>>();
581
582            let (converted_values, dtype) =
583                to_physical_and_dtype(values, field.metadata.as_deref());
584
585            let arrays = arrays
586                .iter()
587                .zip(converted_values)
588                .map(|(arr, values)| {
589                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
590
591                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
592                    Box::from(FixedSizeListArray::new(
593                        dtype,
594                        arr.len(),
595                        values,
596                        arr.validity().cloned(),
597                    )) as ArrayRef
598                })
599                .collect();
600            (arrays, DataType::Array(Box::new(dtype), *size))
601        },
602        ArrowDataType::LargeList(field) => {
603            let values = arrays
604                .iter()
605                .map(|arr| {
606                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
607                    arr.values().clone()
608                })
609                .collect::<Vec<_>>();
610
611            let (converted_values, dtype) =
612                to_physical_and_dtype(values, field.metadata.as_deref());
613
614            let arrays = arrays
615                .iter()
616                .zip(converted_values)
617                .map(|(arr, values)| {
618                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
619
620                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
621                    Box::from(ListArray::<i64>::new(
622                        dtype,
623                        arr.offsets().clone(),
624                        values,
625                        arr.validity().cloned(),
626                    )) as ArrayRef
627                })
628                .collect();
629            (arrays, DataType::List(Box::new(dtype)))
630        },
631        ArrowDataType::Struct(_fields) => {
632            feature_gated!("dtype-struct", {
633                let mut pl_fields = None;
634                let arrays = arrays
635                    .iter()
636                    .map(|arr| {
637                        let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
638                        let (values, dtypes): (Vec<_>, Vec<_>) = arr
639                            .values()
640                            .iter()
641                            .zip(_fields.iter())
642                            .map(|(value, field)| {
643                                let mut out = to_physical_and_dtype(
644                                    vec![value.clone()],
645                                    field.metadata.as_deref(),
646                                );
647                                (out.0.pop().unwrap(), out.1)
648                            })
649                            .unzip();
650
651                        let arrow_fields = values
652                            .iter()
653                            .zip(_fields.iter())
654                            .map(|(arr, field)| {
655                                ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
656                            })
657                            .collect();
658                        let arrow_array = Box::new(StructArray::new(
659                            ArrowDataType::Struct(arrow_fields),
660                            arr.len(),
661                            values,
662                            arr.validity().cloned(),
663                        )) as ArrayRef;
664
665                        if pl_fields.is_none() {
666                            pl_fields = Some(
667                                _fields
668                                    .iter()
669                                    .zip(dtypes)
670                                    .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
671                                    .collect_vec(),
672                            )
673                        }
674
675                        arrow_array
676                    })
677                    .collect_vec();
678
679                (arrays, DataType::Struct(pl_fields.unwrap()))
680            })
681        },
682        // Use Series architecture to convert nested logical types to physical.
683        dt @ (ArrowDataType::Duration(_)
684        | ArrowDataType::Time32(_)
685        | ArrowDataType::Time64(_)
686        | ArrowDataType::Timestamp(_, _)
687        | ArrowDataType::Date32
688        | ArrowDataType::Decimal(_, _)
689        | ArrowDataType::Date64
690        | ArrowDataType::Map(_, _)) => {
691            let dt = dt.clone();
692            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt).unwrap();
693            let dtype = s.dtype().clone();
694            (std::mem::take(s.chunks_mut()), dtype)
695        },
696        dt => {
697            let dtype = DataType::from_arrow(dt, md);
698            (arrays, dtype)
699        },
700    }
701}
702
703#[cfg(feature = "dtype-categorical")]
704unsafe fn import_arrow_dictionary_array(
705    name: PlSmallStr,
706    arr: Box<dyn Array>,
707    key_type: &arrow::datatypes::IntegerType,
708    polars_dtype: &DataType,
709) -> PolarsResult<Series> {
710    use arrow::datatypes::IntegerType as I;
711
712    if matches!(
713        polars_dtype,
714        DataType::Categorical(_, _) | DataType::Enum(_, _)
715    ) {
716        macro_rules! unpack_categorical_chunked {
717            ($dt:ty) => {{
718                let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
719                let keys = arr.keys();
720                let values = arr.values();
721                let values = cast(&**values, &ArrowDataType::Utf8View)?;
722                let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
723                with_match_categorical_physical_type!(polars_dtype.cat_physical().unwrap(), |$C| {
724                    let ca = CategoricalChunked::<$C>::from_str_iter(
725                        name,
726                        polars_dtype.clone(),
727                        keys.iter().map(|k| {
728                            let k: usize = (*k?).try_into().ok()?;
729                            values.get(k)
730                        }),
731                    )?;
732                    Ok(ca.into_series())
733                })
734            }};
735        }
736
737        match key_type {
738            I::Int8 => unpack_categorical_chunked!(i8),
739            I::UInt8 => unpack_categorical_chunked!(u8),
740            I::Int16 => unpack_categorical_chunked!(i16),
741            I::UInt16 => unpack_categorical_chunked!(u16),
742            I::Int32 => unpack_categorical_chunked!(i32),
743            I::UInt32 => unpack_categorical_chunked!(u32),
744            I::Int64 => unpack_categorical_chunked!(i64),
745            I::UInt64 => unpack_categorical_chunked!(u64),
746            _ => polars_bail!(
747                ComputeError: "unsupported arrow key type: {key_type:?}"
748            ),
749        }
750    } else {
751        macro_rules! unpack_keys_values {
752            ($dt:ty) => {{
753                let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
754                let keys = arr.keys();
755                let keys = polars_compute::cast::primitive_to_primitive::<
756                    $dt,
757                    <IdxType as PolarsNumericType>::Native,
758                >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
759                (keys, arr.values())
760            }};
761        }
762
763        let (keys, values) = match key_type {
764            I::Int8 => unpack_keys_values!(i8),
765            I::UInt8 => unpack_keys_values!(u8),
766            I::Int16 => unpack_keys_values!(i16),
767            I::UInt16 => unpack_keys_values!(u16),
768            I::Int32 => unpack_keys_values!(i32),
769            I::UInt32 => unpack_keys_values!(u32),
770            I::Int64 => unpack_keys_values!(i64),
771            I::UInt64 => unpack_keys_values!(u64),
772            _ => polars_bail!(
773                ComputeError: "unsupported arrow key type: {key_type:?}"
774            ),
775        };
776
777        let values = Series::_try_from_arrow_unchecked_with_md(
778            name,
779            vec![values.clone()],
780            values.dtype(),
781            None,
782        )?;
783
784        values.take(&IdxCa::from_chunks_and_dtype(
785            PlSmallStr::EMPTY,
786            vec![keys.to_boxed()],
787            IDX_DTYPE,
788        ))
789    }
790}
791
792#[cfg(feature = "dtype-struct")]
793fn convert_month_day_nano_to_struct(chunk: Box<dyn Array>) -> PolarsResult<Box<dyn Array>> {
794    let arr: &PrimitiveArray<months_days_ns> = chunk.as_any().downcast_ref().unwrap();
795
796    let values: &[months_days_ns] = arr.values();
797
798    let (months_out, days_out, nanoseconds_out): (Vec<i32>, Vec<i32>, Vec<i64>) = values
799        .iter()
800        .map(|x| (x.months(), x.days(), x.ns()))
801        .collect();
802
803    let out = StructArray::new(
804        DataType::_month_days_ns_struct_type()
805            .to_physical()
806            .to_arrow(CompatLevel::newest()),
807        arr.len(),
808        vec![
809            PrimitiveArray::<i32>::from_vec(months_out).boxed(),
810            PrimitiveArray::<i32>::from_vec(days_out).boxed(),
811            PrimitiveArray::<i64>::from_vec(nanoseconds_out).boxed(),
812        ],
813        arr.validity().cloned(),
814    );
815
816    Ok(out.boxed())
817}
818
819fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
820    let mut chunks_iter = chunks.iter();
821    let dtype: ArrowDataType = chunks_iter
822        .next()
823        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
824        .dtype()
825        .clone();
826
827    for chunk in chunks_iter {
828        if chunk.dtype() != &dtype {
829            polars_bail!(
830                ComputeError: "cannot create series from multiple arrays with different types"
831            );
832        }
833    }
834    Ok(dtype)
835}
836
837impl Series {
838    pub fn try_new<T>(
839        name: PlSmallStr,
840        data: T,
841    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
842    where
843        (PlSmallStr, T): TryInto<Self>,
844    {
845        // # TODO
846        // * Remove the TryFrom<tuple> impls in favor of this
847        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
848    }
849}
850
851impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
852    type Error = PolarsError;
853
854    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
855        let (name, chunks) = name_arr;
856
857        let dtype = check_types(&chunks)?;
858        // SAFETY:
859        // dtype is checked
860        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
861    }
862}
863
864impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
865    type Error = PolarsError;
866
867    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
868        let (name, arr) = name_arr;
869        Series::try_from((name, vec![arr]))
870    }
871}
872
873impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
874    type Error = PolarsError;
875
876    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
877        let (field, chunks) = field_arr;
878
879        let dtype = check_types(&chunks)?;
880
881        // SAFETY:
882        // dtype is checked
883        unsafe {
884            Series::_try_from_arrow_unchecked_with_md(
885                field.name.clone(),
886                chunks,
887                &dtype,
888                field.metadata.as_deref(),
889            )
890        }
891    }
892}
893
894impl TryFrom<(&ArrowField, ArrayRef)> for Series {
895    type Error = PolarsError;
896
897    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
898        let (field, arr) = field_arr;
899        Series::try_from((field, vec![arr]))
900    }
901}
902
903/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
904/// into a [`Series`].
905/// # Safety
906///
907/// This trait is marked `unsafe` as the `is_series` return is used
908/// to transmute to `Series`. This must always return `false` except
909/// for `Series` structs.
910pub unsafe trait IntoSeries {
911    fn is_series() -> bool {
912        false
913    }
914
915    fn into_series(self) -> Series
916    where
917        Self: Sized;
918}
919
920impl<T> From<ChunkedArray<T>> for Series
921where
922    T: PolarsDataType,
923    ChunkedArray<T>: IntoSeries,
924{
925    fn from(ca: ChunkedArray<T>) -> Self {
926        ca.into_series()
927    }
928}
929
930#[cfg(feature = "dtype-date")]
931impl From<DateChunked> for Series {
932    fn from(a: DateChunked) -> Self {
933        a.into_series()
934    }
935}
936
937#[cfg(feature = "dtype-datetime")]
938impl From<DatetimeChunked> for Series {
939    fn from(a: DatetimeChunked) -> Self {
940        a.into_series()
941    }
942}
943
944#[cfg(feature = "dtype-duration")]
945impl From<DurationChunked> for Series {
946    fn from(a: DurationChunked) -> Self {
947        a.into_series()
948    }
949}
950
951#[cfg(feature = "dtype-time")]
952impl From<TimeChunked> for Series {
953    fn from(a: TimeChunked) -> Self {
954        a.into_series()
955    }
956}
957
958unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
959    fn into_series(self) -> Series {
960        Series(self)
961    }
962}
963
964unsafe impl IntoSeries for Series {
965    fn is_series() -> bool {
966        true
967    }
968
969    fn into_series(self) -> Series {
970        self
971    }
972}
973
974fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
975    let len = chunks.iter().map(|arr| arr.len()).sum();
976    Series::new_null(name, len)
977}
polars_core/series/from.rs

polars_core/series/
from.rs
