polars_core/series/
from.rs

1use polars_arrow::datatypes::{IntervalUnit, Metadata};
2use polars_arrow::offset::OffsetsBuffer;
3use polars_arrow::types::months_days_ns;
4use polars_compute::cast::cast_unchecked as cast;
5#[cfg(feature = "dtype-decimal")]
6use polars_compute::decimal::dec128_fits;
7use polars_error::feature_gated;
8use polars_utils::itertools::Itertools;
9
10use crate::chunked_array::cast::{CastOptions, cast_chunks};
11#[cfg(feature = "object")]
12use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
13#[cfg(feature = "object")]
14use crate::chunked_array::object::registry::get_object_builder;
15use crate::config::check_allow_importing_interval_as_struct;
16use crate::prelude::*;
17
18impl Series {
19    pub fn from_array<A: ParameterFreeDtypeStaticArray>(name: PlSmallStr, array: A) -> Self {
20        unsafe {
21            Self::from_chunks_and_dtype_unchecked(
22                name,
23                vec![Box::new(array)],
24                &DataType::from_arrow_dtype(&A::get_dtype()),
25            )
26        }
27    }
28
29    /// Construct a Series from a chunk holding the physical representation of `dtype`.
30    ///
31    /// Checks the physical dtype and validates values through [`Series::try_from_physical`].
32    pub fn from_chunk_and_dtype(
33        name: PlSmallStr,
34        chunk: ArrayRef,
35        dtype: &DataType,
36    ) -> PolarsResult<Self> {
37        // Reject objects before construction can reinterpret the chunk as pointers.
38        polars_ensure!(
39            !dtype.contains_objects(),
40            InvalidOperation: "cannot create a series of type '{dtype}' from an arrow chunk: objects are process-local"
41        );
42        polars_ensure!(
43            !dtype.contains_unknown(),
44            InvalidOperation: "cannot create a series of type '{dtype}' from an arrow chunk"
45        );
46        #[cfg(feature = "dtype-map")]
47        dtype.ensure_valid_map_dtypes()?;
48
49        let physical = dtype.to_physical();
50        if &physical.to_arrow(CompatLevel::newest()) != chunk.dtype() {
51            polars_bail!(
52                InvalidOperation: "cannot create a series of type '{dtype}' of arrow chunk with type '{:?}'",
53                chunk.dtype()
54            );
55        }
56
57        // SAFETY: the chunk matches the physical dtype, checked above.
58        let physical =
59            unsafe { Self::from_chunks_and_dtype_unchecked(name, vec![chunk], &physical) };
60        physical.try_from_physical(dtype)
61    }
62
63    /// Takes chunks and a polars datatype and constructs the Series.
64    /// This is faster than creating from chunks and an arrow datatype because there is no
65    /// casting involved.
66    ///
67    /// # Safety
68    ///
69    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
70    ///
71    /// Payloads must also be safe to read as the logical dtype:
72    ///
73    /// - `Categorical` / `Enum`: every non-null code names a category;
74    /// - `Object`: chunks originate from this process;
75    /// - `Map`: storage satisfies the `MapChunked` storage safety contract, so entries and
76    ///   keys are non-null within the offset windows of live rows. Keys may repeat.
77    pub unsafe fn from_chunks_and_dtype_unchecked(
78        name: PlSmallStr,
79        chunks: Vec<ArrayRef>,
80        dtype: &DataType,
81    ) -> Self {
82        use DataType::*;
83        match dtype {
84            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
85            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
86            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
87            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
88            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
89            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
90            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
91            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
92            #[cfg(feature = "dtype-i128")]
93            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
94            #[cfg(feature = "dtype-u128")]
95            UInt128 => UInt128Chunked::from_chunks(name, chunks).into_series(),
96            #[cfg(feature = "dtype-date")]
97            Date => Int32Chunked::from_chunks(name, chunks)
98                .into_date()
99                .into_series(),
100            #[cfg(feature = "dtype-time")]
101            Time => Int64Chunked::from_chunks(name, chunks)
102                .into_time()
103                .into_series(),
104            #[cfg(feature = "dtype-duration")]
105            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
106                .into_duration(*tu)
107                .into_series(),
108            #[cfg(feature = "dtype-datetime")]
109            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
110                .into_datetime(*tu, tz.clone())
111                .into_series(),
112            #[cfg(feature = "dtype-decimal")]
113            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
114                .into_decimal_unchecked(*precision, *scale)
115                .into_series(),
116            #[cfg(feature = "dtype-array")]
117            Array(_, _) => {
118                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
119                    .into_series()
120            },
121            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
122                .into_series(),
123            String => StringChunked::from_chunks(name, chunks).into_series(),
124            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
125            #[cfg(feature = "dtype-categorical")]
126            dt @ (Categorical(_, _) | Enum(_, _)) => {
127                with_match_categorical_physical_type!(dt.cat_physical().unwrap(), |$C| {
128                    let phys = ChunkedArray::from_chunks(name, chunks);
129                    CategoricalChunked::<$C>::from_cats_and_dtype_unchecked(phys, dt.clone()).into_series()
130                })
131            },
132            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
133            #[cfg(feature = "dtype-f16")]
134            Float16 => Float16Chunked::from_chunks(name, chunks).into_series(),
135            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
136            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
137            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
138            #[cfg(feature = "dtype-extension")]
139            Extension(typ, storage) => ExtensionChunked::from_storage(
140                typ.clone(),
141                Series::from_chunks_and_dtype_unchecked(name, chunks, storage),
142            )
143            .into_series(),
144            #[cfg(feature = "dtype-map")]
145            Map(_, _) => {
146                let storage = Series::from_chunks_and_dtype_unchecked(
147                    name,
148                    chunks,
149                    &dtype.map_storage_dtype().unwrap(),
150                );
151                MapChunked::from_storage_unchecked(dtype.clone(), storage).into_series()
152            },
153            #[cfg(feature = "dtype-struct")]
154            Struct(_) => {
155                let mut ca =
156                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
157                StructChunked::propagate_nulls_mut(&mut ca);
158                ca.into_series()
159            },
160            #[cfg(feature = "object")]
161            Object(_) => {
162                if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
163                    assert_eq!(chunks.len(), 1);
164                    // SAFETY:
165                    // this is highly unsafe. it will dereference a raw ptr on the heap
166                    // make sure the ptr is allocated and from this pid
167                    // (the pid is checked before dereference)
168                    {
169                        let pe = PolarsExtension::new(arr.clone());
170                        let s = pe.get_series(&name);
171                        pe.take_and_forget();
172                        s
173                    }
174                } else {
175                    unsafe { get_object_builder(name, 0).from_chunks(chunks) }
176                }
177            },
178            Null => new_null(name, &chunks),
179            Unknown(_) => {
180                panic!("dtype is unknown; consider supplying data-types for all operations")
181            },
182            #[allow(unreachable_patterns)]
183            _ => unreachable!(),
184        }
185    }
186
187    /// # Safety
188    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
189    pub unsafe fn _try_from_arrow_unchecked(
190        name: PlSmallStr,
191        chunks: Vec<ArrayRef>,
192        dtype: &ArrowDataType,
193    ) -> PolarsResult<Self> {
194        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
195    }
196
197    /// Create a new Series without checking if the inner dtype of the chunks is correct
198    ///
199    /// # Safety
200    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
201    pub unsafe fn _try_from_arrow_unchecked_with_md(
202        name: PlSmallStr,
203        mut chunks: Vec<ArrayRef>,
204        dtype: &ArrowDataType,
205        md: Option<&Metadata>,
206    ) -> PolarsResult<Self> {
207        match dtype {
208            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
209            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
210                let chunks =
211                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
212                Ok(StringChunked::from_chunks(name, chunks).into_series())
213            },
214            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
215            ArrowDataType::LargeBinary => {
216                if let Some(md) = md {
217                    if md.maintain_type() {
218                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
219                    }
220                }
221                let chunks =
222                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
223                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
224            },
225            ArrowDataType::Binary => {
226                let chunks =
227                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
228                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
229            },
230            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
231                let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
232                unsafe {
233                    Ok(
234                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
235                            .into_series(),
236                    )
237                }
238            },
239            #[cfg(feature = "dtype-array")]
240            ArrowDataType::FixedSizeList(_, _) => {
241                let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
242                unsafe {
243                    Ok(
244                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
245                            .into_series(),
246                    )
247                }
248            },
249            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
250            #[cfg(feature = "dtype-u8")]
251            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
252            #[cfg(feature = "dtype-u16")]
253            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
254            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
255            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
256            ArrowDataType::UInt128 => feature_gated!(
257                "dtype-u128",
258                Ok(UInt128Chunked::from_chunks(name, chunks).into_series())
259            ),
260            #[cfg(feature = "dtype-i8")]
261            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
262            #[cfg(feature = "dtype-i16")]
263            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
264            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
265            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
266            ArrowDataType::Int128 => feature_gated!(
267                "dtype-i128",
268                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
269            ),
270            #[cfg(feature = "dtype-f16")]
271            ArrowDataType::Float16 => {
272                let chunks =
273                    cast_chunks(&chunks, &DataType::Float16, CastOptions::NonStrict).unwrap();
274                Ok(Float16Chunked::from_chunks(name, chunks).into_series())
275            },
276            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
277            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
278            #[cfg(feature = "dtype-date")]
279            ArrowDataType::Date32 => {
280                let chunks =
281                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
282                Ok(Int32Chunked::from_chunks(name, chunks)
283                    .into_date()
284                    .into_series())
285            },
286            #[cfg(feature = "dtype-datetime")]
287            ArrowDataType::Date64 => {
288                let chunks =
289                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
290                let ca = Int64Chunked::from_chunks(name, chunks);
291                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
292            },
293            #[cfg(feature = "dtype-datetime")]
294            ArrowDataType::Timestamp(tu, tz) => {
295                let tz = TimeZone::opt_try_new(tz.clone())?;
296                let chunks =
297                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
298                let s = Int64Chunked::from_chunks(name, chunks)
299                    .into_datetime(tu.into(), tz)
300                    .into_series();
301                Ok(scale_arrow_values(s, dtype))
302            },
303            #[cfg(feature = "dtype-duration")]
304            ArrowDataType::Duration(tu) => {
305                let chunks =
306                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
307                let s = Int64Chunked::from_chunks(name, chunks)
308                    .into_duration(tu.into())
309                    .into_series();
310                Ok(scale_arrow_values(s, dtype))
311            },
312            #[cfg(feature = "dtype-time")]
313            ArrowDataType::Time64(_) | ArrowDataType::Time32(_) => {
314                let mut chunks = chunks;
315                if matches!(dtype, ArrowDataType::Time32(_)) {
316                    chunks =
317                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
318                }
319                let chunks =
320                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
321                let s = Int64Chunked::from_chunks(name, chunks)
322                    .into_time()
323                    .into_series();
324                Ok(scale_arrow_values(s, dtype))
325            },
326            ArrowDataType::Decimal32(precision, scale) => {
327                feature_gated!("dtype-decimal", {
328                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
329
330                    let mut chunks = chunks;
331                    for chunk in chunks.iter_mut() {
332                        let old_chunk = chunk
333                            .as_any_mut()
334                            .downcast_mut::<PrimitiveArray<i32>>()
335                            .unwrap();
336
337                        // For now, we just cast the whole data to i128.
338                        let (_, values, validity) = std::mem::take(old_chunk).into_inner();
339                        *chunk = PrimitiveArray::new(
340                            ArrowDataType::Int128,
341                            values.iter().map(|&v| v as i128).collect(),
342                            validity,
343                        )
344                        .to_boxed();
345                    }
346
347                    let s = Int128Chunked::from_chunks(name, chunks)
348                        .into_decimal_unchecked(*precision, *scale)
349                        .into_series();
350                    Ok(s)
351                })
352            },
353            ArrowDataType::Decimal64(precision, scale) => {
354                feature_gated!("dtype-decimal", {
355                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
356
357                    let mut chunks = chunks;
358                    for chunk in chunks.iter_mut() {
359                        let old_chunk = chunk
360                            .as_any_mut()
361                            .downcast_mut::<PrimitiveArray<i64>>()
362                            .unwrap();
363
364                        // For now, we just cast the whole data to i128.
365                        let (_, values, validity) = std::mem::take(old_chunk).into_inner();
366                        *chunk = PrimitiveArray::new(
367                            ArrowDataType::Int128,
368                            values.iter().map(|&v| v as i128).collect(),
369                            validity,
370                        )
371                        .to_boxed();
372                    }
373
374                    let s = Int128Chunked::from_chunks(name, chunks)
375                        .into_decimal_unchecked(*precision, *scale)
376                        .into_series();
377                    Ok(s)
378                })
379            },
380            ArrowDataType::Decimal(precision, scale) => {
381                feature_gated!("dtype-decimal", {
382                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
383
384                    let mut chunks = chunks;
385                    for chunk in chunks.iter_mut() {
386                        *chunk = std::mem::take(
387                            chunk
388                                .as_any_mut()
389                                .downcast_mut::<PrimitiveArray<i128>>()
390                                .unwrap(),
391                        )
392                        .to(ArrowDataType::Int128)
393                        .to_boxed();
394                    }
395
396                    let s = Int128Chunked::from_chunks(name, chunks)
397                        .into_decimal_unchecked(*precision, *scale)
398                        .into_series();
399                    Ok(s)
400                })
401            },
402            ArrowDataType::Decimal256(precision, scale) => {
403                feature_gated!("dtype-decimal", {
404                    use polars_arrow::types::i256;
405
406                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
407
408                    let mut chunks = chunks;
409                    for chunk in chunks.iter_mut() {
410                        let arr = std::mem::take(
411                            chunk
412                                .as_any_mut()
413                                .downcast_mut::<PrimitiveArray<i256>>()
414                                .unwrap(),
415                        );
416                        let arr_128: PrimitiveArray<i128> = arr.iter().map(|opt_v| {
417                            if let Some(v) = opt_v {
418                                let smaller: Option<i128> = (*v).try_into().ok();
419                                let smaller = smaller.filter(|v| dec128_fits(*v, *precision));
420                                smaller.ok_or_else(|| {
421                                    polars_err!(ComputeError: "Decimal256 to Decimal128 conversion overflowed, Decimal256 is not (yet) supported in Polars")
422                                }).map(Some)
423                            } else {
424                                Ok(None)
425                            }
426                        }).try_collect_arr_trusted()?;
427
428                        *chunk = arr_128.to(ArrowDataType::Int128).to_boxed();
429                    }
430
431                    let s = Int128Chunked::from_chunks(name, chunks)
432                        .into_decimal_unchecked(*precision, *scale)
433                        .into_series();
434                    Ok(s)
435                })
436            },
437            ArrowDataType::Null => Ok(new_null(name, &chunks)),
438            #[cfg(not(feature = "dtype-categorical"))]
439            ArrowDataType::Dictionary(_, _, _) => {
440                panic!("activate dtype-categorical to convert dictionary arrays")
441            },
442            #[cfg(feature = "dtype-categorical")]
443            ArrowDataType::Dictionary(key_type, _, _) => {
444                let polars_dtype = DataType::from_arrow(chunks[0].dtype(), md);
445
446                let mut series_iter = chunks.into_iter().map(|arr| {
447                    import_arrow_dictionary_array(name.clone(), arr, key_type, &polars_dtype)
448                });
449
450                let mut first = series_iter.next().unwrap()?;
451
452                for s in series_iter {
453                    first.append_owned(s?)?;
454                }
455
456                Ok(first)
457            },
458            #[cfg(feature = "object")]
459            ArrowDataType::Extension(ext)
460                if ext.name == POLARS_OBJECT_EXTENSION_NAME && ext.metadata.is_some() =>
461            {
462                assert_eq!(chunks.len(), 1);
463                let arr = chunks[0]
464                    .as_any()
465                    .downcast_ref::<FixedSizeBinaryArray>()
466                    .unwrap();
467                // SAFETY:
468                // this is highly unsafe. it will dereference a raw ptr on the heap
469                // make sure the ptr is allocated and from this pid
470                // (the pid is checked before dereference)
471                let s = {
472                    let pe = PolarsExtension::new(arr.clone());
473                    let s = pe.get_series(&name);
474                    pe.take_and_forget();
475                    s
476                };
477                Ok(s)
478            },
479            #[cfg(feature = "dtype-extension")]
480            ArrowDataType::Extension(ext) => {
481                use crate::datatypes::extension::get_extension_type_or_storage;
482
483                for chunk in &mut chunks {
484                    debug_assert!(
485                        chunk.dtype() == dtype,
486                        "expected chunk dtype to be {:?}, got {:?}",
487                        dtype,
488                        chunk.dtype()
489                    );
490                    *chunk.dtype_mut() = ext.inner.clone();
491                }
492                let storage = Series::_try_from_arrow_unchecked_with_md(
493                    name.clone(),
494                    chunks,
495                    &ext.inner,
496                    md,
497                )?;
498
499                Ok(
500                    match get_extension_type_or_storage(
501                        &ext.name,
502                        storage.dtype(),
503                        ext.metadata.as_deref(),
504                    ) {
505                        Some(typ) => ExtensionChunked::from_storage(typ, storage).into_series(),
506                        None => storage,
507                    },
508                )
509            },
510
511            #[cfg(feature = "dtype-struct")]
512            ArrowDataType::Struct(fields) => {
513                let names = fields.iter().map(|field| field.name.as_str()).collect_vec();
514                crate::frame::validation::ensure_names_unique(&names)?;
515
516                let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
517
518                unsafe {
519                    let mut ca =
520                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
521                    StructChunked::propagate_nulls_mut(&mut ca);
522                    Ok(ca.into_series())
523                }
524            },
525            ArrowDataType::FixedSizeBinary(_) => {
526                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
527                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
528            },
529            ArrowDataType::Map(field, _keys_sorted) => {
530                let struct_arrays = chunks
531                    .iter()
532                    .map(|arr| {
533                        let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
534                        arr.field().clone()
535                    })
536                    .collect::<Vec<_>>();
537
538                let (phys_struct_arrays, entries_dtype) =
539                    to_physical_and_dtype(struct_arrays, field.metadata.as_deref())?;
540
541                #[cfg(feature = "dtype-map")]
542                let map_dtype = entries_dtype.map_from_positional_entries_dtype();
543                #[cfg(not(feature = "dtype-map"))]
544                let map_dtype: Option<DataType> = None;
545
546                #[cfg(feature = "dtype-map")]
547                let phys_struct_arrays: Vec<ArrayRef> = if map_dtype.is_some() {
548                    phys_struct_arrays
549                        .into_iter()
550                        .map(|mut entries| {
551                            rename_map_entries(&mut entries);
552                            entries
553                        })
554                        .collect()
555                } else {
556                    phys_struct_arrays
557                };
558
559                let storage_dtype = match &map_dtype {
560                    #[cfg(feature = "dtype-map")]
561                    Some(dtype) => dtype.map_storage_dtype().unwrap(),
562                    _ => DataType::List(Box::new(entries_dtype)),
563                };
564
565                let chunks = chunks
566                    .iter()
567                    .zip(phys_struct_arrays)
568                    .map(|(arr, values)| {
569                        let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
570                        let offsets: &OffsetsBuffer<i32> = arr.offsets();
571
572                        let validity = arr.validity().cloned();
573
574                        Box::from(ListArray::<i64>::new(
575                            ListArray::<i64>::default_datatype(values.dtype().clone()),
576                            OffsetsBuffer::<i64>::from(offsets),
577                            values,
578                            validity,
579                        )) as ArrayRef
580                    })
581                    .collect();
582
583                let storage = unsafe {
584                    ListChunked::from_chunks_and_dtype_unchecked(name, chunks, storage_dtype)
585                }
586                .into_series();
587
588                match map_dtype {
589                    #[cfg(feature = "dtype-map")]
590                    Some(dtype) => {
591                        use crate::chunked_array::logical::ensure_live_entries_non_null;
592
593                        // Entries a null row spans are left where the producer put them, so
594                        // the import stays zero-copy. Trust the producer's key uniqueness.
595                        ensure_live_entries_non_null(storage.list().unwrap())?;
596                        // SAFETY: dtype and imported children are valid; live rows own no
597                        // null entries or keys.
598                        Ok(
599                            unsafe { MapChunked::from_storage_unchecked(dtype, storage) }
600                                .into_series(),
601                        )
602                    },
603                    _ => Ok(storage),
604                }
605            },
606            ArrowDataType::Interval(IntervalUnit::MonthDayNano) => {
607                check_allow_importing_interval_as_struct("month_day_nano_interval")?;
608
609                feature_gated!("dtype-struct", {
610                    let chunks = chunks
611                        .into_iter()
612                        .map(convert_month_day_nano_to_struct)
613                        .collect::<PolarsResult<Vec<_>>>()?;
614
615                    Ok(StructChunked::from_chunks_and_dtype_unchecked(
616                        name,
617                        chunks,
618                        DataType::_month_days_ns_struct_type(),
619                    )
620                    .into_series())
621                })
622            },
623
624            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
625        }
626    }
627
628    #[cfg(feature = "dtype-categorical")]
629    pub fn from_cats_and_dtype(
630        cats: &Series,
631        dtype: &DataType,
632        strict: bool,
633    ) -> PolarsResult<Series> {
634        use std::borrow::Cow;
635
636        let phys = dtype.cat_physical()?;
637        let phys_dtype = DataType::from(phys);
638
639        let mut casted = Cow::Borrowed(cats);
640        if cats.dtype() != &phys_dtype {
641            casted = Cow::Owned(cats.cast(&phys_dtype)?);
642        }
643
644        let out = with_match_categorical_physical_type!(phys, |$C| {
645            // SAFETY: we are guarded by the type system.
646            type PhysCa = ChunkedArray<<$C as PolarsCategoricalType>::PolarsPhysical>;
647            let ca: &PhysCa = casted.as_ref().as_ref().as_ref();
648            CategoricalChunked::<$C>::from_cats_and_dtype(ca.clone(), dtype.clone()).into_series()
649        });
650
651        if strict && out.null_count() != casted.null_count() {
652            polars_bail!(
653                ComputeError:
654                "found invalid category value when converting from physical to {dtype}",
655            );
656        }
657
658        Ok(out)
659    }
660}
661
662/// Brings the values of an imported temporal `s` to the unit polars stores its
663/// type in, see [`DataType::arrow_value_scale`].
664#[cfg(any(
665    feature = "dtype-datetime",
666    feature = "dtype-duration",
667    feature = "dtype-time"
668))]
669fn scale_arrow_values(s: Series, dtype: &ArrowDataType) -> Series {
670    match DataType::arrow_value_scale(dtype) {
671        1 => s,
672        factor => &s * factor,
673    }
674}
675
676fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
677    arr.iter().map(|arr| f(&**arr)).collect()
678}
679
680/// Normalize the field names of one chunk of map entries. Only the names change, so the
681/// buffers stay valid.
682#[cfg(feature = "dtype-map")]
683fn rename_map_entries(entries: &mut ArrayRef) {
684    let ArrowDataType::Struct(fields) = entries.dtype_mut() else {
685        unreachable!("map entries are a struct")
686    };
687    let [key, value] = fields.as_mut_slice() else {
688        unreachable!("map entries have two fields")
689    };
690    key.name = MAP_KEY_NAME;
691    value.name = MAP_VALUE_NAME;
692}
693
694/// Converts to physical types and bubbles up the correct [`DataType`].
695///
696/// Errors propagate from the nested logical imports, e.g. a malformed `Map` child.
697#[allow(clippy::only_used_in_recursion)]
698unsafe fn to_physical_and_dtype(
699    arrays: Vec<ArrayRef>,
700    md: Option<&Metadata>,
701) -> PolarsResult<(Vec<ArrayRef>, DataType)> {
702    match arrays[0].dtype() {
703        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
704            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
705            Ok((chunks, DataType::String))
706        },
707        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
708            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
709            Ok((chunks, DataType::Binary))
710        },
711        #[allow(unused_variables)]
712        dt @ ArrowDataType::Dictionary(_, _, _) => {
713            feature_gated!("dtype-categorical", {
714                let s = unsafe {
715                    let dt = dt.clone();
716                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
717                }?;
718                Ok((s.chunks().clone(), s.dtype().clone()))
719            })
720        },
721        dt @ ArrowDataType::Extension(_) => {
722            feature_gated!("dtype-extension", {
723                let s = unsafe {
724                    let dt = dt.clone();
725                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
726                }?;
727                Ok((s.chunks().clone(), s.dtype().clone()))
728            })
729        },
730        ArrowDataType::List(field) => {
731            let out = convert(&arrays, |arr| {
732                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
733            });
734            to_physical_and_dtype(out, md)
735        },
736        #[cfg(feature = "dtype-array")]
737        ArrowDataType::FixedSizeList(field, size) => {
738            let values = arrays
739                .iter()
740                .map(|arr| {
741                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
742                    arr.values().clone()
743                })
744                .collect::<Vec<_>>();
745
746            let (converted_values, dtype) =
747                to_physical_and_dtype(values, field.metadata.as_deref())?;
748
749            let arrays = arrays
750                .iter()
751                .zip(converted_values)
752                .map(|(arr, values)| {
753                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
754
755                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
756                    Box::from(FixedSizeListArray::new(
757                        dtype,
758                        arr.len(),
759                        values,
760                        arr.validity().cloned(),
761                    )) as ArrayRef
762                })
763                .collect();
764            Ok((arrays, DataType::Array(Box::new(dtype), *size)))
765        },
766        ArrowDataType::LargeList(field) => {
767            let values = arrays
768                .iter()
769                .map(|arr| {
770                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
771                    arr.values().clone()
772                })
773                .collect::<Vec<_>>();
774
775            let (converted_values, dtype) =
776                to_physical_and_dtype(values, field.metadata.as_deref())?;
777
778            let arrays = arrays
779                .iter()
780                .zip(converted_values)
781                .map(|(arr, values)| {
782                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
783
784                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
785                    Box::from(ListArray::<i64>::new(
786                        dtype,
787                        arr.offsets().clone(),
788                        values,
789                        arr.validity().cloned(),
790                    )) as ArrayRef
791                })
792                .collect();
793            Ok((arrays, DataType::List(Box::new(dtype))))
794        },
795        ArrowDataType::Struct(_fields) => {
796            feature_gated!("dtype-struct", {
797                let mut pl_fields = None;
798                let mut out_arrays = Vec::with_capacity(arrays.len());
799                for arr in &arrays {
800                    let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
801                    let mut values = Vec::with_capacity(_fields.len());
802                    let mut dtypes = Vec::with_capacity(_fields.len());
803                    for (value, field) in arr.values().iter().zip(_fields.iter()) {
804                        let (mut value, dtype) =
805                            to_physical_and_dtype(vec![value.clone()], field.metadata.as_deref())?;
806                        values.push(value.pop().unwrap());
807                        dtypes.push(dtype);
808                    }
809
810                    let arrow_fields = values
811                        .iter()
812                        .zip(_fields.iter())
813                        .map(|(arr, field)| {
814                            ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
815                        })
816                        .collect();
817                    out_arrays.push(Box::new(StructArray::new(
818                        ArrowDataType::Struct(arrow_fields),
819                        arr.len(),
820                        values,
821                        arr.validity().cloned(),
822                    )) as ArrayRef);
823
824                    if pl_fields.is_none() {
825                        pl_fields = Some(
826                            _fields
827                                .iter()
828                                .zip(dtypes)
829                                .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
830                                .collect_vec(),
831                        )
832                    }
833                }
834
835                Ok((out_arrays, DataType::Struct(pl_fields.unwrap())))
836            })
837        },
838        // Use Series architecture to convert nested logical types to physical.
839        dt @ (ArrowDataType::Duration(_)
840        | ArrowDataType::Time32(_)
841        | ArrowDataType::Time64(_)
842        | ArrowDataType::Timestamp(_, _)
843        | ArrowDataType::Date32
844        | ArrowDataType::Decimal(_, _)
845        | ArrowDataType::Date64
846        | ArrowDataType::Map(_, _)) => {
847            let dt = dt.clone();
848            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt)?;
849            let dtype = s.dtype().clone();
850            Ok((std::mem::take(s.chunks_mut()), dtype))
851        },
852        dt => {
853            let dtype = DataType::from_arrow(dt, md);
854            Ok((arrays, dtype))
855        },
856    }
857}
858
859#[cfg(feature = "dtype-categorical")]
860unsafe fn import_arrow_dictionary_array(
861    name: PlSmallStr,
862    arr: Box<dyn Array>,
863    key_type: &polars_arrow::datatypes::IntegerType,
864    polars_dtype: &DataType,
865) -> PolarsResult<Series> {
866    use polars_arrow::datatypes::IntegerType as I;
867
868    if matches!(
869        polars_dtype,
870        DataType::Categorical(_, _) | DataType::Enum(_, _)
871    ) {
872        macro_rules! unpack_categorical_chunked {
873            ($dt:ty) => {{
874                let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
875                let keys = arr.keys();
876                let values = arr.values();
877                let values = cast(&**values, &ArrowDataType::Utf8View)?;
878                let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
879                with_match_categorical_physical_type!(polars_dtype.cat_physical().unwrap(), |$C| {
880                    let ca = CategoricalChunked::<$C>::from_str_iter(
881                        name,
882                        polars_dtype.clone(),
883                        keys.iter().map(|k| {
884                            let k: usize = (*k?).try_into().ok()?;
885                            values.get(k)
886                        }),
887                    )?;
888                    Ok(ca.into_series())
889                })
890            }};
891        }
892
893        match key_type {
894            I::Int8 => unpack_categorical_chunked!(i8),
895            I::UInt8 => unpack_categorical_chunked!(u8),
896            I::Int16 => unpack_categorical_chunked!(i16),
897            I::UInt16 => unpack_categorical_chunked!(u16),
898            I::Int32 => unpack_categorical_chunked!(i32),
899            I::UInt32 => unpack_categorical_chunked!(u32),
900            I::Int64 => unpack_categorical_chunked!(i64),
901            I::UInt64 => unpack_categorical_chunked!(u64),
902            _ => polars_bail!(
903                ComputeError: "unsupported arrow key type: {key_type:?}"
904            ),
905        }
906    } else {
907        macro_rules! unpack_keys_values {
908            ($dt:ty) => {{
909                let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
910                let keys = arr.keys();
911                let keys = polars_compute::cast::primitive_to_primitive::<
912                    $dt,
913                    <IdxType as PolarsNumericType>::Native,
914                >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
915                (keys, arr.values())
916            }};
917        }
918
919        let (keys, values) = match key_type {
920            I::Int8 => unpack_keys_values!(i8),
921            I::UInt8 => unpack_keys_values!(u8),
922            I::Int16 => unpack_keys_values!(i16),
923            I::UInt16 => unpack_keys_values!(u16),
924            I::Int32 => unpack_keys_values!(i32),
925            I::UInt32 => unpack_keys_values!(u32),
926            I::Int64 => unpack_keys_values!(i64),
927            I::UInt64 => unpack_keys_values!(u64),
928            _ => polars_bail!(
929                ComputeError: "unsupported arrow key type: {key_type:?}"
930            ),
931        };
932
933        let values = Series::_try_from_arrow_unchecked_with_md(
934            name,
935            vec![values.clone()],
936            values.dtype(),
937            None,
938        )?;
939
940        values.take(&IdxCa::from_chunks_and_dtype(
941            PlSmallStr::EMPTY,
942            vec![keys.to_boxed()],
943            IDX_DTYPE,
944        ))
945    }
946}
947
948#[cfg(feature = "dtype-struct")]
949fn convert_month_day_nano_to_struct(chunk: Box<dyn Array>) -> PolarsResult<Box<dyn Array>> {
950    let arr: &PrimitiveArray<months_days_ns> = chunk.as_any().downcast_ref().unwrap();
951
952    let values: &[months_days_ns] = arr.values();
953
954    let (months_out, days_out, nanoseconds_out): (Vec<i32>, Vec<i32>, Vec<i64>) = values
955        .iter()
956        .map(|x| (x.months(), x.days(), x.ns()))
957        .collect();
958
959    let out = StructArray::new(
960        DataType::_month_days_ns_struct_type()
961            .to_physical()
962            .to_arrow(CompatLevel::newest()),
963        arr.len(),
964        vec![
965            PrimitiveArray::<i32>::from_vec(months_out).boxed(),
966            PrimitiveArray::<i32>::from_vec(days_out).boxed(),
967            PrimitiveArray::<i64>::from_vec(nanoseconds_out).boxed(),
968        ],
969        arr.validity().cloned(),
970    );
971
972    Ok(out.boxed())
973}
974
975fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
976    let mut chunks_iter = chunks.iter();
977    let dtype: ArrowDataType = chunks_iter
978        .next()
979        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
980        .dtype()
981        .clone();
982
983    for chunk in chunks_iter {
984        if chunk.dtype() != &dtype {
985            polars_bail!(
986                ComputeError: "cannot create series from multiple arrays with different types"
987            );
988        }
989    }
990    Ok(dtype)
991}
992
993impl Series {
994    pub fn try_new<T>(
995        name: PlSmallStr,
996        data: T,
997    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
998    where
999        (PlSmallStr, T): TryInto<Self>,
1000    {
1001        // # TODO
1002        // * Remove the TryFrom<tuple> impls in favor of this
1003        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
1004    }
1005}
1006
1007impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
1008    type Error = PolarsError;
1009
1010    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
1011        let (name, chunks) = name_arr;
1012
1013        let dtype = check_types(&chunks)?;
1014        // SAFETY:
1015        // dtype is checked
1016        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
1017    }
1018}
1019
1020impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
1021    type Error = PolarsError;
1022
1023    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
1024        let (name, arr) = name_arr;
1025        Series::try_from((name, vec![arr]))
1026    }
1027}
1028
1029impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
1030    type Error = PolarsError;
1031
1032    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
1033        let (field, chunks) = field_arr;
1034        let arrow_dt = field.dtype();
1035        let dtype = check_types(&chunks)?;
1036        let compatible = match (&dtype, arrow_dt) {
1037            // See #26174, we don't care about dictionary ordering.
1038            (
1039                ArrowDataType::Dictionary(int0, inner0, _ord0),
1040                ArrowDataType::Dictionary(int1, inner1, _ord1),
1041            ) => (int0, inner0) == (int1, inner1),
1042            (l, r) => l == r,
1043        };
1044        polars_ensure!(compatible, ComputeError: "Arrow Field dtype does not match the ArrayRef dtypes");
1045
1046        // SAFETY:
1047        // dtype is checked
1048        unsafe {
1049            Series::_try_from_arrow_unchecked_with_md(
1050                field.name.clone(),
1051                chunks,
1052                &dtype,
1053                field.metadata.as_deref(),
1054            )
1055        }
1056    }
1057}
1058
1059impl TryFrom<(&ArrowField, ArrayRef)> for Series {
1060    type Error = PolarsError;
1061
1062    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
1063        let (field, arr) = field_arr;
1064        Series::try_from((field, vec![arr]))
1065    }
1066}
1067
1068/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
1069/// into a [`Series`].
1070/// # Safety
1071///
1072/// This trait is marked `unsafe` as the `is_series` return is used
1073/// to transmute to `Series`. This must always return `false` except
1074/// for `Series` structs.
1075pub unsafe trait IntoSeries {
1076    fn is_series() -> bool {
1077        false
1078    }
1079
1080    fn into_series(self) -> Series
1081    where
1082        Self: Sized;
1083}
1084
1085impl<T> From<ChunkedArray<T>> for Series
1086where
1087    T: PolarsDataType,
1088    ChunkedArray<T>: IntoSeries,
1089{
1090    fn from(ca: ChunkedArray<T>) -> Self {
1091        ca.into_series()
1092    }
1093}
1094
1095#[cfg(feature = "dtype-date")]
1096impl From<DateChunked> for Series {
1097    fn from(a: DateChunked) -> Self {
1098        a.into_series()
1099    }
1100}
1101
1102#[cfg(feature = "dtype-datetime")]
1103impl From<DatetimeChunked> for Series {
1104    fn from(a: DatetimeChunked) -> Self {
1105        a.into_series()
1106    }
1107}
1108
1109#[cfg(feature = "dtype-duration")]
1110impl From<DurationChunked> for Series {
1111    fn from(a: DurationChunked) -> Self {
1112        a.into_series()
1113    }
1114}
1115
1116#[cfg(feature = "dtype-time")]
1117impl From<TimeChunked> for Series {
1118    fn from(a: TimeChunked) -> Self {
1119        a.into_series()
1120    }
1121}
1122
1123unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
1124    fn into_series(self) -> Series {
1125        Series(self)
1126    }
1127}
1128
1129unsafe impl IntoSeries for Series {
1130    fn is_series() -> bool {
1131        true
1132    }
1133
1134    #[inline]
1135    fn into_series(self) -> Series {
1136        self
1137    }
1138}
1139
1140fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
1141    let len = chunks.iter().map(|arr| arr.len()).sum();
1142    Series::new_null(name, len)
1143}
1144
1145#[cfg(test)]
1146#[cfg(all(
1147    feature = "dtype-datetime",
1148    feature = "dtype-duration",
1149    feature = "dtype-time"
1150))]
1151mod tests {
1152    use polars_arrow::array::PrimitiveArray;
1153
1154    use super::*;
1155
1156    #[test]
1157    fn imported_values_are_scaled_like_the_dtype_says() {
1158        let import = |dtype: ArrowDataType, value: i64| {
1159            let array = PrimitiveArray::from_vec(vec![value]).to(dtype.clone());
1160            let s = Series::from_arrow(PlSmallStr::EMPTY, array.boxed()).unwrap();
1161            let physical = s.to_physical_repr().i64().unwrap().get(0).unwrap();
1162            assert_eq!(physical, value * DataType::arrow_value_scale(&dtype));
1163            (s.dtype().clone(), physical)
1164        };
1165        let seconds = ArrowDataType::Timestamp(ArrowTimeUnit::Second, None);
1166        assert_eq!(
1167            import(seconds, -7),
1168            (DataType::Datetime(TimeUnit::Milliseconds, None), -7_000)
1169        );
1170        assert_eq!(
1171            import(ArrowDataType::Duration(ArrowTimeUnit::Second), 7),
1172            (DataType::Duration(TimeUnit::Milliseconds), 7_000)
1173        );
1174        assert_eq!(
1175            import(ArrowDataType::Duration(ArrowTimeUnit::Microsecond), 7),
1176            (DataType::Duration(TimeUnit::Microseconds), 7)
1177        );
1178        assert_eq!(
1179            import(ArrowDataType::Time64(ArrowTimeUnit::Microsecond), 7),
1180            (DataType::Time, 7_000)
1181        );
1182        assert_eq!(
1183            import(ArrowDataType::Date64, 86_400_000),
1184            (DataType::Datetime(TimeUnit::Milliseconds, None), 86_400_000)
1185        );
1186        let array = PrimitiveArray::from_vec(vec![7i32])
1187            .to(ArrowDataType::Time32(ArrowTimeUnit::Millisecond));
1188        let s = Series::from_arrow(PlSmallStr::EMPTY, array.boxed()).unwrap();
1189        assert_eq!(s.dtype(), &DataType::Time);
1190        assert_eq!(s.to_physical_repr().i64().unwrap().get(0), Some(7_000_000));
1191    }
1192}
polars_core/series/from.rs

polars_core/series/
from.rs
