polars_core/series/
from.rs

1use arrow::datatypes::{IntervalUnit, Metadata};
2use arrow::offset::OffsetsBuffer;
3#[cfg(any(
4    feature = "dtype-date",
5    feature = "dtype-datetime",
6    feature = "dtype-time",
7    feature = "dtype-duration"
8))]
9use arrow::temporal_conversions::*;
10use arrow::types::months_days_ns;
11use polars_compute::cast::cast_unchecked as cast;
12#[cfg(feature = "dtype-decimal")]
13use polars_compute::decimal::dec128_fits;
14use polars_error::feature_gated;
15use polars_utils::itertools::Itertools;
16
17use crate::chunked_array::cast::{CastOptions, cast_chunks};
18#[cfg(feature = "object")]
19use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
20#[cfg(feature = "object")]
21use crate::chunked_array::object::registry::get_object_builder;
22use crate::config::check_allow_importing_interval_as_struct;
23use crate::prelude::*;
24
25impl Series {
26    pub fn from_array<A: ParameterFreeDtypeStaticArray>(name: PlSmallStr, array: A) -> Self {
27        unsafe {
28            Self::from_chunks_and_dtype_unchecked(
29                name,
30                vec![Box::new(array)],
31                &DataType::from_arrow_dtype(&A::get_dtype()),
32            )
33        }
34    }
35
36    /// Construct a Series from a chunk holding the physical representation of `dtype`.
37    ///
38    /// Checks the physical dtype and validates values through [`Series::try_from_physical`].
39    pub fn from_chunk_and_dtype(
40        name: PlSmallStr,
41        chunk: ArrayRef,
42        dtype: &DataType,
43    ) -> PolarsResult<Self> {
44        // Reject objects before construction can reinterpret the chunk as pointers.
45        polars_ensure!(
46            !dtype.contains_objects(),
47            InvalidOperation: "cannot create a series of type '{dtype}' from an arrow chunk: objects are process-local"
48        );
49        polars_ensure!(
50            !dtype.contains_unknown(),
51            InvalidOperation: "cannot create a series of type '{dtype}' from an arrow chunk"
52        );
53        #[cfg(feature = "dtype-map")]
54        dtype.ensure_valid_map_dtypes()?;
55
56        let physical = dtype.to_physical();
57        if &physical.to_arrow(CompatLevel::newest()) != chunk.dtype() {
58            polars_bail!(
59                InvalidOperation: "cannot create a series of type '{dtype}' of arrow chunk with type '{:?}'",
60                chunk.dtype()
61            );
62        }
63
64        // SAFETY: the chunk matches the physical dtype, checked above.
65        let physical =
66            unsafe { Self::from_chunks_and_dtype_unchecked(name, vec![chunk], &physical) };
67        physical.try_from_physical(dtype)
68    }
69
70    /// Takes chunks and a polars datatype and constructs the Series.
71    /// This is faster than creating from chunks and an arrow datatype because there is no
72    /// casting involved.
73    ///
74    /// # Safety
75    ///
76    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
77    ///
78    /// Payloads must also be safe to read as the logical dtype:
79    ///
80    /// - `Categorical` / `Enum`: every non-null code names a category;
81    /// - `Object`: chunks originate from this process;
82    /// - `Map`: storage satisfies the `MapChunked` storage safety contract, so entries and
83    ///   keys are non-null within the offset windows of live rows. Keys may repeat.
84    pub unsafe fn from_chunks_and_dtype_unchecked(
85        name: PlSmallStr,
86        chunks: Vec<ArrayRef>,
87        dtype: &DataType,
88    ) -> Self {
89        use DataType::*;
90        match dtype {
91            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
92            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
93            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
94            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
95            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
96            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
97            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
98            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
99            #[cfg(feature = "dtype-i128")]
100            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
101            #[cfg(feature = "dtype-u128")]
102            UInt128 => UInt128Chunked::from_chunks(name, chunks).into_series(),
103            #[cfg(feature = "dtype-date")]
104            Date => Int32Chunked::from_chunks(name, chunks)
105                .into_date()
106                .into_series(),
107            #[cfg(feature = "dtype-time")]
108            Time => Int64Chunked::from_chunks(name, chunks)
109                .into_time()
110                .into_series(),
111            #[cfg(feature = "dtype-duration")]
112            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
113                .into_duration(*tu)
114                .into_series(),
115            #[cfg(feature = "dtype-datetime")]
116            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
117                .into_datetime(*tu, tz.clone())
118                .into_series(),
119            #[cfg(feature = "dtype-decimal")]
120            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
121                .into_decimal_unchecked(*precision, *scale)
122                .into_series(),
123            #[cfg(feature = "dtype-array")]
124            Array(_, _) => {
125                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
126                    .into_series()
127            },
128            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
129                .into_series(),
130            String => StringChunked::from_chunks(name, chunks).into_series(),
131            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
132            #[cfg(feature = "dtype-categorical")]
133            dt @ (Categorical(_, _) | Enum(_, _)) => {
134                with_match_categorical_physical_type!(dt.cat_physical().unwrap(), |$C| {
135                    let phys = ChunkedArray::from_chunks(name, chunks);
136                    CategoricalChunked::<$C>::from_cats_and_dtype_unchecked(phys, dt.clone()).into_series()
137                })
138            },
139            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
140            #[cfg(feature = "dtype-f16")]
141            Float16 => Float16Chunked::from_chunks(name, chunks).into_series(),
142            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
143            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
144            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
145            #[cfg(feature = "dtype-extension")]
146            Extension(typ, storage) => ExtensionChunked::from_storage(
147                typ.clone(),
148                Series::from_chunks_and_dtype_unchecked(name, chunks, storage),
149            )
150            .into_series(),
151            #[cfg(feature = "dtype-map")]
152            Map(_, _) => {
153                let storage = Series::from_chunks_and_dtype_unchecked(
154                    name,
155                    chunks,
156                    &dtype.map_storage_dtype().unwrap(),
157                );
158                MapChunked::from_storage_unchecked(dtype.clone(), storage).into_series()
159            },
160            #[cfg(feature = "dtype-struct")]
161            Struct(_) => {
162                let mut ca =
163                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
164                StructChunked::propagate_nulls_mut(&mut ca);
165                ca.into_series()
166            },
167            #[cfg(feature = "object")]
168            Object(_) => {
169                if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
170                    assert_eq!(chunks.len(), 1);
171                    // SAFETY:
172                    // this is highly unsafe. it will dereference a raw ptr on the heap
173                    // make sure the ptr is allocated and from this pid
174                    // (the pid is checked before dereference)
175                    {
176                        let pe = PolarsExtension::new(arr.clone());
177                        let s = pe.get_series(&name);
178                        pe.take_and_forget();
179                        s
180                    }
181                } else {
182                    unsafe { get_object_builder(name, 0).from_chunks(chunks) }
183                }
184            },
185            Null => new_null(name, &chunks),
186            Unknown(_) => {
187                panic!("dtype is unknown; consider supplying data-types for all operations")
188            },
189            #[allow(unreachable_patterns)]
190            _ => unreachable!(),
191        }
192    }
193
194    /// # Safety
195    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
196    pub unsafe fn _try_from_arrow_unchecked(
197        name: PlSmallStr,
198        chunks: Vec<ArrayRef>,
199        dtype: &ArrowDataType,
200    ) -> PolarsResult<Self> {
201        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
202    }
203
204    /// Create a new Series without checking if the inner dtype of the chunks is correct
205    ///
206    /// # Safety
207    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
208    pub unsafe fn _try_from_arrow_unchecked_with_md(
209        name: PlSmallStr,
210        mut chunks: Vec<ArrayRef>,
211        dtype: &ArrowDataType,
212        md: Option<&Metadata>,
213    ) -> PolarsResult<Self> {
214        match dtype {
215            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
216            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
217                let chunks =
218                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
219                Ok(StringChunked::from_chunks(name, chunks).into_series())
220            },
221            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
222            ArrowDataType::LargeBinary => {
223                if let Some(md) = md {
224                    if md.maintain_type() {
225                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
226                    }
227                }
228                let chunks =
229                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
230                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
231            },
232            ArrowDataType::Binary => {
233                let chunks =
234                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
235                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
236            },
237            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
238                let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
239                unsafe {
240                    Ok(
241                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
242                            .into_series(),
243                    )
244                }
245            },
246            #[cfg(feature = "dtype-array")]
247            ArrowDataType::FixedSizeList(_, _) => {
248                let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
249                unsafe {
250                    Ok(
251                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
252                            .into_series(),
253                    )
254                }
255            },
256            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
257            #[cfg(feature = "dtype-u8")]
258            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
259            #[cfg(feature = "dtype-u16")]
260            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
261            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
262            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
263            ArrowDataType::UInt128 => feature_gated!(
264                "dtype-u128",
265                Ok(UInt128Chunked::from_chunks(name, chunks).into_series())
266            ),
267            #[cfg(feature = "dtype-i8")]
268            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
269            #[cfg(feature = "dtype-i16")]
270            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
271            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
272            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
273            ArrowDataType::Int128 => feature_gated!(
274                "dtype-i128",
275                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
276            ),
277            #[cfg(feature = "dtype-f16")]
278            ArrowDataType::Float16 => {
279                let chunks =
280                    cast_chunks(&chunks, &DataType::Float16, CastOptions::NonStrict).unwrap();
281                Ok(Float16Chunked::from_chunks(name, chunks).into_series())
282            },
283            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
284            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
285            #[cfg(feature = "dtype-date")]
286            ArrowDataType::Date32 => {
287                let chunks =
288                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
289                Ok(Int32Chunked::from_chunks(name, chunks)
290                    .into_date()
291                    .into_series())
292            },
293            #[cfg(feature = "dtype-datetime")]
294            ArrowDataType::Date64 => {
295                let chunks =
296                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
297                let ca = Int64Chunked::from_chunks(name, chunks);
298                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
299            },
300            #[cfg(feature = "dtype-datetime")]
301            ArrowDataType::Timestamp(tu, tz) => {
302                let tz = TimeZone::opt_try_new(tz.clone())?;
303                let chunks =
304                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
305                let s = Int64Chunked::from_chunks(name, chunks)
306                    .into_datetime(tu.into(), tz)
307                    .into_series();
308                Ok(match tu {
309                    ArrowTimeUnit::Second => &s * MILLISECONDS,
310                    ArrowTimeUnit::Millisecond => s,
311                    ArrowTimeUnit::Microsecond => s,
312                    ArrowTimeUnit::Nanosecond => s,
313                })
314            },
315            #[cfg(feature = "dtype-duration")]
316            ArrowDataType::Duration(tu) => {
317                let chunks =
318                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
319                let s = Int64Chunked::from_chunks(name, chunks)
320                    .into_duration(tu.into())
321                    .into_series();
322                Ok(match tu {
323                    ArrowTimeUnit::Second => &s * MILLISECONDS,
324                    ArrowTimeUnit::Millisecond => s,
325                    ArrowTimeUnit::Microsecond => s,
326                    ArrowTimeUnit::Nanosecond => s,
327                })
328            },
329            #[cfg(feature = "dtype-time")]
330            ArrowDataType::Time64(tu) | ArrowDataType::Time32(tu) => {
331                let mut chunks = chunks;
332                if matches!(dtype, ArrowDataType::Time32(_)) {
333                    chunks =
334                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
335                }
336                let chunks =
337                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
338                let s = Int64Chunked::from_chunks(name, chunks)
339                    .into_time()
340                    .into_series();
341                Ok(match tu {
342                    ArrowTimeUnit::Second => &s * NANOSECONDS,
343                    ArrowTimeUnit::Millisecond => &s * 1_000_000,
344                    ArrowTimeUnit::Microsecond => &s * 1_000,
345                    ArrowTimeUnit::Nanosecond => s,
346                })
347            },
348            ArrowDataType::Decimal32(precision, scale) => {
349                feature_gated!("dtype-decimal", {
350                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
351
352                    let mut chunks = chunks;
353                    for chunk in chunks.iter_mut() {
354                        let old_chunk = chunk
355                            .as_any_mut()
356                            .downcast_mut::<PrimitiveArray<i32>>()
357                            .unwrap();
358
359                        // For now, we just cast the whole data to i128.
360                        let (_, values, validity) = std::mem::take(old_chunk).into_inner();
361                        *chunk = PrimitiveArray::new(
362                            ArrowDataType::Int128,
363                            values.iter().map(|&v| v as i128).collect(),
364                            validity,
365                        )
366                        .to_boxed();
367                    }
368
369                    let s = Int128Chunked::from_chunks(name, chunks)
370                        .into_decimal_unchecked(*precision, *scale)
371                        .into_series();
372                    Ok(s)
373                })
374            },
375            ArrowDataType::Decimal64(precision, scale) => {
376                feature_gated!("dtype-decimal", {
377                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
378
379                    let mut chunks = chunks;
380                    for chunk in chunks.iter_mut() {
381                        let old_chunk = chunk
382                            .as_any_mut()
383                            .downcast_mut::<PrimitiveArray<i64>>()
384                            .unwrap();
385
386                        // For now, we just cast the whole data to i128.
387                        let (_, values, validity) = std::mem::take(old_chunk).into_inner();
388                        *chunk = PrimitiveArray::new(
389                            ArrowDataType::Int128,
390                            values.iter().map(|&v| v as i128).collect(),
391                            validity,
392                        )
393                        .to_boxed();
394                    }
395
396                    let s = Int128Chunked::from_chunks(name, chunks)
397                        .into_decimal_unchecked(*precision, *scale)
398                        .into_series();
399                    Ok(s)
400                })
401            },
402            ArrowDataType::Decimal(precision, scale) => {
403                feature_gated!("dtype-decimal", {
404                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
405
406                    let mut chunks = chunks;
407                    for chunk in chunks.iter_mut() {
408                        *chunk = std::mem::take(
409                            chunk
410                                .as_any_mut()
411                                .downcast_mut::<PrimitiveArray<i128>>()
412                                .unwrap(),
413                        )
414                        .to(ArrowDataType::Int128)
415                        .to_boxed();
416                    }
417
418                    let s = Int128Chunked::from_chunks(name, chunks)
419                        .into_decimal_unchecked(*precision, *scale)
420                        .into_series();
421                    Ok(s)
422                })
423            },
424            ArrowDataType::Decimal256(precision, scale) => {
425                feature_gated!("dtype-decimal", {
426                    use arrow::types::i256;
427
428                    polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
429
430                    let mut chunks = chunks;
431                    for chunk in chunks.iter_mut() {
432                        let arr = std::mem::take(
433                            chunk
434                                .as_any_mut()
435                                .downcast_mut::<PrimitiveArray<i256>>()
436                                .unwrap(),
437                        );
438                        let arr_128: PrimitiveArray<i128> = arr.iter().map(|opt_v| {
439                            if let Some(v) = opt_v {
440                                let smaller: Option<i128> = (*v).try_into().ok();
441                                let smaller = smaller.filter(|v| dec128_fits(*v, *precision));
442                                smaller.ok_or_else(|| {
443                                    polars_err!(ComputeError: "Decimal256 to Decimal128 conversion overflowed, Decimal256 is not (yet) supported in Polars")
444                                }).map(Some)
445                            } else {
446                                Ok(None)
447                            }
448                        }).try_collect_arr_trusted()?;
449
450                        *chunk = arr_128.to(ArrowDataType::Int128).to_boxed();
451                    }
452
453                    let s = Int128Chunked::from_chunks(name, chunks)
454                        .into_decimal_unchecked(*precision, *scale)
455                        .into_series();
456                    Ok(s)
457                })
458            },
459            ArrowDataType::Null => Ok(new_null(name, &chunks)),
460            #[cfg(not(feature = "dtype-categorical"))]
461            ArrowDataType::Dictionary(_, _, _) => {
462                panic!("activate dtype-categorical to convert dictionary arrays")
463            },
464            #[cfg(feature = "dtype-categorical")]
465            ArrowDataType::Dictionary(key_type, _, _) => {
466                let polars_dtype = DataType::from_arrow(chunks[0].dtype(), md);
467
468                let mut series_iter = chunks.into_iter().map(|arr| {
469                    import_arrow_dictionary_array(name.clone(), arr, key_type, &polars_dtype)
470                });
471
472                let mut first = series_iter.next().unwrap()?;
473
474                for s in series_iter {
475                    first.append_owned(s?)?;
476                }
477
478                Ok(first)
479            },
480            #[cfg(feature = "object")]
481            ArrowDataType::Extension(ext)
482                if ext.name == POLARS_OBJECT_EXTENSION_NAME && ext.metadata.is_some() =>
483            {
484                assert_eq!(chunks.len(), 1);
485                let arr = chunks[0]
486                    .as_any()
487                    .downcast_ref::<FixedSizeBinaryArray>()
488                    .unwrap();
489                // SAFETY:
490                // this is highly unsafe. it will dereference a raw ptr on the heap
491                // make sure the ptr is allocated and from this pid
492                // (the pid is checked before dereference)
493                let s = {
494                    let pe = PolarsExtension::new(arr.clone());
495                    let s = pe.get_series(&name);
496                    pe.take_and_forget();
497                    s
498                };
499                Ok(s)
500            },
501            #[cfg(feature = "dtype-extension")]
502            ArrowDataType::Extension(ext) => {
503                use crate::datatypes::extension::get_extension_type_or_storage;
504
505                for chunk in &mut chunks {
506                    debug_assert!(
507                        chunk.dtype() == dtype,
508                        "expected chunk dtype to be {:?}, got {:?}",
509                        dtype,
510                        chunk.dtype()
511                    );
512                    *chunk.dtype_mut() = ext.inner.clone();
513                }
514                let storage = Series::_try_from_arrow_unchecked_with_md(
515                    name.clone(),
516                    chunks,
517                    &ext.inner,
518                    md,
519                )?;
520
521                Ok(
522                    match get_extension_type_or_storage(
523                        &ext.name,
524                        storage.dtype(),
525                        ext.metadata.as_deref(),
526                    ) {
527                        Some(typ) => ExtensionChunked::from_storage(typ, storage).into_series(),
528                        None => storage,
529                    },
530                )
531            },
532
533            #[cfg(feature = "dtype-struct")]
534            ArrowDataType::Struct(_) => {
535                let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
536
537                unsafe {
538                    let mut ca =
539                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
540                    StructChunked::propagate_nulls_mut(&mut ca);
541                    Ok(ca.into_series())
542                }
543            },
544            ArrowDataType::FixedSizeBinary(_) => {
545                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
546                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
547            },
548            ArrowDataType::Map(field, _keys_sorted) => {
549                let struct_arrays = chunks
550                    .iter()
551                    .map(|arr| {
552                        let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
553                        arr.field().clone()
554                    })
555                    .collect::<Vec<_>>();
556
557                let (phys_struct_arrays, entries_dtype) =
558                    to_physical_and_dtype(struct_arrays, field.metadata.as_deref())?;
559
560                #[cfg(feature = "dtype-map")]
561                let map_dtype = entries_dtype.map_from_positional_entries_dtype();
562                #[cfg(not(feature = "dtype-map"))]
563                let map_dtype: Option<DataType> = None;
564
565                #[cfg(feature = "dtype-map")]
566                let phys_struct_arrays: Vec<ArrayRef> = if map_dtype.is_some() {
567                    phys_struct_arrays
568                        .into_iter()
569                        .map(|mut entries| {
570                            rename_map_entries(&mut entries);
571                            entries
572                        })
573                        .collect()
574                } else {
575                    phys_struct_arrays
576                };
577
578                let storage_dtype = match &map_dtype {
579                    #[cfg(feature = "dtype-map")]
580                    Some(dtype) => dtype.map_storage_dtype().unwrap(),
581                    _ => DataType::List(Box::new(entries_dtype)),
582                };
583
584                let chunks = chunks
585                    .iter()
586                    .zip(phys_struct_arrays)
587                    .map(|(arr, values)| {
588                        let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
589                        let offsets: &OffsetsBuffer<i32> = arr.offsets();
590
591                        let validity = arr.validity().cloned();
592
593                        Box::from(ListArray::<i64>::new(
594                            ListArray::<i64>::default_datatype(values.dtype().clone()),
595                            OffsetsBuffer::<i64>::from(offsets),
596                            values,
597                            validity,
598                        )) as ArrayRef
599                    })
600                    .collect();
601
602                let storage = unsafe {
603                    ListChunked::from_chunks_and_dtype_unchecked(name, chunks, storage_dtype)
604                }
605                .into_series();
606
607                match map_dtype {
608                    #[cfg(feature = "dtype-map")]
609                    Some(dtype) => {
610                        use crate::chunked_array::logical::ensure_live_entries_non_null;
611
612                        // Entries a null row spans are left where the producer put them, so
613                        // the import stays zero-copy. Trust the producer's key uniqueness.
614                        ensure_live_entries_non_null(storage.list().unwrap())?;
615                        // SAFETY: dtype and imported children are valid; live rows own no
616                        // null entries or keys.
617                        Ok(
618                            unsafe { MapChunked::from_storage_unchecked(dtype, storage) }
619                                .into_series(),
620                        )
621                    },
622                    _ => Ok(storage),
623                }
624            },
625            ArrowDataType::Interval(IntervalUnit::MonthDayNano) => {
626                check_allow_importing_interval_as_struct("month_day_nano_interval")?;
627
628                feature_gated!("dtype-struct", {
629                    let chunks = chunks
630                        .into_iter()
631                        .map(convert_month_day_nano_to_struct)
632                        .collect::<PolarsResult<Vec<_>>>()?;
633
634                    Ok(StructChunked::from_chunks_and_dtype_unchecked(
635                        name,
636                        chunks,
637                        DataType::_month_days_ns_struct_type(),
638                    )
639                    .into_series())
640                })
641            },
642
643            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
644        }
645    }
646
647    #[cfg(feature = "dtype-categorical")]
648    pub fn from_cats_and_dtype(
649        cats: &Series,
650        dtype: &DataType,
651        strict: bool,
652    ) -> PolarsResult<Series> {
653        use std::borrow::Cow;
654
655        let phys = dtype.cat_physical()?;
656        let phys_dtype = DataType::from(phys);
657
658        let mut casted = Cow::Borrowed(cats);
659        if cats.dtype() != &phys_dtype {
660            casted = Cow::Owned(cats.cast(&phys_dtype)?);
661        }
662
663        let out = with_match_categorical_physical_type!(phys, |$C| {
664            // SAFETY: we are guarded by the type system.
665            type PhysCa = ChunkedArray<<$C as PolarsCategoricalType>::PolarsPhysical>;
666            let ca: &PhysCa = casted.as_ref().as_ref().as_ref();
667            CategoricalChunked::<$C>::from_cats_and_dtype(ca.clone(), dtype.clone()).into_series()
668        });
669
670        if strict && out.null_count() != casted.null_count() {
671            polars_bail!(
672                ComputeError:
673                "found invalid category value when converting from physical to {dtype}",
674            );
675        }
676
677        Ok(out)
678    }
679}
680
681fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
682    arr.iter().map(|arr| f(&**arr)).collect()
683}
684
685/// Normalize the field names of one chunk of map entries. Only the names change, so the
686/// buffers stay valid.
687#[cfg(feature = "dtype-map")]
688fn rename_map_entries(entries: &mut ArrayRef) {
689    let ArrowDataType::Struct(fields) = entries.dtype_mut() else {
690        unreachable!("map entries are a struct")
691    };
692    let [key, value] = fields.as_mut_slice() else {
693        unreachable!("map entries have two fields")
694    };
695    key.name = MAP_KEY_NAME;
696    value.name = MAP_VALUE_NAME;
697}
698
699/// Converts to physical types and bubbles up the correct [`DataType`].
700///
701/// Errors propagate from the nested logical imports, e.g. a malformed `Map` child.
702#[allow(clippy::only_used_in_recursion)]
703unsafe fn to_physical_and_dtype(
704    arrays: Vec<ArrayRef>,
705    md: Option<&Metadata>,
706) -> PolarsResult<(Vec<ArrayRef>, DataType)> {
707    match arrays[0].dtype() {
708        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
709            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
710            Ok((chunks, DataType::String))
711        },
712        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
713            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
714            Ok((chunks, DataType::Binary))
715        },
716        #[allow(unused_variables)]
717        dt @ ArrowDataType::Dictionary(_, _, _) => {
718            feature_gated!("dtype-categorical", {
719                let s = unsafe {
720                    let dt = dt.clone();
721                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
722                }?;
723                Ok((s.chunks().clone(), s.dtype().clone()))
724            })
725        },
726        dt @ ArrowDataType::Extension(_) => {
727            feature_gated!("dtype-extension", {
728                let s = unsafe {
729                    let dt = dt.clone();
730                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
731                }?;
732                Ok((s.chunks().clone(), s.dtype().clone()))
733            })
734        },
735        ArrowDataType::List(field) => {
736            let out = convert(&arrays, |arr| {
737                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
738            });
739            to_physical_and_dtype(out, md)
740        },
741        #[cfg(feature = "dtype-array")]
742        ArrowDataType::FixedSizeList(field, size) => {
743            let values = arrays
744                .iter()
745                .map(|arr| {
746                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
747                    arr.values().clone()
748                })
749                .collect::<Vec<_>>();
750
751            let (converted_values, dtype) =
752                to_physical_and_dtype(values, field.metadata.as_deref())?;
753
754            let arrays = arrays
755                .iter()
756                .zip(converted_values)
757                .map(|(arr, values)| {
758                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
759
760                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
761                    Box::from(FixedSizeListArray::new(
762                        dtype,
763                        arr.len(),
764                        values,
765                        arr.validity().cloned(),
766                    )) as ArrayRef
767                })
768                .collect();
769            Ok((arrays, DataType::Array(Box::new(dtype), *size)))
770        },
771        ArrowDataType::LargeList(field) => {
772            let values = arrays
773                .iter()
774                .map(|arr| {
775                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
776                    arr.values().clone()
777                })
778                .collect::<Vec<_>>();
779
780            let (converted_values, dtype) =
781                to_physical_and_dtype(values, field.metadata.as_deref())?;
782
783            let arrays = arrays
784                .iter()
785                .zip(converted_values)
786                .map(|(arr, values)| {
787                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
788
789                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
790                    Box::from(ListArray::<i64>::new(
791                        dtype,
792                        arr.offsets().clone(),
793                        values,
794                        arr.validity().cloned(),
795                    )) as ArrayRef
796                })
797                .collect();
798            Ok((arrays, DataType::List(Box::new(dtype))))
799        },
800        ArrowDataType::Struct(_fields) => {
801            feature_gated!("dtype-struct", {
802                let mut pl_fields = None;
803                let mut out_arrays = Vec::with_capacity(arrays.len());
804                for arr in &arrays {
805                    let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
806                    let mut values = Vec::with_capacity(_fields.len());
807                    let mut dtypes = Vec::with_capacity(_fields.len());
808                    for (value, field) in arr.values().iter().zip(_fields.iter()) {
809                        let (mut value, dtype) =
810                            to_physical_and_dtype(vec![value.clone()], field.metadata.as_deref())?;
811                        values.push(value.pop().unwrap());
812                        dtypes.push(dtype);
813                    }
814
815                    let arrow_fields = values
816                        .iter()
817                        .zip(_fields.iter())
818                        .map(|(arr, field)| {
819                            ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
820                        })
821                        .collect();
822                    out_arrays.push(Box::new(StructArray::new(
823                        ArrowDataType::Struct(arrow_fields),
824                        arr.len(),
825                        values,
826                        arr.validity().cloned(),
827                    )) as ArrayRef);
828
829                    if pl_fields.is_none() {
830                        pl_fields = Some(
831                            _fields
832                                .iter()
833                                .zip(dtypes)
834                                .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
835                                .collect_vec(),
836                        )
837                    }
838                }
839
840                Ok((out_arrays, DataType::Struct(pl_fields.unwrap())))
841            })
842        },
843        // Use Series architecture to convert nested logical types to physical.
844        dt @ (ArrowDataType::Duration(_)
845        | ArrowDataType::Time32(_)
846        | ArrowDataType::Time64(_)
847        | ArrowDataType::Timestamp(_, _)
848        | ArrowDataType::Date32
849        | ArrowDataType::Decimal(_, _)
850        | ArrowDataType::Date64
851        | ArrowDataType::Map(_, _)) => {
852            let dt = dt.clone();
853            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt)?;
854            let dtype = s.dtype().clone();
855            Ok((std::mem::take(s.chunks_mut()), dtype))
856        },
857        dt => {
858            let dtype = DataType::from_arrow(dt, md);
859            Ok((arrays, dtype))
860        },
861    }
862}
863
864#[cfg(feature = "dtype-categorical")]
865unsafe fn import_arrow_dictionary_array(
866    name: PlSmallStr,
867    arr: Box<dyn Array>,
868    key_type: &arrow::datatypes::IntegerType,
869    polars_dtype: &DataType,
870) -> PolarsResult<Series> {
871    use arrow::datatypes::IntegerType as I;
872
873    if matches!(
874        polars_dtype,
875        DataType::Categorical(_, _) | DataType::Enum(_, _)
876    ) {
877        macro_rules! unpack_categorical_chunked {
878            ($dt:ty) => {{
879                let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
880                let keys = arr.keys();
881                let values = arr.values();
882                let values = cast(&**values, &ArrowDataType::Utf8View)?;
883                let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
884                with_match_categorical_physical_type!(polars_dtype.cat_physical().unwrap(), |$C| {
885                    let ca = CategoricalChunked::<$C>::from_str_iter(
886                        name,
887                        polars_dtype.clone(),
888                        keys.iter().map(|k| {
889                            let k: usize = (*k?).try_into().ok()?;
890                            values.get(k)
891                        }),
892                    )?;
893                    Ok(ca.into_series())
894                })
895            }};
896        }
897
898        match key_type {
899            I::Int8 => unpack_categorical_chunked!(i8),
900            I::UInt8 => unpack_categorical_chunked!(u8),
901            I::Int16 => unpack_categorical_chunked!(i16),
902            I::UInt16 => unpack_categorical_chunked!(u16),
903            I::Int32 => unpack_categorical_chunked!(i32),
904            I::UInt32 => unpack_categorical_chunked!(u32),
905            I::Int64 => unpack_categorical_chunked!(i64),
906            I::UInt64 => unpack_categorical_chunked!(u64),
907            _ => polars_bail!(
908                ComputeError: "unsupported arrow key type: {key_type:?}"
909            ),
910        }
911    } else {
912        macro_rules! unpack_keys_values {
913            ($dt:ty) => {{
914                let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
915                let keys = arr.keys();
916                let keys = polars_compute::cast::primitive_to_primitive::<
917                    $dt,
918                    <IdxType as PolarsNumericType>::Native,
919                >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
920                (keys, arr.values())
921            }};
922        }
923
924        let (keys, values) = match key_type {
925            I::Int8 => unpack_keys_values!(i8),
926            I::UInt8 => unpack_keys_values!(u8),
927            I::Int16 => unpack_keys_values!(i16),
928            I::UInt16 => unpack_keys_values!(u16),
929            I::Int32 => unpack_keys_values!(i32),
930            I::UInt32 => unpack_keys_values!(u32),
931            I::Int64 => unpack_keys_values!(i64),
932            I::UInt64 => unpack_keys_values!(u64),
933            _ => polars_bail!(
934                ComputeError: "unsupported arrow key type: {key_type:?}"
935            ),
936        };
937
938        let values = Series::_try_from_arrow_unchecked_with_md(
939            name,
940            vec![values.clone()],
941            values.dtype(),
942            None,
943        )?;
944
945        values.take(&IdxCa::from_chunks_and_dtype(
946            PlSmallStr::EMPTY,
947            vec![keys.to_boxed()],
948            IDX_DTYPE,
949        ))
950    }
951}
952
953#[cfg(feature = "dtype-struct")]
954fn convert_month_day_nano_to_struct(chunk: Box<dyn Array>) -> PolarsResult<Box<dyn Array>> {
955    let arr: &PrimitiveArray<months_days_ns> = chunk.as_any().downcast_ref().unwrap();
956
957    let values: &[months_days_ns] = arr.values();
958
959    let (months_out, days_out, nanoseconds_out): (Vec<i32>, Vec<i32>, Vec<i64>) = values
960        .iter()
961        .map(|x| (x.months(), x.days(), x.ns()))
962        .collect();
963
964    let out = StructArray::new(
965        DataType::_month_days_ns_struct_type()
966            .to_physical()
967            .to_arrow(CompatLevel::newest()),
968        arr.len(),
969        vec![
970            PrimitiveArray::<i32>::from_vec(months_out).boxed(),
971            PrimitiveArray::<i32>::from_vec(days_out).boxed(),
972            PrimitiveArray::<i64>::from_vec(nanoseconds_out).boxed(),
973        ],
974        arr.validity().cloned(),
975    );
976
977    Ok(out.boxed())
978}
979
980fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
981    let mut chunks_iter = chunks.iter();
982    let dtype: ArrowDataType = chunks_iter
983        .next()
984        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
985        .dtype()
986        .clone();
987
988    for chunk in chunks_iter {
989        if chunk.dtype() != &dtype {
990            polars_bail!(
991                ComputeError: "cannot create series from multiple arrays with different types"
992            );
993        }
994    }
995    Ok(dtype)
996}
997
998impl Series {
999    pub fn try_new<T>(
1000        name: PlSmallStr,
1001        data: T,
1002    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
1003    where
1004        (PlSmallStr, T): TryInto<Self>,
1005    {
1006        // # TODO
1007        // * Remove the TryFrom<tuple> impls in favor of this
1008        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
1009    }
1010}
1011
1012impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
1013    type Error = PolarsError;
1014
1015    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
1016        let (name, chunks) = name_arr;
1017
1018        let dtype = check_types(&chunks)?;
1019        // SAFETY:
1020        // dtype is checked
1021        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
1022    }
1023}
1024
1025impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
1026    type Error = PolarsError;
1027
1028    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
1029        let (name, arr) = name_arr;
1030        Series::try_from((name, vec![arr]))
1031    }
1032}
1033
1034impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
1035    type Error = PolarsError;
1036
1037    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
1038        let (field, chunks) = field_arr;
1039        let arrow_dt = field.dtype();
1040        let dtype = check_types(&chunks)?;
1041        let compatible = match (&dtype, arrow_dt) {
1042            // See #26174, we don't care about dictionary ordering.
1043            (
1044                ArrowDataType::Dictionary(int0, inner0, _ord0),
1045                ArrowDataType::Dictionary(int1, inner1, _ord1),
1046            ) => (int0, inner0) == (int1, inner1),
1047            (l, r) => l == r,
1048        };
1049        polars_ensure!(compatible, ComputeError: "Arrow Field dtype does not match the ArrayRef dtypes");
1050
1051        // SAFETY:
1052        // dtype is checked
1053        unsafe {
1054            Series::_try_from_arrow_unchecked_with_md(
1055                field.name.clone(),
1056                chunks,
1057                &dtype,
1058                field.metadata.as_deref(),
1059            )
1060        }
1061    }
1062}
1063
1064impl TryFrom<(&ArrowField, ArrayRef)> for Series {
1065    type Error = PolarsError;
1066
1067    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
1068        let (field, arr) = field_arr;
1069        Series::try_from((field, vec![arr]))
1070    }
1071}
1072
1073/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
1074/// into a [`Series`].
1075/// # Safety
1076///
1077/// This trait is marked `unsafe` as the `is_series` return is used
1078/// to transmute to `Series`. This must always return `false` except
1079/// for `Series` structs.
1080pub unsafe trait IntoSeries {
1081    fn is_series() -> bool {
1082        false
1083    }
1084
1085    fn into_series(self) -> Series
1086    where
1087        Self: Sized;
1088}
1089
1090impl<T> From<ChunkedArray<T>> for Series
1091where
1092    T: PolarsDataType,
1093    ChunkedArray<T>: IntoSeries,
1094{
1095    fn from(ca: ChunkedArray<T>) -> Self {
1096        ca.into_series()
1097    }
1098}
1099
1100#[cfg(feature = "dtype-date")]
1101impl From<DateChunked> for Series {
1102    fn from(a: DateChunked) -> Self {
1103        a.into_series()
1104    }
1105}
1106
1107#[cfg(feature = "dtype-datetime")]
1108impl From<DatetimeChunked> for Series {
1109    fn from(a: DatetimeChunked) -> Self {
1110        a.into_series()
1111    }
1112}
1113
1114#[cfg(feature = "dtype-duration")]
1115impl From<DurationChunked> for Series {
1116    fn from(a: DurationChunked) -> Self {
1117        a.into_series()
1118    }
1119}
1120
1121#[cfg(feature = "dtype-time")]
1122impl From<TimeChunked> for Series {
1123    fn from(a: TimeChunked) -> Self {
1124        a.into_series()
1125    }
1126}
1127
1128unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
1129    fn into_series(self) -> Series {
1130        Series(self)
1131    }
1132}
1133
1134unsafe impl IntoSeries for Series {
1135    fn is_series() -> bool {
1136        true
1137    }
1138
1139    #[inline]
1140    fn into_series(self) -> Series {
1141        self
1142    }
1143}
1144
1145fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
1146    let len = chunks.iter().map(|arr| arr.len()).sum();
1147    Series::new_null(name, len)
1148}
polars_core/series/from.rs

polars_core/series/
from.rs
