1use polars_arrow::datatypes::{IntervalUnit, Metadata};
2use polars_arrow::offset::OffsetsBuffer;
3use polars_arrow::types::months_days_ns;
4use polars_compute::cast::cast_unchecked as cast;
5#[cfg(feature = "dtype-decimal")]
6use polars_compute::decimal::dec128_fits;
7use polars_error::feature_gated;
8use polars_utils::itertools::Itertools;
9
10use crate::chunked_array::cast::{CastOptions, cast_chunks};
11#[cfg(feature = "object")]
12use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
13#[cfg(feature = "object")]
14use crate::chunked_array::object::registry::get_object_builder;
15use crate::config::check_allow_importing_interval_as_struct;
16use crate::prelude::*;
17
18impl Series {
19 pub fn from_array<A: ParameterFreeDtypeStaticArray>(name: PlSmallStr, array: A) -> Self {
20 unsafe {
21 Self::from_chunks_and_dtype_unchecked(
22 name,
23 vec![Box::new(array)],
24 &DataType::from_arrow_dtype(&A::get_dtype()),
25 )
26 }
27 }
28
29 pub fn from_chunk_and_dtype(
33 name: PlSmallStr,
34 chunk: ArrayRef,
35 dtype: &DataType,
36 ) -> PolarsResult<Self> {
37 polars_ensure!(
39 !dtype.contains_objects(),
40 InvalidOperation: "cannot create a series of type '{dtype}' from an arrow chunk: objects are process-local"
41 );
42 polars_ensure!(
43 !dtype.contains_unknown(),
44 InvalidOperation: "cannot create a series of type '{dtype}' from an arrow chunk"
45 );
46 #[cfg(feature = "dtype-map")]
47 dtype.ensure_valid_map_dtypes()?;
48
49 let physical = dtype.to_physical();
50 if &physical.to_arrow(CompatLevel::newest()) != chunk.dtype() {
51 polars_bail!(
52 InvalidOperation: "cannot create a series of type '{dtype}' of arrow chunk with type '{:?}'",
53 chunk.dtype()
54 );
55 }
56
57 let physical =
59 unsafe { Self::from_chunks_and_dtype_unchecked(name, vec![chunk], &physical) };
60 physical.try_from_physical(dtype)
61 }
62
63 pub unsafe fn from_chunks_and_dtype_unchecked(
78 name: PlSmallStr,
79 chunks: Vec<ArrayRef>,
80 dtype: &DataType,
81 ) -> Self {
82 use DataType::*;
83 match dtype {
84 Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
85 Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
86 Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
87 Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
88 UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
89 UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
90 UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
91 UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
92 #[cfg(feature = "dtype-i128")]
93 Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
94 #[cfg(feature = "dtype-u128")]
95 UInt128 => UInt128Chunked::from_chunks(name, chunks).into_series(),
96 #[cfg(feature = "dtype-date")]
97 Date => Int32Chunked::from_chunks(name, chunks)
98 .into_date()
99 .into_series(),
100 #[cfg(feature = "dtype-time")]
101 Time => Int64Chunked::from_chunks(name, chunks)
102 .into_time()
103 .into_series(),
104 #[cfg(feature = "dtype-duration")]
105 Duration(tu) => Int64Chunked::from_chunks(name, chunks)
106 .into_duration(*tu)
107 .into_series(),
108 #[cfg(feature = "dtype-datetime")]
109 Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
110 .into_datetime(*tu, tz.clone())
111 .into_series(),
112 #[cfg(feature = "dtype-decimal")]
113 Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
114 .into_decimal_unchecked(*precision, *scale)
115 .into_series(),
116 #[cfg(feature = "dtype-array")]
117 Array(_, _) => {
118 ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
119 .into_series()
120 },
121 List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
122 .into_series(),
123 String => StringChunked::from_chunks(name, chunks).into_series(),
124 Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
125 #[cfg(feature = "dtype-categorical")]
126 dt @ (Categorical(_, _) | Enum(_, _)) => {
127 with_match_categorical_physical_type!(dt.cat_physical().unwrap(), |$C| {
128 let phys = ChunkedArray::from_chunks(name, chunks);
129 CategoricalChunked::<$C>::from_cats_and_dtype_unchecked(phys, dt.clone()).into_series()
130 })
131 },
132 Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
133 #[cfg(feature = "dtype-f16")]
134 Float16 => Float16Chunked::from_chunks(name, chunks).into_series(),
135 Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
136 Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
137 BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
138 #[cfg(feature = "dtype-extension")]
139 Extension(typ, storage) => ExtensionChunked::from_storage(
140 typ.clone(),
141 Series::from_chunks_and_dtype_unchecked(name, chunks, storage),
142 )
143 .into_series(),
144 #[cfg(feature = "dtype-map")]
145 Map(_, _) => {
146 let storage = Series::from_chunks_and_dtype_unchecked(
147 name,
148 chunks,
149 &dtype.map_storage_dtype().unwrap(),
150 );
151 MapChunked::from_storage_unchecked(dtype.clone(), storage).into_series()
152 },
153 #[cfg(feature = "dtype-struct")]
154 Struct(_) => {
155 let mut ca =
156 StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
157 StructChunked::propagate_nulls_mut(&mut ca);
158 ca.into_series()
159 },
160 #[cfg(feature = "object")]
161 Object(_) => {
162 if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
163 assert_eq!(chunks.len(), 1);
164 {
169 let pe = PolarsExtension::new(arr.clone());
170 let s = pe.get_series(&name);
171 pe.take_and_forget();
172 s
173 }
174 } else {
175 unsafe { get_object_builder(name, 0).from_chunks(chunks) }
176 }
177 },
178 Null => new_null(name, &chunks),
179 Unknown(_) => {
180 panic!("dtype is unknown; consider supplying data-types for all operations")
181 },
182 #[allow(unreachable_patterns)]
183 _ => unreachable!(),
184 }
185 }
186
187 pub unsafe fn _try_from_arrow_unchecked(
190 name: PlSmallStr,
191 chunks: Vec<ArrayRef>,
192 dtype: &ArrowDataType,
193 ) -> PolarsResult<Self> {
194 Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
195 }
196
197 pub unsafe fn _try_from_arrow_unchecked_with_md(
202 name: PlSmallStr,
203 mut chunks: Vec<ArrayRef>,
204 dtype: &ArrowDataType,
205 md: Option<&Metadata>,
206 ) -> PolarsResult<Self> {
207 match dtype {
208 ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
209 ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
210 let chunks =
211 cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
212 Ok(StringChunked::from_chunks(name, chunks).into_series())
213 },
214 ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
215 ArrowDataType::LargeBinary => {
216 if let Some(md) = md {
217 if md.maintain_type() {
218 return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
219 }
220 }
221 let chunks =
222 cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
223 Ok(BinaryChunked::from_chunks(name, chunks).into_series())
224 },
225 ArrowDataType::Binary => {
226 let chunks =
227 cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
228 Ok(BinaryChunked::from_chunks(name, chunks).into_series())
229 },
230 ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
231 let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
232 unsafe {
233 Ok(
234 ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
235 .into_series(),
236 )
237 }
238 },
239 #[cfg(feature = "dtype-array")]
240 ArrowDataType::FixedSizeList(_, _) => {
241 let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
242 unsafe {
243 Ok(
244 ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
245 .into_series(),
246 )
247 }
248 },
249 ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
250 #[cfg(feature = "dtype-u8")]
251 ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
252 #[cfg(feature = "dtype-u16")]
253 ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
254 ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
255 ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
256 ArrowDataType::UInt128 => feature_gated!(
257 "dtype-u128",
258 Ok(UInt128Chunked::from_chunks(name, chunks).into_series())
259 ),
260 #[cfg(feature = "dtype-i8")]
261 ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
262 #[cfg(feature = "dtype-i16")]
263 ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
264 ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
265 ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
266 ArrowDataType::Int128 => feature_gated!(
267 "dtype-i128",
268 Ok(Int128Chunked::from_chunks(name, chunks).into_series())
269 ),
270 #[cfg(feature = "dtype-f16")]
271 ArrowDataType::Float16 => {
272 let chunks =
273 cast_chunks(&chunks, &DataType::Float16, CastOptions::NonStrict).unwrap();
274 Ok(Float16Chunked::from_chunks(name, chunks).into_series())
275 },
276 ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
277 ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
278 #[cfg(feature = "dtype-date")]
279 ArrowDataType::Date32 => {
280 let chunks =
281 cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
282 Ok(Int32Chunked::from_chunks(name, chunks)
283 .into_date()
284 .into_series())
285 },
286 #[cfg(feature = "dtype-datetime")]
287 ArrowDataType::Date64 => {
288 let chunks =
289 cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
290 let ca = Int64Chunked::from_chunks(name, chunks);
291 Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
292 },
293 #[cfg(feature = "dtype-datetime")]
294 ArrowDataType::Timestamp(tu, tz) => {
295 let tz = TimeZone::opt_try_new(tz.clone())?;
296 let chunks =
297 cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
298 let s = Int64Chunked::from_chunks(name, chunks)
299 .into_datetime(tu.into(), tz)
300 .into_series();
301 Ok(scale_arrow_values(s, dtype))
302 },
303 #[cfg(feature = "dtype-duration")]
304 ArrowDataType::Duration(tu) => {
305 let chunks =
306 cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
307 let s = Int64Chunked::from_chunks(name, chunks)
308 .into_duration(tu.into())
309 .into_series();
310 Ok(scale_arrow_values(s, dtype))
311 },
312 #[cfg(feature = "dtype-time")]
313 ArrowDataType::Time64(_) | ArrowDataType::Time32(_) => {
314 let mut chunks = chunks;
315 if matches!(dtype, ArrowDataType::Time32(_)) {
316 chunks =
317 cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
318 }
319 let chunks =
320 cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
321 let s = Int64Chunked::from_chunks(name, chunks)
322 .into_time()
323 .into_series();
324 Ok(scale_arrow_values(s, dtype))
325 },
326 ArrowDataType::Decimal32(precision, scale) => {
327 feature_gated!("dtype-decimal", {
328 polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
329
330 let mut chunks = chunks;
331 for chunk in chunks.iter_mut() {
332 let old_chunk = chunk
333 .as_any_mut()
334 .downcast_mut::<PrimitiveArray<i32>>()
335 .unwrap();
336
337 let (_, values, validity) = std::mem::take(old_chunk).into_inner();
339 *chunk = PrimitiveArray::new(
340 ArrowDataType::Int128,
341 values.iter().map(|&v| v as i128).collect(),
342 validity,
343 )
344 .to_boxed();
345 }
346
347 let s = Int128Chunked::from_chunks(name, chunks)
348 .into_decimal_unchecked(*precision, *scale)
349 .into_series();
350 Ok(s)
351 })
352 },
353 ArrowDataType::Decimal64(precision, scale) => {
354 feature_gated!("dtype-decimal", {
355 polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
356
357 let mut chunks = chunks;
358 for chunk in chunks.iter_mut() {
359 let old_chunk = chunk
360 .as_any_mut()
361 .downcast_mut::<PrimitiveArray<i64>>()
362 .unwrap();
363
364 let (_, values, validity) = std::mem::take(old_chunk).into_inner();
366 *chunk = PrimitiveArray::new(
367 ArrowDataType::Int128,
368 values.iter().map(|&v| v as i128).collect(),
369 validity,
370 )
371 .to_boxed();
372 }
373
374 let s = Int128Chunked::from_chunks(name, chunks)
375 .into_decimal_unchecked(*precision, *scale)
376 .into_series();
377 Ok(s)
378 })
379 },
380 ArrowDataType::Decimal(precision, scale) => {
381 feature_gated!("dtype-decimal", {
382 polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
383
384 let mut chunks = chunks;
385 for chunk in chunks.iter_mut() {
386 *chunk = std::mem::take(
387 chunk
388 .as_any_mut()
389 .downcast_mut::<PrimitiveArray<i128>>()
390 .unwrap(),
391 )
392 .to(ArrowDataType::Int128)
393 .to_boxed();
394 }
395
396 let s = Int128Chunked::from_chunks(name, chunks)
397 .into_decimal_unchecked(*precision, *scale)
398 .into_series();
399 Ok(s)
400 })
401 },
402 ArrowDataType::Decimal256(precision, scale) => {
403 feature_gated!("dtype-decimal", {
404 use polars_arrow::types::i256;
405
406 polars_compute::decimal::dec128_verify_prec_scale(*precision, *scale)?;
407
408 let mut chunks = chunks;
409 for chunk in chunks.iter_mut() {
410 let arr = std::mem::take(
411 chunk
412 .as_any_mut()
413 .downcast_mut::<PrimitiveArray<i256>>()
414 .unwrap(),
415 );
416 let arr_128: PrimitiveArray<i128> = arr.iter().map(|opt_v| {
417 if let Some(v) = opt_v {
418 let smaller: Option<i128> = (*v).try_into().ok();
419 let smaller = smaller.filter(|v| dec128_fits(*v, *precision));
420 smaller.ok_or_else(|| {
421 polars_err!(ComputeError: "Decimal256 to Decimal128 conversion overflowed, Decimal256 is not (yet) supported in Polars")
422 }).map(Some)
423 } else {
424 Ok(None)
425 }
426 }).try_collect_arr_trusted()?;
427
428 *chunk = arr_128.to(ArrowDataType::Int128).to_boxed();
429 }
430
431 let s = Int128Chunked::from_chunks(name, chunks)
432 .into_decimal_unchecked(*precision, *scale)
433 .into_series();
434 Ok(s)
435 })
436 },
437 ArrowDataType::Null => Ok(new_null(name, &chunks)),
438 #[cfg(not(feature = "dtype-categorical"))]
439 ArrowDataType::Dictionary(_, _, _) => {
440 panic!("activate dtype-categorical to convert dictionary arrays")
441 },
442 #[cfg(feature = "dtype-categorical")]
443 ArrowDataType::Dictionary(key_type, _, _) => {
444 let polars_dtype = DataType::from_arrow(chunks[0].dtype(), md);
445
446 let mut series_iter = chunks.into_iter().map(|arr| {
447 import_arrow_dictionary_array(name.clone(), arr, key_type, &polars_dtype)
448 });
449
450 let mut first = series_iter.next().unwrap()?;
451
452 for s in series_iter {
453 first.append_owned(s?)?;
454 }
455
456 Ok(first)
457 },
458 #[cfg(feature = "object")]
459 ArrowDataType::Extension(ext)
460 if ext.name == POLARS_OBJECT_EXTENSION_NAME && ext.metadata.is_some() =>
461 {
462 assert_eq!(chunks.len(), 1);
463 let arr = chunks[0]
464 .as_any()
465 .downcast_ref::<FixedSizeBinaryArray>()
466 .unwrap();
467 let s = {
472 let pe = PolarsExtension::new(arr.clone());
473 let s = pe.get_series(&name);
474 pe.take_and_forget();
475 s
476 };
477 Ok(s)
478 },
479 #[cfg(feature = "dtype-extension")]
480 ArrowDataType::Extension(ext) => {
481 use crate::datatypes::extension::get_extension_type_or_storage;
482
483 for chunk in &mut chunks {
484 debug_assert!(
485 chunk.dtype() == dtype,
486 "expected chunk dtype to be {:?}, got {:?}",
487 dtype,
488 chunk.dtype()
489 );
490 *chunk.dtype_mut() = ext.inner.clone();
491 }
492 let storage = Series::_try_from_arrow_unchecked_with_md(
493 name.clone(),
494 chunks,
495 &ext.inner,
496 md,
497 )?;
498
499 Ok(
500 match get_extension_type_or_storage(
501 &ext.name,
502 storage.dtype(),
503 ext.metadata.as_deref(),
504 ) {
505 Some(typ) => ExtensionChunked::from_storage(typ, storage).into_series(),
506 None => storage,
507 },
508 )
509 },
510
511 #[cfg(feature = "dtype-struct")]
512 ArrowDataType::Struct(fields) => {
513 let names = fields.iter().map(|field| field.name.as_str()).collect_vec();
514 crate::frame::validation::ensure_names_unique(&names)?;
515
516 let (chunks, dtype) = to_physical_and_dtype(chunks, md)?;
517
518 unsafe {
519 let mut ca =
520 StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
521 StructChunked::propagate_nulls_mut(&mut ca);
522 Ok(ca.into_series())
523 }
524 },
525 ArrowDataType::FixedSizeBinary(_) => {
526 let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
527 Ok(BinaryChunked::from_chunks(name, chunks).into_series())
528 },
529 ArrowDataType::Map(field, _keys_sorted) => {
530 let struct_arrays = chunks
531 .iter()
532 .map(|arr| {
533 let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
534 arr.field().clone()
535 })
536 .collect::<Vec<_>>();
537
538 let (phys_struct_arrays, entries_dtype) =
539 to_physical_and_dtype(struct_arrays, field.metadata.as_deref())?;
540
541 #[cfg(feature = "dtype-map")]
542 let map_dtype = entries_dtype.map_from_positional_entries_dtype();
543 #[cfg(not(feature = "dtype-map"))]
544 let map_dtype: Option<DataType> = None;
545
546 #[cfg(feature = "dtype-map")]
547 let phys_struct_arrays: Vec<ArrayRef> = if map_dtype.is_some() {
548 phys_struct_arrays
549 .into_iter()
550 .map(|mut entries| {
551 rename_map_entries(&mut entries);
552 entries
553 })
554 .collect()
555 } else {
556 phys_struct_arrays
557 };
558
559 let storage_dtype = match &map_dtype {
560 #[cfg(feature = "dtype-map")]
561 Some(dtype) => dtype.map_storage_dtype().unwrap(),
562 _ => DataType::List(Box::new(entries_dtype)),
563 };
564
565 let chunks = chunks
566 .iter()
567 .zip(phys_struct_arrays)
568 .map(|(arr, values)| {
569 let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
570 let offsets: &OffsetsBuffer<i32> = arr.offsets();
571
572 let validity = arr.validity().cloned();
573
574 Box::from(ListArray::<i64>::new(
575 ListArray::<i64>::default_datatype(values.dtype().clone()),
576 OffsetsBuffer::<i64>::from(offsets),
577 values,
578 validity,
579 )) as ArrayRef
580 })
581 .collect();
582
583 let storage = unsafe {
584 ListChunked::from_chunks_and_dtype_unchecked(name, chunks, storage_dtype)
585 }
586 .into_series();
587
588 match map_dtype {
589 #[cfg(feature = "dtype-map")]
590 Some(dtype) => {
591 use crate::chunked_array::logical::ensure_live_entries_non_null;
592
593 ensure_live_entries_non_null(storage.list().unwrap())?;
596 Ok(
599 unsafe { MapChunked::from_storage_unchecked(dtype, storage) }
600 .into_series(),
601 )
602 },
603 _ => Ok(storage),
604 }
605 },
606 ArrowDataType::Interval(IntervalUnit::MonthDayNano) => {
607 check_allow_importing_interval_as_struct("month_day_nano_interval")?;
608
609 feature_gated!("dtype-struct", {
610 let chunks = chunks
611 .into_iter()
612 .map(convert_month_day_nano_to_struct)
613 .collect::<PolarsResult<Vec<_>>>()?;
614
615 let mut ca = StructChunked::from_chunks_and_dtype_unchecked(
616 name,
617 chunks,
618 DataType::_month_days_ns_struct_type(),
619 );
620 StructChunked::propagate_nulls_mut(&mut ca);
621 Ok(ca.into_series())
622 })
623 },
624
625 dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
626 }
627 }
628
629 #[cfg(feature = "dtype-categorical")]
630 pub fn from_cats_and_dtype(
631 cats: &Series,
632 dtype: &DataType,
633 strict: bool,
634 ) -> PolarsResult<Series> {
635 use std::borrow::Cow;
636
637 let phys = dtype.cat_physical()?;
638 let phys_dtype = DataType::from(phys);
639
640 let mut casted = Cow::Borrowed(cats);
641 if cats.dtype() != &phys_dtype {
642 casted = Cow::Owned(cats.cast(&phys_dtype)?);
643 }
644
645 let out = with_match_categorical_physical_type!(phys, |$C| {
646 type PhysCa = ChunkedArray<<$C as PolarsCategoricalType>::PolarsPhysical>;
648 let ca: &PhysCa = casted.as_ref().as_ref().as_ref();
649 CategoricalChunked::<$C>::from_cats_and_dtype(ca.clone(), dtype.clone()).into_series()
650 });
651
652 if strict && out.null_count() != casted.null_count() {
653 polars_bail!(
654 ComputeError:
655 "found invalid category value when converting from physical to {dtype}",
656 );
657 }
658
659 Ok(out)
660 }
661}
662
663#[cfg(any(
666 feature = "dtype-datetime",
667 feature = "dtype-duration",
668 feature = "dtype-time"
669))]
670fn scale_arrow_values(s: Series, dtype: &ArrowDataType) -> Series {
671 match DataType::arrow_value_scale(dtype) {
672 1 => s,
673 factor => &s * factor,
674 }
675}
676
677fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
678 arr.iter().map(|arr| f(&**arr)).collect()
679}
680
681#[cfg(feature = "dtype-map")]
684fn rename_map_entries(entries: &mut ArrayRef) {
685 let ArrowDataType::Struct(fields) = entries.dtype_mut() else {
686 unreachable!("map entries are a struct")
687 };
688 let [key, value] = fields.as_mut_slice() else {
689 unreachable!("map entries have two fields")
690 };
691 key.name = MAP_KEY_NAME;
692 value.name = MAP_VALUE_NAME;
693}
694
695#[allow(clippy::only_used_in_recursion)]
699unsafe fn to_physical_and_dtype(
700 arrays: Vec<ArrayRef>,
701 md: Option<&Metadata>,
702) -> PolarsResult<(Vec<ArrayRef>, DataType)> {
703 match arrays[0].dtype() {
704 ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
705 let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
706 Ok((chunks, DataType::String))
707 },
708 ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
709 let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
710 Ok((chunks, DataType::Binary))
711 },
712 #[allow(unused_variables)]
713 dt @ ArrowDataType::Dictionary(_, _, _) => {
714 feature_gated!("dtype-categorical", {
715 let s = unsafe {
716 let dt = dt.clone();
717 Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
718 }?;
719 Ok((s.chunks().clone(), s.dtype().clone()))
720 })
721 },
722 dt @ ArrowDataType::Extension(_) => {
723 feature_gated!("dtype-extension", {
724 let s = unsafe {
725 let dt = dt.clone();
726 Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
727 }?;
728 Ok((s.chunks().clone(), s.dtype().clone()))
729 })
730 },
731 ArrowDataType::List(field) => {
732 let out = convert(&arrays, |arr| {
733 cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
734 });
735 to_physical_and_dtype(out, md)
736 },
737 #[cfg(feature = "dtype-array")]
738 ArrowDataType::FixedSizeList(field, size) => {
739 let values = arrays
740 .iter()
741 .map(|arr| {
742 let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
743 arr.values().clone()
744 })
745 .collect::<Vec<_>>();
746
747 let (converted_values, dtype) =
748 to_physical_and_dtype(values, field.metadata.as_deref())?;
749
750 let arrays = arrays
751 .iter()
752 .zip(converted_values)
753 .map(|(arr, values)| {
754 let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
755
756 let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
757 Box::from(FixedSizeListArray::new(
758 dtype,
759 arr.len(),
760 values,
761 arr.validity().cloned(),
762 )) as ArrayRef
763 })
764 .collect();
765 Ok((arrays, DataType::Array(Box::new(dtype), *size)))
766 },
767 ArrowDataType::LargeList(field) => {
768 let values = arrays
769 .iter()
770 .map(|arr| {
771 let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
772 arr.values().clone()
773 })
774 .collect::<Vec<_>>();
775
776 let (converted_values, dtype) =
777 to_physical_and_dtype(values, field.metadata.as_deref())?;
778
779 let arrays = arrays
780 .iter()
781 .zip(converted_values)
782 .map(|(arr, values)| {
783 let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
784
785 let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
786 Box::from(ListArray::<i64>::new(
787 dtype,
788 arr.offsets().clone(),
789 values,
790 arr.validity().cloned(),
791 )) as ArrayRef
792 })
793 .collect();
794 Ok((arrays, DataType::List(Box::new(dtype))))
795 },
796 ArrowDataType::Struct(_fields) => {
797 feature_gated!("dtype-struct", {
798 let mut pl_fields = None;
799 let mut out_arrays = Vec::with_capacity(arrays.len());
800 for arr in &arrays {
801 let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
802 let mut values = Vec::with_capacity(_fields.len());
803 let mut dtypes = Vec::with_capacity(_fields.len());
804 for (value, field) in arr.values().iter().zip(_fields.iter()) {
805 let (mut value, dtype) =
806 to_physical_and_dtype(vec![value.clone()], field.metadata.as_deref())?;
807 values.push(value.pop().unwrap());
808 dtypes.push(dtype);
809 }
810
811 let arrow_fields = values
812 .iter()
813 .zip(_fields.iter())
814 .map(|(arr, field)| {
815 ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
816 })
817 .collect();
818 out_arrays.push(Box::new(StructArray::new(
819 ArrowDataType::Struct(arrow_fields),
820 arr.len(),
821 values,
822 arr.validity().cloned(),
823 )) as ArrayRef);
824
825 if pl_fields.is_none() {
826 pl_fields = Some(
827 _fields
828 .iter()
829 .zip(dtypes)
830 .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
831 .collect_vec(),
832 )
833 }
834 }
835
836 Ok((out_arrays, DataType::Struct(pl_fields.unwrap())))
837 })
838 },
839 dt @ (ArrowDataType::Duration(_)
841 | ArrowDataType::Time32(_)
842 | ArrowDataType::Time64(_)
843 | ArrowDataType::Timestamp(_, _)
844 | ArrowDataType::Date32
845 | ArrowDataType::Decimal32(_, _)
846 | ArrowDataType::Decimal64(_, _)
847 | ArrowDataType::Decimal(_, _)
848 | ArrowDataType::Decimal256(_, _)
849 | ArrowDataType::Date64
850 | ArrowDataType::Map(_, _)
851 | ArrowDataType::Interval(_)) => {
852 let dt = dt.clone();
853 let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt)?;
854 let dtype = s.dtype().clone();
855 Ok((std::mem::take(s.chunks_mut()), dtype))
856 },
857 dt => {
858 let dtype = DataType::from_arrow(dt, md);
859 Ok((arrays, dtype))
860 },
861 }
862}
863
864#[cfg(feature = "dtype-categorical")]
866fn unsigned_dictionary_keys<K: DictionaryKey, O: DictionaryKey>(
867 keys: &PrimitiveArray<K>,
868) -> PrimitiveArray<O> {
869 if let Some(keys) = (keys as &dyn std::any::Any).downcast_ref::<PrimitiveArray<O>>() {
870 return keys.clone();
871 }
872 let max = unsafe { O::from_usize_unchecked(O::MAX_USIZE_VALUE) };
874 let mut has_negative = false;
875 let values: Vec<O> = keys
876 .values()
877 .iter()
878 .map(|k| {
879 let k: Result<usize, _> = (*k).try_into();
880 has_negative |= k.is_err();
881 k.map_or(O::default(), |k| O::try_from(k).unwrap_or(max))
882 })
883 .collect_trusted();
884 let mut validity = keys.validity().cloned();
885 if has_negative {
886 let is_non_negative: polars_arrow::bitmap::Bitmap = keys
887 .values()
888 .iter()
889 .map(|k| TryInto::<usize>::try_into(*k).is_ok())
890 .collect_trusted();
891 validity = polars_arrow::compute::utils::combine_validities_and(
892 validity.as_ref(),
893 Some(&is_non_negative),
894 );
895 }
896 PrimitiveArray::from_vec(values).with_validity(validity)
897}
898
899#[cfg(feature = "dtype-categorical")]
900fn import_str_dictionary<O: DictionaryKey>(
901 name: PlSmallStr,
902 keys: &PrimitiveArray<O>,
903 values: &dyn Array,
904 dtype: &DataType,
905) -> PolarsResult<Series> {
906 with_match_categorical_physical_type!(dtype.cat_physical().unwrap(), |$C| {
907 let ca = CategoricalChunked::<$C>::from_str_dictionary(name, dtype.clone(), keys, values)?;
908 Ok(ca.into_series())
909 })
910}
911
912#[cfg(feature = "dtype-categorical")]
913unsafe fn import_arrow_dictionary_array(
914 name: PlSmallStr,
915 arr: Box<dyn Array>,
916 key_type: &polars_arrow::datatypes::IntegerType,
917 polars_dtype: &DataType,
918) -> PolarsResult<Series> {
919 use polars_arrow::datatypes::IntegerType as I;
920
921 if matches!(
922 polars_dtype,
923 DataType::Categorical(_, _) | DataType::Enum(_, _)
924 ) {
925 macro_rules! unpack_categorical_chunked {
926 ($dt:ty) => {{
927 let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
928 let values = arr.values().as_ref();
929 if values.len() <= u32::MAX as usize {
931 let keys = unsigned_dictionary_keys::<$dt, u32>(arr.keys());
932 import_str_dictionary(name, &keys, values, polars_dtype)
933 } else {
934 let keys = unsigned_dictionary_keys::<$dt, u64>(arr.keys());
935 import_str_dictionary(name, &keys, values, polars_dtype)
936 }
937 }};
938 }
939
940 match key_type {
941 I::Int8 => unpack_categorical_chunked!(i8),
942 I::UInt8 => unpack_categorical_chunked!(u8),
943 I::Int16 => unpack_categorical_chunked!(i16),
944 I::UInt16 => unpack_categorical_chunked!(u16),
945 I::Int32 => unpack_categorical_chunked!(i32),
946 I::UInt32 => unpack_categorical_chunked!(u32),
947 I::Int64 => unpack_categorical_chunked!(i64),
948 I::UInt64 => unpack_categorical_chunked!(u64),
949 _ => polars_bail!(
950 ComputeError: "unsupported arrow key type: {key_type:?}"
951 ),
952 }
953 } else {
954 macro_rules! unpack_keys_values {
955 ($dt:ty) => {{
956 let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
957 let keys = arr.keys();
958 let keys = polars_compute::cast::primitive_to_primitive::<
959 $dt,
960 <IdxType as PolarsNumericType>::Native,
961 >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
962 (keys, arr.values())
963 }};
964 }
965
966 let (keys, values) = match key_type {
967 I::Int8 => unpack_keys_values!(i8),
968 I::UInt8 => unpack_keys_values!(u8),
969 I::Int16 => unpack_keys_values!(i16),
970 I::UInt16 => unpack_keys_values!(u16),
971 I::Int32 => unpack_keys_values!(i32),
972 I::UInt32 => unpack_keys_values!(u32),
973 I::Int64 => unpack_keys_values!(i64),
974 I::UInt64 => unpack_keys_values!(u64),
975 _ => polars_bail!(
976 ComputeError: "unsupported arrow key type: {key_type:?}"
977 ),
978 };
979
980 let values = Series::_try_from_arrow_unchecked_with_md(
981 name,
982 vec![values.clone()],
983 values.dtype(),
984 None,
985 )?;
986
987 values.take(&IdxCa::from_chunks_and_dtype(
988 PlSmallStr::EMPTY,
989 vec![keys.to_boxed()],
990 IDX_DTYPE,
991 ))
992 }
993}
994
995#[cfg(feature = "dtype-struct")]
996fn convert_month_day_nano_to_struct(chunk: Box<dyn Array>) -> PolarsResult<Box<dyn Array>> {
997 let arr: &PrimitiveArray<months_days_ns> = chunk.as_any().downcast_ref().unwrap();
998
999 let values: &[months_days_ns] = arr.values();
1000
1001 let (months_out, days_out, nanoseconds_out): (Vec<i32>, Vec<i32>, Vec<i64>) = values
1002 .iter()
1003 .map(|x| (x.months(), x.days(), x.ns()))
1004 .collect();
1005
1006 let out = StructArray::new(
1007 DataType::_month_days_ns_struct_type()
1008 .to_physical()
1009 .to_arrow(CompatLevel::newest()),
1010 arr.len(),
1011 vec![
1012 PrimitiveArray::<i32>::from_vec(months_out).boxed(),
1013 PrimitiveArray::<i32>::from_vec(days_out).boxed(),
1014 PrimitiveArray::<i64>::from_vec(nanoseconds_out).boxed(),
1015 ],
1016 arr.validity().cloned(),
1017 );
1018
1019 Ok(out.boxed())
1020}
1021
1022fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
1023 let mut chunks_iter = chunks.iter();
1024 let dtype: ArrowDataType = chunks_iter
1025 .next()
1026 .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
1027 .dtype()
1028 .clone();
1029
1030 for chunk in chunks_iter {
1031 if chunk.dtype() != &dtype {
1032 polars_bail!(
1033 ComputeError: "cannot create series from multiple arrays with different types"
1034 );
1035 }
1036 }
1037 Ok(dtype)
1038}
1039
1040impl Series {
1041 pub fn try_new<T>(
1042 name: PlSmallStr,
1043 data: T,
1044 ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
1045 where
1046 (PlSmallStr, T): TryInto<Self>,
1047 {
1048 <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
1051 }
1052}
1053
1054impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
1055 type Error = PolarsError;
1056
1057 fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
1058 let (name, chunks) = name_arr;
1059
1060 let dtype = check_types(&chunks)?;
1061 unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
1064 }
1065}
1066
1067impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
1068 type Error = PolarsError;
1069
1070 fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
1071 let (name, arr) = name_arr;
1072 Series::try_from((name, vec![arr]))
1073 }
1074}
1075
1076impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
1077 type Error = PolarsError;
1078
1079 fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
1080 let (field, chunks) = field_arr;
1081 let arrow_dt = field.dtype();
1082 let dtype = check_types(&chunks)?;
1083 let compatible = match (&dtype, arrow_dt) {
1084 (
1086 ArrowDataType::Dictionary(int0, inner0, _ord0),
1087 ArrowDataType::Dictionary(int1, inner1, _ord1),
1088 ) => (int0, inner0) == (int1, inner1),
1089 (l, r) => l == r,
1090 };
1091 polars_ensure!(compatible, ComputeError: "Arrow Field dtype does not match the ArrayRef dtypes");
1092
1093 unsafe {
1096 Series::_try_from_arrow_unchecked_with_md(
1097 field.name.clone(),
1098 chunks,
1099 &dtype,
1100 field.metadata.as_deref(),
1101 )
1102 }
1103 }
1104}
1105
1106impl TryFrom<(&ArrowField, ArrayRef)> for Series {
1107 type Error = PolarsError;
1108
1109 fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
1110 let (field, arr) = field_arr;
1111 Series::try_from((field, vec![arr]))
1112 }
1113}
1114
1115pub unsafe trait IntoSeries {
1123 fn is_series() -> bool {
1124 false
1125 }
1126
1127 fn into_series(self) -> Series
1128 where
1129 Self: Sized;
1130}
1131
1132impl<T> From<ChunkedArray<T>> for Series
1133where
1134 T: PolarsDataType,
1135 ChunkedArray<T>: IntoSeries,
1136{
1137 fn from(ca: ChunkedArray<T>) -> Self {
1138 ca.into_series()
1139 }
1140}
1141
1142#[cfg(feature = "dtype-date")]
1143impl From<DateChunked> for Series {
1144 fn from(a: DateChunked) -> Self {
1145 a.into_series()
1146 }
1147}
1148
1149#[cfg(feature = "dtype-datetime")]
1150impl From<DatetimeChunked> for Series {
1151 fn from(a: DatetimeChunked) -> Self {
1152 a.into_series()
1153 }
1154}
1155
1156#[cfg(feature = "dtype-duration")]
1157impl From<DurationChunked> for Series {
1158 fn from(a: DurationChunked) -> Self {
1159 a.into_series()
1160 }
1161}
1162
1163#[cfg(feature = "dtype-time")]
1164impl From<TimeChunked> for Series {
1165 fn from(a: TimeChunked) -> Self {
1166 a.into_series()
1167 }
1168}
1169
1170unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
1171 fn into_series(self) -> Series {
1172 Series(self)
1173 }
1174}
1175
1176unsafe impl IntoSeries for Series {
1177 fn is_series() -> bool {
1178 true
1179 }
1180
1181 #[inline]
1182 fn into_series(self) -> Series {
1183 self
1184 }
1185}
1186
1187fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
1188 let len = chunks.iter().map(|arr| arr.len()).sum();
1189 Series::new_null(name, len)
1190}
1191
1192#[cfg(test)]
1193#[cfg(all(
1194 feature = "dtype-datetime",
1195 feature = "dtype-duration",
1196 feature = "dtype-time"
1197))]
1198mod tests {
1199 use polars_arrow::array::PrimitiveArray;
1200
1201 use super::*;
1202
1203 #[test]
1204 fn imported_values_are_scaled_like_the_dtype_says() {
1205 let import = |dtype: ArrowDataType, value: i64| {
1206 let array = PrimitiveArray::from_vec(vec![value]).to(dtype.clone());
1207 let s = Series::from_arrow(PlSmallStr::EMPTY, array.boxed()).unwrap();
1208 let physical = s.to_physical_repr().i64().unwrap().get(0).unwrap();
1209 assert_eq!(physical, value * DataType::arrow_value_scale(&dtype));
1210 (s.dtype().clone(), physical)
1211 };
1212 let seconds = ArrowDataType::Timestamp(ArrowTimeUnit::Second, None);
1213 assert_eq!(
1214 import(seconds, -7),
1215 (DataType::Datetime(TimeUnit::Milliseconds, None), -7_000)
1216 );
1217 assert_eq!(
1218 import(ArrowDataType::Duration(ArrowTimeUnit::Second), 7),
1219 (DataType::Duration(TimeUnit::Milliseconds), 7_000)
1220 );
1221 assert_eq!(
1222 import(ArrowDataType::Duration(ArrowTimeUnit::Microsecond), 7),
1223 (DataType::Duration(TimeUnit::Microseconds), 7)
1224 );
1225 assert_eq!(
1226 import(ArrowDataType::Time64(ArrowTimeUnit::Microsecond), 7),
1227 (DataType::Time, 7_000)
1228 );
1229 assert_eq!(
1230 import(ArrowDataType::Date64, 86_400_000),
1231 (DataType::Datetime(TimeUnit::Milliseconds, None), 86_400_000)
1232 );
1233 let array = PrimitiveArray::from_vec(vec![7i32])
1234 .to(ArrowDataType::Time32(ArrowTimeUnit::Millisecond));
1235 let s = Series::from_arrow(PlSmallStr::EMPTY, array.boxed()).unwrap();
1236 assert_eq!(s.dtype(), &DataType::Time);
1237 assert_eq!(s.to_physical_repr().i64().unwrap().get(0), Some(7_000_000));
1238 }
1239}