polars_core/series/
mod.rs

1#![allow(unsafe_op_in_unsafe_fn)]
2//! Type agnostic columnar data structure.
3use crate::chunked_array::flags::StatisticsFlags;
4pub use crate::prelude::ChunkCompareEq;
5use crate::prelude::*;
6use crate::{HEAD_DEFAULT_LENGTH, TAIL_DEFAULT_LENGTH};
7
8macro_rules! invalid_operation_panic {
9    ($op:ident, $s:expr) => {
10        panic!(
11            "`{}` operation not supported for dtype `{}`",
12            stringify!($op),
13            $s._dtype()
14        )
15    };
16}
17
18pub mod amortized_iter;
19mod any_value;
20pub mod arithmetic;
21pub mod arrow_export;
22pub mod builder;
23
24mod comparison;
25mod from;
26pub mod implementations;
27pub(crate) mod iterator;
28pub mod ops;
29#[cfg(feature = "proptest")]
30pub mod proptest;
31mod series_trait;
32
33use std::borrow::Cow;
34use std::hash::{Hash, Hasher};
35use std::ops::Deref;
36
37use arrow::compute::aggregate::estimated_bytes_size;
38pub use from::*;
39pub use iterator::{SeriesIter, SeriesPhysIter};
40use num_traits::NumCast;
41use polars_error::feature_gated;
42use polars_utils::float::IsFloat;
43pub use series_trait::{IsSorted, *};
44
45use crate::chunked_array::cast::CastOptions;
46use crate::runtime::RAYON;
47#[cfg(feature = "zip_with")]
48use crate::series::arithmetic::coerce_lhs_rhs;
49use crate::utils::{Wrap, handle_casting_failures, materialize_dyn_int};
50
51/// # Series
52/// The columnar data type for a DataFrame.
53///
54/// Most of the available functions are defined in the [SeriesTrait trait](crate::series::SeriesTrait).
55///
56/// The `Series` struct consists
57/// of typed [ChunkedArray]'s. To quickly cast
58/// a `Series` to a `ChunkedArray` you can call the method with the name of the type:
59///
60/// ```
61/// # use polars_core::prelude::*;
62/// let s: Series = [1, 2, 3].iter().collect();
63/// // Quickly obtain the ChunkedArray wrapped by the Series.
64/// let chunked_array = s.i32().unwrap();
65/// ```
66///
67/// ## Arithmetic
68///
69/// You can do standard arithmetic on series.
70/// ```
71/// # use polars_core::prelude::*;
72/// let s = Series::new("a".into(), [1 , 2, 3]);
73/// let out_add = &s + &s;
74/// let out_sub = &s - &s;
75/// let out_div = &s / &s;
76/// let out_mul = &s * &s;
77/// ```
78///
79/// Or with series and numbers.
80///
81/// ```
82/// # use polars_core::prelude::*;
83/// let s: Series = (1..3).collect();
84/// let out_add_one = &s + 1;
85/// let out_multiply = &s * 10;
86///
87/// // Could not overload left hand side operator.
88/// let out_divide = 1.div(&s);
89/// let out_add = 1.add(&s);
90/// let out_subtract = 1.sub(&s);
91/// let out_multiply = 1.mul(&s);
92/// ```
93///
94/// ## Comparison
95/// You can obtain boolean mask by comparing series.
96///
97/// ```
98/// # use polars_core::prelude::*;
99/// let s = Series::new("dollars".into(), &[1, 2, 3]);
100/// let mask = s.equal(1).unwrap();
101/// let valid = [true, false, false].iter();
102/// assert!(mask
103///     .iter()
104///     .map(|opt_bool| opt_bool.unwrap()) // option, because series can be null
105///     .zip(valid)
106///     .all(|(a, b)| a == *b))
107/// ```
108///
109/// See all the comparison operators in the [ChunkCompareEq trait](crate::chunked_array::ops::ChunkCompareEq) and
110/// [ChunkCompareIneq trait](crate::chunked_array::ops::ChunkCompareIneq).
111///
112/// ## Iterators
113/// The Series variants contain differently typed [ChunkedArray]s.
114/// These structs can be turned into iterators, making it possible to use any function/ closure you want
115/// on a Series.
116///
117/// These iterators return an `Option<T>` because the values of a series may be null.
118///
119/// ```
120/// use polars_core::prelude::*;
121/// let pi = 3.14;
122/// let s = Series::new("angle".into(), [2f32 * pi, pi, 1.5 * pi].as_ref());
123/// let s_cos: Series = s.f32()
124///                     .expect("series was not an f32 dtype")
125///                     .iter()
126///                     .map(|opt_angle| opt_angle.map(|angle| angle.cos()))
127///                     .collect();
128/// ```
129///
130/// ## Creation
131/// Series can be create from different data structures. Below we'll show a few ways we can create
132/// a Series object.
133///
134/// ```
135/// # use polars_core::prelude::*;
136/// // Series can be created from Vec's, slices and arrays
137/// Series::new("boolean series".into(), &[true, false, true]);
138/// Series::new("int series".into(), &[1, 2, 3]);
139/// // And can be nullable
140/// Series::new("got nulls".into(), &[Some(1), None, Some(2)]);
141///
142/// // Series can also be collected from iterators
143/// let from_iter: Series = (0..10)
144///     .into_iter()
145///     .collect();
146///
147/// ```
148#[derive(Clone)]
149#[must_use]
150pub struct Series(pub Arc<dyn SeriesTrait>);
151
152impl PartialEq for Wrap<Series> {
153    fn eq(&self, other: &Self) -> bool {
154        self.0.equals_missing(other)
155    }
156}
157
158impl Eq for Wrap<Series> {}
159
160impl Hash for Wrap<Series> {
161    fn hash<H: Hasher>(&self, state: &mut H) {
162        self.dtype().hash(state);
163        self.len().hash(state);
164
165        for av in self.iter() {
166            av.hash(state);
167        }
168    }
169}
170
171impl Series {
172    /// Create a new empty Series.
173    pub fn new_empty(name: PlSmallStr, dtype: &DataType) -> Series {
174        Series::full_null(name, 0, dtype)
175    }
176
177    pub fn clear(&self) -> Series {
178        if self.is_empty() {
179            self.clone()
180        } else {
181            match self.dtype() {
182                #[cfg(feature = "object")]
183                DataType::Object(_) => self
184                    .take(&ChunkedArray::<IdxType>::new_vec(PlSmallStr::EMPTY, vec![]))
185                    .unwrap(),
186                dt => Series::new_empty(self.name().clone(), dt),
187            }
188        }
189    }
190
191    #[doc(hidden)]
192    pub fn _get_inner_mut(&mut self) -> &mut dyn SeriesTrait {
193        if Arc::weak_count(&self.0) + Arc::strong_count(&self.0) != 1 {
194            self.0 = self.0.clone_inner();
195        }
196        Arc::get_mut(&mut self.0).expect("implementation error")
197    }
198
199    /// Take or clone a owned copy of the inner [`ChunkedArray`].
200    pub fn take_inner<T: PolarsPhysicalType>(self) -> ChunkedArray<T> {
201        let arc_any = self.0.as_arc_any();
202        let downcast = arc_any
203            .downcast::<implementations::SeriesWrap<ChunkedArray<T>>>()
204            .unwrap();
205
206        match Arc::try_unwrap(downcast) {
207            Ok(ca) => ca.0,
208            Err(ca) => ca.as_ref().as_ref().clone(),
209        }
210    }
211
212    /// Returns a reference to the Arrow ArrayRef
213    #[inline]
214    pub fn array_ref(&self, chunk_idx: usize) -> &ArrayRef {
215        &self.chunks()[chunk_idx] as &ArrayRef
216    }
217
218    /// # Safety
219    /// The caller must ensure the length and the data types of `ArrayRef` does not change.
220    /// And that the null_count is updated (e.g. with a `compute_len()`)
221    pub unsafe fn chunks_mut(&mut self) -> &mut Vec<ArrayRef> {
222        #[allow(unused_mut)]
223        let mut ca = self._get_inner_mut();
224        ca.chunks_mut()
225    }
226
227    pub fn into_chunks(mut self) -> Vec<ArrayRef> {
228        let ca = self._get_inner_mut();
229        let chunks = std::mem::take(unsafe { ca.chunks_mut() });
230        ca.compute_len();
231        chunks
232    }
233
234    // TODO! this probably can now be removed, now we don't have special case for structs.
235    pub fn select_chunk(&self, i: usize) -> Self {
236        let mut new = self.clear();
237        let mut flags = self.get_flags();
238
239        use StatisticsFlags as F;
240        flags &= F::IS_SORTED_ANY | F::CAN_FAST_EXPLODE_LIST;
241
242        // Assign mut so we go through arc only once.
243        let mut_new = new._get_inner_mut();
244        let chunks = unsafe { mut_new.chunks_mut() };
245        let chunk = self.chunks()[i].clone();
246        chunks.clear();
247        chunks.push(chunk);
248        mut_new.compute_len();
249        mut_new._set_flags(flags);
250        new
251    }
252
253    pub fn is_sorted_flag(&self) -> IsSorted {
254        if self.len() <= 1 {
255            return IsSorted::Ascending;
256        }
257        self.get_flags().is_sorted()
258    }
259
260    pub fn set_sorted_flag(&mut self, sorted: IsSorted) {
261        let mut flags = self.get_flags();
262        flags.set_sorted(sorted);
263        self.set_flags(flags);
264    }
265
266    pub(crate) fn clear_flags(&mut self) {
267        self.set_flags(StatisticsFlags::empty());
268    }
269    pub fn get_flags(&self) -> StatisticsFlags {
270        self.0._get_flags()
271    }
272
273    pub(crate) fn set_flags(&mut self, flags: StatisticsFlags) {
274        self._get_inner_mut()._set_flags(flags)
275    }
276
277    pub fn into_frame(self) -> DataFrame {
278        // SAFETY: A single-column dataframe cannot have length mismatches or duplicate names
279        unsafe { DataFrame::new_unchecked(self.len(), vec![self.into()]) }
280    }
281
282    /// Rename series.
283    pub fn rename(&mut self, name: PlSmallStr) -> &mut Series {
284        self._get_inner_mut().rename(name);
285        self
286    }
287
288    /// Return this Series with a new name.
289    pub fn with_name(mut self, name: PlSmallStr) -> Series {
290        self.rename(name);
291        self
292    }
293
294    pub fn from_arrow_chunks(name: PlSmallStr, arrays: Vec<ArrayRef>) -> PolarsResult<Series> {
295        Self::try_from((name, arrays))
296    }
297
298    pub fn from_arrow(name: PlSmallStr, array: ArrayRef) -> PolarsResult<Series> {
299        Self::try_from((name, array))
300    }
301
302    /// Shrink the capacity of this array to fit its length.
303    pub fn shrink_to_fit(&mut self) {
304        self._get_inner_mut().shrink_to_fit()
305    }
306
307    /// Append in place. This is done by adding the chunks of `other` to this [`Series`].
308    ///
309    /// See [`ChunkedArray::append`] and [`ChunkedArray::extend`].
310    pub fn append(&mut self, other: &Series) -> PolarsResult<&mut Self> {
311        let must_cast = other.dtype().matches_schema_type(self.dtype())?;
312        if must_cast {
313            let other = other.cast(self.dtype())?;
314            self.append_owned(other)?;
315        } else {
316            self._get_inner_mut().append(other)?;
317        }
318        Ok(self)
319    }
320
321    /// Append in place. This is done by adding the chunks of `other` to this [`Series`].
322    ///
323    /// See [`ChunkedArray::append_owned`] and [`ChunkedArray::extend`].
324    pub fn append_owned(&mut self, other: Series) -> PolarsResult<&mut Self> {
325        let must_cast = other.dtype().matches_schema_type(self.dtype())?;
326        if must_cast {
327            let other = other.cast(self.dtype())?;
328            self._get_inner_mut().append_owned(other)?;
329        } else {
330            self._get_inner_mut().append_owned(other)?;
331        }
332        Ok(self)
333    }
334
335    /// Redo a length and null_count compute
336    pub fn compute_len(&mut self) {
337        self._get_inner_mut().compute_len()
338    }
339
340    /// Extend the memory backed by this array with the values from `other`.
341    ///
342    /// See [`ChunkedArray::extend`] and [`ChunkedArray::append`].
343    pub fn extend(&mut self, other: &Series) -> PolarsResult<&mut Self> {
344        let must_cast = other.dtype().matches_schema_type(self.dtype())?;
345        if must_cast {
346            let other = other.cast(self.dtype())?;
347            self._get_inner_mut().extend(&other)?;
348        } else {
349            self._get_inner_mut().extend(other)?;
350        }
351        Ok(self)
352    }
353
354    /// Sort the series with specific options.
355    ///
356    /// # Example
357    ///
358    /// ```rust
359    /// # use polars_core::prelude::*;
360    /// # fn main() -> PolarsResult<()> {
361    /// let s = Series::new("foo".into(), [2, 1, 3]);
362    /// let sorted = s.sort(SortOptions::default())?;
363    /// assert_eq!(sorted, Series::new("foo".into(), [1, 2, 3]));
364    /// # Ok(())
365    /// }
366    /// ```
367    ///
368    /// See [`SortOptions`] for more options.
369    pub fn sort(&self, sort_options: SortOptions) -> PolarsResult<Self> {
370        self.sort_with(sort_options)
371    }
372
373    /// Only implemented for numeric types
374    pub fn as_single_ptr(&mut self) -> PolarsResult<usize> {
375        self._get_inner_mut().as_single_ptr()
376    }
377
378    pub fn cast(&self, dtype: &DataType) -> PolarsResult<Self> {
379        self.cast_with_options(dtype, CastOptions::NonStrict)
380    }
381
382    /// Cast [`Series`] to another [`DataType`].
383    pub fn cast_with_options(&self, dtype: &DataType, options: CastOptions) -> PolarsResult<Self> {
384        let slf = self
385            .trim_lists_to_normalized_offsets()
386            .map_or(Cow::Borrowed(self), Cow::Owned);
387        let slf = slf.propagate_nulls().map_or(slf, Cow::Owned);
388
389        use DataType as D;
390        let do_clone = match dtype {
391            D::Unknown(UnknownKind::Any) => true,
392            D::Unknown(UnknownKind::Int(_)) if slf.dtype().is_integer() => true,
393            D::Unknown(UnknownKind::Float) if slf.dtype().is_float() => true,
394            D::Unknown(UnknownKind::Str)
395                if slf.dtype().is_string() | slf.dtype().is_categorical() =>
396            {
397                true
398            },
399            dt if (dt.is_primitive() || dt.is_extension()) && dt == slf.dtype() => true,
400            _ => false,
401        };
402
403        if do_clone {
404            return Ok(slf.into_owned());
405        }
406
407        pub fn cast_dtype(dtype: &DataType) -> Option<DataType> {
408            match dtype {
409                D::Unknown(UnknownKind::Int(v)) => Some(materialize_dyn_int(*v).dtype()),
410                D::Unknown(UnknownKind::Float) => Some(DataType::Float64),
411                D::Unknown(UnknownKind::Str) => Some(DataType::String),
412                // Best leave as is.
413                D::List(inner) => cast_dtype(inner.as_ref()).map(Box::new).map(D::List),
414                #[cfg(feature = "dtype-struct")]
415                D::Struct(fields) => {
416                    // @NOTE: We only allocate if we really need to.
417
418                    let mut field_iter = fields.iter().enumerate();
419                    let mut new_fields = loop {
420                        let (i, field) = field_iter.next()?;
421
422                        if let Some(dtype) = cast_dtype(&field.dtype) {
423                            let mut new_fields = Vec::with_capacity(fields.len());
424                            new_fields.extend(fields.iter().take(i).cloned());
425                            new_fields.push(Field {
426                                name: field.name.clone(),
427                                dtype,
428                            });
429                            break new_fields;
430                        }
431                    };
432
433                    new_fields.extend(fields.iter().skip(new_fields.len()).cloned().map(|field| {
434                        let dtype = cast_dtype(&field.dtype).unwrap_or(field.dtype);
435                        Field {
436                            name: field.name,
437                            dtype,
438                        }
439                    }));
440
441                    Some(D::Struct(new_fields))
442                },
443                _ => None,
444            }
445        }
446
447        let mut casted = cast_dtype(dtype);
448        if dtype.is_list() && dtype.inner_dtype().is_some_and(|dt| dt.is_null()) {
449            if let Some(from_inner_dtype) = slf.dtype().inner_dtype() {
450                casted = Some(DataType::List(Box::new(from_inner_dtype.clone())));
451            }
452        }
453        let dtype = match casted {
454            None => dtype,
455            Some(ref dtype) => dtype,
456        };
457
458        // Always allow casting all nulls to other all nulls.
459        let len = slf.len();
460        if slf.null_count() == len {
461            return Ok(Series::full_null(slf.name().clone(), len, dtype));
462        }
463
464        let new_options = match options {
465            // Strictness is handled on this level to improve error messages, if not nested.
466            // Nested types could hide cast errors, so have to be done internally.
467            CastOptions::Strict if !dtype.is_nested() => CastOptions::NonStrict,
468            opt => opt,
469        };
470
471        let out = slf.0.cast(dtype, new_options)?;
472        if options.is_strict() {
473            handle_casting_failures(slf.as_ref(), &out)?;
474        }
475        Ok(out)
476    }
477
478    /// Cast from physical to logical types without any checks on the validity of the cast.
479    ///
480    /// # Safety
481    ///
482    /// This can lead to invalid memory access in downstream code.
483    pub unsafe fn cast_unchecked(&self, dtype: &DataType) -> PolarsResult<Self> {
484        match self.dtype() {
485            #[cfg(feature = "dtype-struct")]
486            DataType::Struct(_) => self.struct_().unwrap().cast_unchecked(dtype),
487            DataType::List(_) => self.list().unwrap().cast_unchecked(dtype),
488            dt if dt.is_primitive_numeric() => {
489                with_match_physical_numeric_polars_type!(dt, |$T| {
490                    let ca: &ChunkedArray<$T> = self.as_ref().as_ref().as_ref();
491                        ca.cast_unchecked(dtype)
492                })
493            },
494            DataType::Binary => self.binary().unwrap().cast_unchecked(dtype),
495            _ => self.cast_with_options(dtype, CastOptions::Overflowing),
496        }
497    }
498
499    /// Convert a non-logical series back into a logical series without casting.
500    ///
501    /// # Safety
502    ///
503    /// This can lead to invalid memory access in downstream code.
504    pub unsafe fn from_physical_unchecked(&self, dtype: &DataType) -> PolarsResult<Self> {
505        debug_assert!(!self.dtype().is_logical(), "{:?}", self.dtype());
506
507        if self.dtype() == dtype {
508            return Ok(self.clone());
509        }
510
511        use DataType as D;
512        match (self.dtype(), dtype) {
513            #[cfg(feature = "dtype-decimal")]
514            (D::Int128, D::Decimal(precision, scale)) => {
515                let ca = self.i128().unwrap();
516                Ok(ca
517                    .clone()
518                    .into_decimal_unchecked(*precision, *scale)
519                    .into_series())
520            },
521
522            #[cfg(feature = "dtype-categorical")]
523            (phys, D::Categorical(cats, _)) if &cats.physical().dtype() == phys => {
524                with_match_categorical_physical_type!(cats.physical(), |$C| {
525                    type CA = ChunkedArray<<$C as PolarsCategoricalType>::PolarsPhysical>;
526                    let ca = self.as_ref().as_any().downcast_ref::<CA>().unwrap();
527                    Ok(CategoricalChunked::<$C>::from_cats_and_dtype_unchecked(
528                        ca.clone(),
529                        dtype.clone(),
530                    )
531                    .into_series())
532                })
533            },
534            #[cfg(feature = "dtype-categorical")]
535            (phys, D::Enum(fcats, _)) if &fcats.physical().dtype() == phys => {
536                with_match_categorical_physical_type!(fcats.physical(), |$C| {
537                    type CA = ChunkedArray<<$C as PolarsCategoricalType>::PolarsPhysical>;
538                    let ca = self.as_ref().as_any().downcast_ref::<CA>().unwrap();
539                    Ok(CategoricalChunked::<$C>::from_cats_and_dtype_unchecked(
540                        ca.clone(),
541                        dtype.clone(),
542                    )
543                    .into_series())
544                })
545            },
546
547            (D::Int32, D::Date) => feature_gated!("dtype-time", Ok(self.clone().into_date())),
548            (D::Int64, D::Datetime(tu, tz)) => feature_gated!(
549                "dtype-datetime",
550                Ok(self.clone().into_datetime(*tu, tz.clone()))
551            ),
552            (D::Int64, D::Duration(tu)) => {
553                feature_gated!("dtype-duration", Ok(self.clone().into_duration(*tu)))
554            },
555            (D::Int64, D::Time) => feature_gated!("dtype-time", Ok(self.clone().into_time())),
556
557            (D::List(_), D::List(to)) => unsafe {
558                self.list()
559                    .unwrap()
560                    .from_physical_unchecked(to.as_ref().clone())
561                    .map(|ca| ca.into_series())
562            },
563            #[cfg(feature = "dtype-array")]
564            (D::Array(_, lw), D::Array(to, rw)) if lw == rw => unsafe {
565                self.array()
566                    .unwrap()
567                    .from_physical_unchecked(to.as_ref().clone())
568                    .map(|ca| ca.into_series())
569            },
570            #[cfg(feature = "dtype-struct")]
571            (D::Struct(_), D::Struct(to)) => unsafe {
572                self.struct_()
573                    .unwrap()
574                    .from_physical_unchecked(to.as_slice())
575                    .map(|ca| ca.into_series())
576            },
577
578            #[cfg(feature = "dtype-extension")]
579            (_, D::Extension(typ, storage)) => {
580                let storage_series = self.from_physical_unchecked(storage.as_ref())?;
581                let ext = ExtensionChunked::from_storage(typ.clone(), storage_series);
582                Ok(ext.into_series())
583            },
584
585            _ => panic!("invalid from_physical({dtype:?}) for {:?}", self.dtype()),
586        }
587    }
588
589    #[cfg(feature = "dtype-extension")]
590    pub fn into_extension(self, typ: ExtensionTypeInstance) -> Series {
591        assert!(!self.dtype().is_extension());
592        let ext = ExtensionChunked::from_storage(typ, self);
593        ext.into_series()
594    }
595
596    /// Cast numerical types to f64, and keep floats as is.
597    pub fn to_float(&self) -> PolarsResult<Series> {
598        match self.dtype() {
599            DataType::Float32 | DataType::Float64 => Ok(self.clone()),
600            _ => self.cast_with_options(&DataType::Float64, CastOptions::Overflowing),
601        }
602    }
603
604    /// Get the sum of the Series as a `Scalar`.
605    /// Returns a `Scalar` with a zeroed value if self is an empty numeric series.
606    ///
607    /// If the [`DataType`] is one of `{Int8, UInt8, Int16, UInt16}` the sum is
608    /// computed in an `Int64` accumulator and the result is returned as `Int64`
609    /// to prevent overflow issues.
610    pub fn sum<T>(&self) -> PolarsResult<T>
611    where
612        T: NumCast + IsFloat,
613    {
614        let sum = self.sum_reduce()?;
615        let sum = sum.value().extract().unwrap();
616        Ok(sum)
617    }
618
619    /// Returns the minimum value in the array, according to the natural order.
620    /// Returns an option because the array is nullable.
621    pub fn min<T>(&self) -> PolarsResult<Option<T>>
622    where
623        T: NumCast + IsFloat,
624    {
625        let min = self.min_reduce()?;
626        let min = min.value().extract::<T>();
627        Ok(min)
628    }
629
630    /// Returns the maximum value in the array, according to the natural order.
631    /// Returns an option because the array is nullable.
632    pub fn max<T>(&self) -> PolarsResult<Option<T>>
633    where
634        T: NumCast + IsFloat,
635    {
636        let max = self.max_reduce()?;
637        let max = max.value().extract::<T>();
638        Ok(max)
639    }
640
641    /// Explode a list Series. This expands every item to a new row..
642    pub fn explode(&self, options: ExplodeOptions) -> PolarsResult<Series> {
643        match self.dtype() {
644            DataType::List(_) => self.list().unwrap().explode(options),
645            #[cfg(feature = "dtype-array")]
646            DataType::Array(_, _) => self.array().unwrap().explode(options),
647            _ => Ok(self.clone()),
648        }
649    }
650
651    /// Check if numeric value is NaN (note this is different than missing/ null)
652    pub fn is_nan(&self) -> PolarsResult<BooleanChunked> {
653        match self.dtype() {
654            #[cfg(feature = "dtype-f16")]
655            DataType::Float16 => Ok(self.f16().unwrap().is_nan()),
656            DataType::Float32 => Ok(self.f32().unwrap().is_nan()),
657            DataType::Float64 => Ok(self.f64().unwrap().is_nan()),
658            DataType::Null => Ok(BooleanChunked::full_null(self.name().clone(), self.len())),
659            dt if dt.is_primitive_numeric() => {
660                let arr = BooleanArray::full(self.len(), false, ArrowDataType::Boolean)
661                    .with_validity(self.rechunk_validity());
662                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
663            },
664            _ => polars_bail!(opq = is_nan, self.dtype()),
665        }
666    }
667
668    /// Check if numeric value is NaN (note this is different than missing/null)
669    pub fn is_not_nan(&self) -> PolarsResult<BooleanChunked> {
670        match self.dtype() {
671            #[cfg(feature = "dtype-f16")]
672            DataType::Float16 => Ok(self.f16().unwrap().is_not_nan()),
673            DataType::Float32 => Ok(self.f32().unwrap().is_not_nan()),
674            DataType::Float64 => Ok(self.f64().unwrap().is_not_nan()),
675            dt if dt.is_primitive_numeric() => {
676                let arr = BooleanArray::full(self.len(), true, ArrowDataType::Boolean)
677                    .with_validity(self.rechunk_validity());
678                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
679            },
680            _ => polars_bail!(opq = is_not_nan, self.dtype()),
681        }
682    }
683
684    /// Check if numeric value is finite
685    pub fn is_finite(&self) -> PolarsResult<BooleanChunked> {
686        match self.dtype() {
687            #[cfg(feature = "dtype-f16")]
688            DataType::Float16 => Ok(self.f16().unwrap().is_finite()),
689            DataType::Float32 => Ok(self.f32().unwrap().is_finite()),
690            DataType::Float64 => Ok(self.f64().unwrap().is_finite()),
691            DataType::Null => Ok(BooleanChunked::full_null(self.name().clone(), self.len())),
692            dt if dt.is_primitive_numeric() => {
693                let arr = BooleanArray::full(self.len(), true, ArrowDataType::Boolean)
694                    .with_validity(self.rechunk_validity());
695                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
696            },
697            _ => polars_bail!(opq = is_finite, self.dtype()),
698        }
699    }
700
701    /// Check if numeric value is infinite
702    pub fn is_infinite(&self) -> PolarsResult<BooleanChunked> {
703        match self.dtype() {
704            #[cfg(feature = "dtype-f16")]
705            DataType::Float16 => Ok(self.f16().unwrap().is_infinite()),
706            DataType::Float32 => Ok(self.f32().unwrap().is_infinite()),
707            DataType::Float64 => Ok(self.f64().unwrap().is_infinite()),
708            DataType::Null => Ok(BooleanChunked::full_null(self.name().clone(), self.len())),
709            dt if dt.is_primitive_numeric() => {
710                let arr = BooleanArray::full(self.len(), false, ArrowDataType::Boolean)
711                    .with_validity(self.rechunk_validity());
712                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
713            },
714            _ => polars_bail!(opq = is_infinite, self.dtype()),
715        }
716    }
717
718    /// Create a new ChunkedArray with values from self where the mask evaluates `true` and values
719    /// from `other` where the mask evaluates `false`. This function automatically broadcasts unit
720    /// length inputs.
721    #[cfg(feature = "zip_with")]
722    pub fn zip_with(&self, mask: &BooleanChunked, other: &Series) -> PolarsResult<Series> {
723        let (lhs, rhs) = coerce_lhs_rhs(self, other)?;
724        lhs.zip_with_same_type(mask, rhs.as_ref())
725    }
726
727    /// Converts a Series to their physical representation, if they have one,
728    /// otherwise the series is left unchanged.
729    ///
730    /// * Date -> Int32
731    /// * Datetime -> Int64
732    /// * Duration -> Int64
733    /// * Decimal -> Int128
734    /// * Time -> Int64
735    /// * Categorical -> U8/U16/U32
736    /// * List(inner) -> List(physical of inner)
737    /// * Array(inner) -> Array(physical of inner)
738    /// * Struct -> Struct with physical repr of each struct column
739    /// * Extension -> physical of storage type
740    pub fn to_physical_repr(&self) -> Cow<'_, Series> {
741        use DataType::*;
742        match self.dtype() {
743            // NOTE: Don't use cast here, as it might rechunk (if all nulls)
744            // which is not allowed in a phys repr.
745            #[cfg(feature = "dtype-date")]
746            Date => Cow::Owned(self.date().unwrap().phys.clone().into_series()),
747            #[cfg(feature = "dtype-datetime")]
748            Datetime(_, _) => Cow::Owned(self.datetime().unwrap().phys.clone().into_series()),
749            #[cfg(feature = "dtype-duration")]
750            Duration(_) => Cow::Owned(self.duration().unwrap().phys.clone().into_series()),
751            #[cfg(feature = "dtype-time")]
752            Time => Cow::Owned(self.time().unwrap().phys.clone().into_series()),
753            #[cfg(feature = "dtype-categorical")]
754            dt @ (Categorical(_, _) | Enum(_, _)) => {
755                with_match_categorical_physical_type!(dt.cat_physical().unwrap(), |$C| {
756                    let ca = self.cat::<$C>().unwrap();
757                    Cow::Owned(ca.physical().clone().into_series())
758                })
759            },
760            #[cfg(feature = "dtype-decimal")]
761            Decimal(_, _) => Cow::Owned(self.decimal().unwrap().phys.clone().into_series()),
762            List(_) => match self.list().unwrap().to_physical_repr() {
763                Cow::Borrowed(_) => Cow::Borrowed(self),
764                Cow::Owned(ca) => Cow::Owned(ca.into_series()),
765            },
766            #[cfg(feature = "dtype-array")]
767            Array(_, _) => match self.array().unwrap().to_physical_repr() {
768                Cow::Borrowed(_) => Cow::Borrowed(self),
769                Cow::Owned(ca) => Cow::Owned(ca.into_series()),
770            },
771            #[cfg(feature = "dtype-struct")]
772            Struct(_) => match self.struct_().unwrap().to_physical_repr() {
773                Cow::Borrowed(_) => Cow::Borrowed(self),
774                Cow::Owned(ca) => Cow::Owned(ca.into_series()),
775            },
776            #[cfg(feature = "dtype-extension")]
777            Extension(_, _) => self.ext().unwrap().storage().to_physical_repr(),
778            _ => Cow::Borrowed(self),
779        }
780    }
781
782    /// If the Series is an Extension type, return its storage Series.
783    /// Otherwise, return itself.
784    pub fn to_storage(&self) -> &Series {
785        #[cfg(feature = "dtype-extension")]
786        {
787            if let DataType::Extension(_, _) = self.dtype() {
788                return self.ext().unwrap().storage();
789            }
790        }
791        self
792    }
793
794    /// Traverse and collect every nth element in a new array.
795    pub fn gather_every(&self, n: usize, offset: usize) -> PolarsResult<Series> {
796        polars_ensure!(n > 0, ComputeError: "cannot perform gather every for `n=0`");
797        let idx = ((offset as IdxSize)..self.len() as IdxSize)
798            .step_by(n)
799            .collect_ca(PlSmallStr::EMPTY);
800        // SAFETY: we stay in-bounds.
801        Ok(unsafe { self.take_unchecked(&idx) })
802    }
803
804    #[cfg(feature = "dot_product")]
805    pub fn dot(&self, other: &Series) -> PolarsResult<f64> {
806        std::ops::Mul::mul(self, other)?.sum::<f64>()
807    }
808
809    /// Get the sum of the [`ChunkedArray`] as a `Scalar`.
810    /// Returns a `Scalar` with a single zeroed value if self is an empty numeric series.
811    ///
812    /// If the [`DataType`] is one of `{Int8, UInt8, Int16, UInt16}` the sum is
813    /// computed in an `Int64` accumulator and the result is returned as `Int64`
814    /// to prevent overflow issues.
815    pub fn sum_reduce(&self) -> PolarsResult<Scalar> {
816        self.0.sum_reduce()
817    }
818
819    /// Get the mean of the Series as a new Series of length 1.
820    /// Returns a Series with a single null entry if self is an empty numeric series.
821    pub fn mean_reduce(&self) -> PolarsResult<Scalar> {
822        self.0.mean_reduce()
823    }
824
825    /// Get the product of an array.
826    ///
827    /// If the [`DataType`] is one of `{Int8, UInt8, Int16, UInt16}` the `Series` is
828    /// first cast to `Int64` to prevent overflow issues.
829    pub fn product(&self) -> PolarsResult<Scalar> {
830        #[cfg(feature = "product")]
831        {
832            use DataType::*;
833            match self.dtype() {
834                Boolean => self.cast(&DataType::Int64).unwrap().product(),
835                Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 => {
836                    let s = self.cast(&Int64).unwrap();
837                    s.product()
838                },
839                Int64 => Ok(self.i64().unwrap().prod_reduce()),
840                UInt64 => Ok(self.u64().unwrap().prod_reduce()),
841                #[cfg(feature = "dtype-i128")]
842                Int128 => Ok(self.i128().unwrap().prod_reduce()),
843                #[cfg(feature = "dtype-u128")]
844                UInt128 => Ok(self.u128().unwrap().prod_reduce()),
845                #[cfg(feature = "dtype-f16")]
846                Float16 => Ok(self.f16().unwrap().prod_reduce()),
847                Float32 => Ok(self.f32().unwrap().prod_reduce()),
848                Float64 => Ok(self.f64().unwrap().prod_reduce()),
849                #[cfg(feature = "dtype-decimal")]
850                Decimal(..) => Ok(self.decimal().unwrap().prod_reduce()),
851                dt => {
852                    polars_bail!(InvalidOperation: "`product` operation not supported for dtype `{dt}`")
853                },
854            }
855        }
856        #[cfg(not(feature = "product"))]
857        {
858            panic!("activate 'product' feature")
859        }
860    }
861
862    /// Cast throws an error if conversion had overflows
863    pub fn strict_cast(&self, dtype: &DataType) -> PolarsResult<Series> {
864        self.cast_with_options(dtype, CastOptions::Strict)
865    }
866
867    #[cfg(feature = "dtype-decimal")]
868    pub fn into_decimal(self, precision: usize, scale: usize) -> PolarsResult<Series> {
869        match self.dtype() {
870            DataType::Int128 => Ok(self
871                .i128()
872                .unwrap()
873                .clone()
874                .into_decimal(precision, scale)?
875                .into_series()),
876            DataType::Decimal(cur_prec, cur_scale)
877                if scale == *cur_scale && precision >= *cur_prec =>
878            {
879                Ok(self)
880            },
881            dt => panic!("into_decimal({precision:?}, {scale}) not implemented for {dt:?}"),
882        }
883    }
884
885    #[cfg(feature = "dtype-time")]
886    pub fn into_time(self) -> Series {
887        match self.dtype() {
888            DataType::Int64 => self.i64().unwrap().clone().into_time().into_series(),
889            DataType::Time => self
890                .time()
891                .unwrap()
892                .physical()
893                .clone()
894                .into_time()
895                .into_series(),
896            dt => panic!("date not implemented for {dt:?}"),
897        }
898    }
899
900    pub fn into_date(self) -> Series {
901        #[cfg(not(feature = "dtype-date"))]
902        {
903            panic!("activate feature dtype-date")
904        }
905        #[cfg(feature = "dtype-date")]
906        match self.dtype() {
907            DataType::Int32 => self.i32().unwrap().clone().into_date().into_series(),
908            DataType::Date => self
909                .date()
910                .unwrap()
911                .physical()
912                .clone()
913                .into_date()
914                .into_series(),
915            dt => panic!("date not implemented for {dt:?}"),
916        }
917    }
918
919    #[allow(unused_variables)]
920    pub fn into_datetime(self, timeunit: TimeUnit, tz: Option<TimeZone>) -> Series {
921        #[cfg(not(feature = "dtype-datetime"))]
922        {
923            panic!("activate feature dtype-datetime")
924        }
925
926        #[cfg(feature = "dtype-datetime")]
927        match self.dtype() {
928            DataType::Int64 => self
929                .i64()
930                .unwrap()
931                .clone()
932                .into_datetime(timeunit, tz)
933                .into_series(),
934            DataType::Datetime(_, _) => self
935                .datetime()
936                .unwrap()
937                .physical()
938                .clone()
939                .into_datetime(timeunit, tz)
940                .into_series(),
941            dt => panic!("into_datetime not implemented for {dt:?}"),
942        }
943    }
944
945    #[allow(unused_variables)]
946    pub fn into_duration(self, timeunit: TimeUnit) -> Series {
947        #[cfg(not(feature = "dtype-duration"))]
948        {
949            panic!("activate feature dtype-duration")
950        }
951        #[cfg(feature = "dtype-duration")]
952        match self.dtype() {
953            DataType::Int64 => self
954                .i64()
955                .unwrap()
956                .clone()
957                .into_duration(timeunit)
958                .into_series(),
959            DataType::Duration(_) => self
960                .duration()
961                .unwrap()
962                .physical()
963                .clone()
964                .into_duration(timeunit)
965                .into_series(),
966            dt => panic!("into_duration not implemented for {dt:?}"),
967        }
968    }
969
970    // used for formatting
971    pub fn str_value(&self, index: usize) -> PolarsResult<Cow<'_, str>> {
972        Ok(self.0.get(index)?.str_value())
973    }
974    /// Get the head of the Series.
975    pub fn head(&self, length: Option<usize>) -> Series {
976        let len = length.unwrap_or(HEAD_DEFAULT_LENGTH);
977        self.slice(0, std::cmp::min(len, self.len()))
978    }
979
980    /// Get the tail of the Series.
981    pub fn tail(&self, length: Option<usize>) -> Series {
982        let len = length.unwrap_or(TAIL_DEFAULT_LENGTH);
983        let len = std::cmp::min(len, self.len());
984        self.slice(-(len as i64), len)
985    }
986
987    /// Compute the unique elements, but maintain order. This requires more work
988    /// than a naive [`Series::unique`](SeriesTrait::unique).
989    pub fn unique_stable(&self) -> PolarsResult<Series> {
990        let idx = self.arg_unique()?;
991        // SAFETY: Indices are in bounds.
992        unsafe { Ok(self.take_unchecked(&idx)) }
993    }
994
995    pub fn try_idx(&self) -> Option<&IdxCa> {
996        #[cfg(feature = "bigidx")]
997        {
998            self.try_u64()
999        }
1000        #[cfg(not(feature = "bigidx"))]
1001        {
1002            self.try_u32()
1003        }
1004    }
1005
1006    pub fn idx(&self) -> PolarsResult<&IdxCa> {
1007        #[cfg(feature = "bigidx")]
1008        {
1009            self.u64()
1010        }
1011        #[cfg(not(feature = "bigidx"))]
1012        {
1013            self.u32()
1014        }
1015    }
1016
1017    /// Returns an estimation of the total (heap) allocated size of the `Series` in bytes.
1018    ///
1019    /// # Implementation
1020    /// This estimation is the sum of the size of its buffers, validity, including nested arrays.
1021    /// Multiple arrays may share buffers and bitmaps. Therefore, the size of 2 arrays is not the
1022    /// sum of the sizes computed from this function. In particular, [`StructArray`]'s size is an upper bound.
1023    ///
1024    /// When an array is sliced, its allocated size remains constant because the buffer unchanged.
1025    /// However, this function will yield a smaller number. This is because this function returns
1026    /// the visible size of the buffer, not its total capacity.
1027    ///
1028    /// FFI buffers are included in this estimation.
1029    pub fn estimated_size(&self) -> usize {
1030        let mut size = 0;
1031        match self.dtype() {
1032            // TODO @ cat-rework: include mapping size here?
1033            #[cfg(feature = "object")]
1034            DataType::Object(_) => {
1035                let ArrowDataType::FixedSizeBinary(size) = self.chunks()[0].dtype() else {
1036                    unreachable!()
1037                };
1038                // This is only the pointer size in python. So will be a huge underestimation.
1039                return self.len() * *size;
1040            },
1041            _ => {},
1042        }
1043
1044        size += self
1045            .chunks()
1046            .iter()
1047            .map(|arr| estimated_bytes_size(&**arr))
1048            .sum::<usize>();
1049
1050        size
1051    }
1052
1053    pub fn row_encode_unordered(&self) -> PolarsResult<BinaryOffsetChunked> {
1054        row_encode::_get_rows_encoded_ca_unordered(
1055            self.name().clone(),
1056            &[self.clone().into_column()],
1057        )
1058    }
1059
1060    pub fn row_encode_ordered(
1061        &self,
1062        descending: bool,
1063        nulls_last: bool,
1064    ) -> PolarsResult<BinaryOffsetChunked> {
1065        row_encode::_get_rows_encoded_ca(
1066            self.name().clone(),
1067            &[self.clone().into_column()],
1068            &[descending],
1069            &[nulls_last],
1070            false,
1071        )
1072    }
1073}
1074
1075impl Default for Series {
1076    fn default() -> Self {
1077        NullChunked::new(PlSmallStr::EMPTY, 0).into_series()
1078    }
1079}
1080
1081impl Deref for Series {
1082    type Target = dyn SeriesTrait;
1083
1084    fn deref(&self) -> &Self::Target {
1085        self.0.as_ref()
1086    }
1087}
1088
1089impl<'a> AsRef<dyn SeriesTrait + 'a> for Series {
1090    fn as_ref(&self) -> &(dyn SeriesTrait + 'a) {
1091        self.0.as_ref()
1092    }
1093}
1094
1095impl<T: PolarsPhysicalType> AsRef<ChunkedArray<T>> for dyn SeriesTrait + '_ {
1096    fn as_ref(&self) -> &ChunkedArray<T> {
1097        // @NOTE: SeriesTrait `as_any` returns a std::any::Any for the underlying ChunkedArray /
1098        // Logical (so not the SeriesWrap).
1099        let Some(ca) = self.as_any().downcast_ref::<ChunkedArray<T>>() else {
1100            panic!(
1101                "implementation error, cannot get ref {:?} from {:?}",
1102                T::get_static_dtype(),
1103                self.dtype()
1104            );
1105        };
1106
1107        ca
1108    }
1109}
1110
1111impl<T: PolarsPhysicalType> AsMut<ChunkedArray<T>> for dyn SeriesTrait + '_ {
1112    fn as_mut(&mut self) -> &mut ChunkedArray<T> {
1113        if !self.as_any_mut().is::<ChunkedArray<T>>() {
1114            panic!(
1115                "implementation error, cannot get ref {:?} from {:?}",
1116                T::get_static_dtype(),
1117                self.dtype()
1118            );
1119        }
1120
1121        // @NOTE: SeriesTrait `as_any` returns a std::any::Any for the underlying ChunkedArray /
1122        // Logical (so not the SeriesWrap).
1123        self.as_any_mut().downcast_mut::<ChunkedArray<T>>().unwrap()
1124    }
1125}
1126
1127#[cfg(test)]
1128mod test {
1129    use crate::prelude::*;
1130    use crate::series::*;
1131
1132    #[test]
1133    fn cast() {
1134        let ar = UInt32Chunked::new("a".into(), &[1, 2]);
1135        let s = ar.into_series();
1136        let s2 = s.cast(&DataType::Int64).unwrap();
1137
1138        assert!(s2.i64().is_ok());
1139        let s2 = s.cast(&DataType::Float32).unwrap();
1140        assert!(s2.f32().is_ok());
1141    }
1142
1143    #[test]
1144    fn new_series() {
1145        let _ = Series::new("boolean series".into(), &vec![true, false, true]);
1146        let _ = Series::new("int series".into(), &[1, 2, 3]);
1147        let ca = Int32Chunked::new("a".into(), &[1, 2, 3]);
1148        let _ = ca.into_series();
1149    }
1150
1151    #[test]
1152    #[cfg(feature = "dtype-date")]
1153    fn roundtrip_list_logical_20311() {
1154        let list = ListChunked::from_chunk_iter(
1155            PlSmallStr::from_static("a"),
1156            [ListArray::new(
1157                ArrowDataType::LargeList(Box::new(ArrowField::new(
1158                    LIST_VALUES_NAME,
1159                    ArrowDataType::Int32,
1160                    true,
1161                ))),
1162                unsafe { arrow::offset::Offsets::new_unchecked(vec![0, 1]) }.into(),
1163                PrimitiveArray::new(ArrowDataType::Int32, vec![1i32].into(), None).to_boxed(),
1164                None,
1165            )],
1166        );
1167        let list = unsafe { list.from_physical_unchecked(DataType::Date) }.unwrap();
1168        assert_eq!(list.dtype(), &DataType::List(Box::new(DataType::Date)));
1169    }
1170
1171    #[test]
1172    #[cfg(feature = "dtype-struct")]
1173    fn new_series_from_empty_structs() {
1174        let dtype = DataType::Struct(vec![]);
1175        let empties = vec![AnyValue::StructOwned(Box::new((vec![], vec![]))); 3];
1176        let s = Series::from_any_values_and_dtype("".into(), &empties, &dtype, false).unwrap();
1177        assert_eq!(s.len(), 3);
1178    }
1179    #[test]
1180    fn new_series_from_arrow_primitive_array() {
1181        let array = UInt32Array::from_slice([1, 2, 3, 4, 5]);
1182        let array_ref: ArrayRef = Box::new(array);
1183
1184        let _ = Series::try_new("foo".into(), array_ref).unwrap();
1185    }
1186
1187    #[test]
1188    fn series_append() {
1189        let mut s1 = Series::new("a".into(), &[1, 2]);
1190        let s2 = Series::new("b".into(), &[3]);
1191        s1.append(&s2).unwrap();
1192        assert_eq!(s1.len(), 3);
1193
1194        // add wrong type
1195        let s2 = Series::new("b".into(), &[3.0]);
1196        assert!(s1.append(&s2).is_err())
1197    }
1198
1199    #[test]
1200    #[cfg(feature = "dtype-decimal")]
1201    fn series_append_decimal() {
1202        let s1 = Series::new("a".into(), &[1.1, 2.3])
1203            .cast(&DataType::Decimal(38, 2))
1204            .unwrap();
1205        let s2 = Series::new("b".into(), &[3])
1206            .cast(&DataType::Decimal(38, 0))
1207            .unwrap();
1208
1209        {
1210            let mut s1 = s1.clone();
1211            s1.append(&s2).unwrap();
1212            assert_eq!(s1.len(), 3);
1213            assert_eq!(s1.get(2).unwrap(), AnyValue::Decimal(300, 38, 2));
1214        }
1215
1216        {
1217            let mut s2 = s2;
1218            s2.extend(&s1).unwrap();
1219            assert_eq!(s2.get(2).unwrap(), AnyValue::Decimal(2, 38, 0));
1220        }
1221    }
1222
1223    #[test]
1224    fn series_slice_works() {
1225        let series = Series::new("a".into(), &[1i64, 2, 3, 4, 5]);
1226
1227        let slice_1 = series.slice(-3, 3);
1228        let slice_2 = series.slice(-5, 5);
1229        let slice_3 = series.slice(0, 5);
1230
1231        assert_eq!(slice_1.get(0).unwrap(), AnyValue::Int64(3));
1232        assert_eq!(slice_2.get(0).unwrap(), AnyValue::Int64(1));
1233        assert_eq!(slice_3.get(0).unwrap(), AnyValue::Int64(1));
1234    }
1235
1236    #[test]
1237    fn out_of_range_slice_does_not_panic() {
1238        let series = Series::new("a".into(), &[1i64, 2, 3, 4, 5]);
1239
1240        let _ = series.slice(-3, 4);
1241        let _ = series.slice(-6, 2);
1242        let _ = series.slice(4, 2);
1243    }
1244}
polars_core/series/mod.rs

polars_core/series/
mod.rs
