polars_core/series/
mod.rs

1#![allow(unsafe_op_in_unsafe_fn)]
2//! Type agnostic columnar data structure.
3use crate::chunked_array::flags::StatisticsFlags;
4pub use crate::prelude::ChunkCompareEq;
5use crate::prelude::*;
6use crate::{HEAD_DEFAULT_LENGTH, TAIL_DEFAULT_LENGTH};
7
8macro_rules! invalid_operation_panic {
9    ($op:ident, $s:expr) => {
10        panic!(
11            "`{}` operation not supported for dtype `{}`",
12            stringify!($op),
13            $s._dtype()
14        )
15    };
16}
17
18pub mod amortized_iter;
19mod any_value;
20pub mod arithmetic;
21pub mod builder;
22mod comparison;
23mod from;
24pub mod implementations;
25mod into;
26pub(crate) mod iterator;
27pub mod ops;
28mod series_trait;
29
30use std::borrow::Cow;
31use std::hash::{Hash, Hasher};
32use std::ops::Deref;
33
34use arrow::compute::aggregate::estimated_bytes_size;
35use arrow::offset::Offsets;
36pub use from::*;
37pub use iterator::{SeriesIter, SeriesPhysIter};
38use num_traits::NumCast;
39use polars_error::feature_gated;
40pub use series_trait::{IsSorted, *};
41
42use crate::POOL;
43use crate::chunked_array::cast::CastOptions;
44#[cfg(feature = "zip_with")]
45use crate::series::arithmetic::coerce_lhs_rhs;
46use crate::utils::{Wrap, handle_casting_failures, materialize_dyn_int};
47
48/// # Series
49/// The columnar data type for a DataFrame.
50///
51/// Most of the available functions are defined in the [SeriesTrait trait](crate::series::SeriesTrait).
52///
53/// The `Series` struct consists
54/// of typed [ChunkedArray]'s. To quickly cast
55/// a `Series` to a `ChunkedArray` you can call the method with the name of the type:
56///
57/// ```
58/// # use polars_core::prelude::*;
59/// let s: Series = [1, 2, 3].iter().collect();
60/// // Quickly obtain the ChunkedArray wrapped by the Series.
61/// let chunked_array = s.i32().unwrap();
62/// ```
63///
64/// ## Arithmetic
65///
66/// You can do standard arithmetic on series.
67/// ```
68/// # use polars_core::prelude::*;
69/// let s = Series::new("a".into(), [1 , 2, 3]);
70/// let out_add = &s + &s;
71/// let out_sub = &s - &s;
72/// let out_div = &s / &s;
73/// let out_mul = &s * &s;
74/// ```
75///
76/// Or with series and numbers.
77///
78/// ```
79/// # use polars_core::prelude::*;
80/// let s: Series = (1..3).collect();
81/// let out_add_one = &s + 1;
82/// let out_multiply = &s * 10;
83///
84/// // Could not overload left hand side operator.
85/// let out_divide = 1.div(&s);
86/// let out_add = 1.add(&s);
87/// let out_subtract = 1.sub(&s);
88/// let out_multiply = 1.mul(&s);
89/// ```
90///
91/// ## Comparison
92/// You can obtain boolean mask by comparing series.
93///
94/// ```
95/// # use polars_core::prelude::*;
96/// let s = Series::new("dollars".into(), &[1, 2, 3]);
97/// let mask = s.equal(1).unwrap();
98/// let valid = [true, false, false].iter();
99/// assert!(mask
100///     .into_iter()
101///     .map(|opt_bool| opt_bool.unwrap()) // option, because series can be null
102///     .zip(valid)
103///     .all(|(a, b)| a == *b))
104/// ```
105///
106/// See all the comparison operators in the [ChunkCompareEq trait](crate::chunked_array::ops::ChunkCompareEq) and
107/// [ChunkCompareIneq trait](crate::chunked_array::ops::ChunkCompareIneq).
108///
109/// ## Iterators
110/// The Series variants contain differently typed [ChunkedArray]s.
111/// These structs can be turned into iterators, making it possible to use any function/ closure you want
112/// on a Series.
113///
114/// These iterators return an `Option<T>` because the values of a series may be null.
115///
116/// ```
117/// use polars_core::prelude::*;
118/// let pi = 3.14;
119/// let s = Series::new("angle".into(), [2f32 * pi, pi, 1.5 * pi].as_ref());
120/// let s_cos: Series = s.f32()
121///                     .expect("series was not an f32 dtype")
122///                     .into_iter()
123///                     .map(|opt_angle| opt_angle.map(|angle| angle.cos()))
124///                     .collect();
125/// ```
126///
127/// ## Creation
128/// Series can be create from different data structures. Below we'll show a few ways we can create
129/// a Series object.
130///
131/// ```
132/// # use polars_core::prelude::*;
133/// // Series can be created from Vec's, slices and arrays
134/// Series::new("boolean series".into(), &[true, false, true]);
135/// Series::new("int series".into(), &[1, 2, 3]);
136/// // And can be nullable
137/// Series::new("got nulls".into(), &[Some(1), None, Some(2)]);
138///
139/// // Series can also be collected from iterators
140/// let from_iter: Series = (0..10)
141///     .into_iter()
142///     .collect();
143///
144/// ```
145#[derive(Clone)]
146#[must_use]
147pub struct Series(pub Arc<dyn SeriesTrait>);
148
149impl PartialEq for Wrap<Series> {
150    fn eq(&self, other: &Self) -> bool {
151        self.0.equals_missing(other)
152    }
153}
154
155impl Eq for Wrap<Series> {}
156
157impl Hash for Wrap<Series> {
158    fn hash<H: Hasher>(&self, state: &mut H) {
159        let rs = PlSeedableRandomStateQuality::fixed();
160        let mut h = vec![];
161        if self.0.vec_hash(rs, &mut h).is_ok() {
162            let h = h.into_iter().fold(0, |a: u64, b| a.wrapping_add(b));
163            h.hash(state)
164        } else {
165            self.len().hash(state);
166            self.null_count().hash(state);
167            self.dtype().hash(state);
168        }
169    }
170}
171
172impl Series {
173    /// Create a new empty Series.
174    pub fn new_empty(name: PlSmallStr, dtype: &DataType) -> Series {
175        Series::full_null(name, 0, dtype)
176    }
177
178    pub fn clear(&self) -> Series {
179        if self.is_empty() {
180            self.clone()
181        } else {
182            match self.dtype() {
183                #[cfg(feature = "object")]
184                DataType::Object(_) => self
185                    .take(&ChunkedArray::<IdxType>::new_vec(PlSmallStr::EMPTY, vec![]))
186                    .unwrap(),
187                dt => Series::new_empty(self.name().clone(), dt),
188            }
189        }
190    }
191
192    #[doc(hidden)]
193    pub fn _get_inner_mut(&mut self) -> &mut dyn SeriesTrait {
194        if Arc::weak_count(&self.0) + Arc::strong_count(&self.0) != 1 {
195            self.0 = self.0.clone_inner();
196        }
197        Arc::get_mut(&mut self.0).expect("implementation error")
198    }
199
200    /// Take or clone a owned copy of the inner [`ChunkedArray`].
201    pub fn take_inner<T>(self) -> ChunkedArray<T>
202    where
203        T: 'static + PolarsDataType<IsLogical = FalseT>,
204    {
205        let arc_any = self.0.as_arc_any();
206        let downcast = arc_any
207            .downcast::<implementations::SeriesWrap<ChunkedArray<T>>>()
208            .unwrap();
209
210        match Arc::try_unwrap(downcast) {
211            Ok(ca) => ca.0,
212            Err(ca) => ca.as_ref().as_ref().clone(),
213        }
214    }
215
216    /// # Safety
217    /// The caller must ensure the length and the data types of `ArrayRef` does not change.
218    /// And that the null_count is updated (e.g. with a `compute_len()`)
219    pub unsafe fn chunks_mut(&mut self) -> &mut Vec<ArrayRef> {
220        #[allow(unused_mut)]
221        let mut ca = self._get_inner_mut();
222        ca.chunks_mut()
223    }
224
225    pub fn into_chunks(mut self) -> Vec<ArrayRef> {
226        let ca = self._get_inner_mut();
227        let chunks = std::mem::take(unsafe { ca.chunks_mut() });
228        ca.compute_len();
229        chunks
230    }
231
232    // TODO! this probably can now be removed, now we don't have special case for structs.
233    pub fn select_chunk(&self, i: usize) -> Self {
234        let mut new = self.clear();
235        let mut flags = self.get_flags();
236
237        use StatisticsFlags as F;
238        flags &= F::IS_SORTED_ANY | F::CAN_FAST_EXPLODE_LIST;
239
240        // Assign mut so we go through arc only once.
241        let mut_new = new._get_inner_mut();
242        let chunks = unsafe { mut_new.chunks_mut() };
243        let chunk = self.chunks()[i].clone();
244        chunks.clear();
245        chunks.push(chunk);
246        mut_new.compute_len();
247        mut_new._set_flags(flags);
248        new
249    }
250
251    pub fn is_sorted_flag(&self) -> IsSorted {
252        if self.len() <= 1 {
253            return IsSorted::Ascending;
254        }
255        self.get_flags().is_sorted()
256    }
257
258    pub fn set_sorted_flag(&mut self, sorted: IsSorted) {
259        let mut flags = self.get_flags();
260        flags.set_sorted(sorted);
261        self.set_flags(flags);
262    }
263
264    pub(crate) fn clear_flags(&mut self) {
265        self.set_flags(StatisticsFlags::empty());
266    }
267    pub fn get_flags(&self) -> StatisticsFlags {
268        self.0._get_flags()
269    }
270
271    pub(crate) fn set_flags(&mut self, flags: StatisticsFlags) {
272        self._get_inner_mut()._set_flags(flags)
273    }
274
275    pub fn into_frame(self) -> DataFrame {
276        // SAFETY: A single-column dataframe cannot have length mismatches or duplicate names
277        unsafe { DataFrame::new_no_checks(self.len(), vec![self.into()]) }
278    }
279
280    /// Rename series.
281    pub fn rename(&mut self, name: PlSmallStr) -> &mut Series {
282        self._get_inner_mut().rename(name);
283        self
284    }
285
286    /// Return this Series with a new name.
287    pub fn with_name(mut self, name: PlSmallStr) -> Series {
288        self.rename(name);
289        self
290    }
291
292    pub fn from_arrow_chunks(name: PlSmallStr, arrays: Vec<ArrayRef>) -> PolarsResult<Series> {
293        Self::try_from((name, arrays))
294    }
295
296    pub fn from_arrow(name: PlSmallStr, array: ArrayRef) -> PolarsResult<Series> {
297        Self::try_from((name, array))
298    }
299
300    /// Shrink the capacity of this array to fit its length.
301    pub fn shrink_to_fit(&mut self) {
302        self._get_inner_mut().shrink_to_fit()
303    }
304
305    /// Append in place. This is done by adding the chunks of `other` to this [`Series`].
306    ///
307    /// See [`ChunkedArray::append`] and [`ChunkedArray::extend`].
308    pub fn append(&mut self, other: &Series) -> PolarsResult<&mut Self> {
309        let must_cast = other.dtype().matches_schema_type(self.dtype())?;
310        if must_cast {
311            let other = other.cast(self.dtype())?;
312            self.append_owned(other)?;
313        } else {
314            self._get_inner_mut().append(other)?;
315        }
316        Ok(self)
317    }
318
319    /// Append in place. This is done by adding the chunks of `other` to this [`Series`].
320    ///
321    /// See [`ChunkedArray::append_owned`] and [`ChunkedArray::extend`].
322    pub fn append_owned(&mut self, other: Series) -> PolarsResult<&mut Self> {
323        let must_cast = other.dtype().matches_schema_type(self.dtype())?;
324        if must_cast {
325            let other = other.cast(self.dtype())?;
326            self._get_inner_mut().append_owned(other)?;
327        } else {
328            self._get_inner_mut().append_owned(other)?;
329        }
330        Ok(self)
331    }
332
333    /// Redo a length and null_count compute
334    pub fn compute_len(&mut self) {
335        self._get_inner_mut().compute_len()
336    }
337
338    /// Extend the memory backed by this array with the values from `other`.
339    ///
340    /// See [`ChunkedArray::extend`] and [`ChunkedArray::append`].
341    pub fn extend(&mut self, other: &Series) -> PolarsResult<&mut Self> {
342        let must_cast = other.dtype().matches_schema_type(self.dtype())?;
343        if must_cast {
344            let other = other.cast(self.dtype())?;
345            self._get_inner_mut().extend(&other)?;
346        } else {
347            self._get_inner_mut().extend(other)?;
348        }
349        Ok(self)
350    }
351
352    /// Sort the series with specific options.
353    ///
354    /// # Example
355    ///
356    /// ```rust
357    /// # use polars_core::prelude::*;
358    /// # fn main() -> PolarsResult<()> {
359    /// let s = Series::new("foo".into(), [2, 1, 3]);
360    /// let sorted = s.sort(SortOptions::default())?;
361    /// assert_eq!(sorted, Series::new("foo".into(), [1, 2, 3]));
362    /// # Ok(())
363    /// }
364    /// ```
365    ///
366    /// See [`SortOptions`] for more options.
367    pub fn sort(&self, sort_options: SortOptions) -> PolarsResult<Self> {
368        self.sort_with(sort_options)
369    }
370
371    /// Only implemented for numeric types
372    pub fn as_single_ptr(&mut self) -> PolarsResult<usize> {
373        self._get_inner_mut().as_single_ptr()
374    }
375
376    pub fn cast(&self, dtype: &DataType) -> PolarsResult<Self> {
377        self.cast_with_options(dtype, CastOptions::NonStrict)
378    }
379
380    /// Cast [`Series`] to another [`DataType`].
381    pub fn cast_with_options(&self, dtype: &DataType, options: CastOptions) -> PolarsResult<Self> {
382        let slf = self
383            .trim_lists_to_normalized_offsets()
384            .map_or(Cow::Borrowed(self), Cow::Owned);
385        let slf = slf.propagate_nulls().map_or(slf, Cow::Owned);
386
387        use DataType as D;
388        let do_clone = match dtype {
389            D::Unknown(UnknownKind::Any) => true,
390            D::Unknown(UnknownKind::Int(_)) if slf.dtype().is_integer() => true,
391            D::Unknown(UnknownKind::Float) if slf.dtype().is_float() => true,
392            D::Unknown(UnknownKind::Str)
393                if slf.dtype().is_string() | slf.dtype().is_categorical() =>
394            {
395                true
396            },
397            dt if dt.is_primitive() && dt == slf.dtype() => true,
398            #[cfg(feature = "dtype-categorical")]
399            D::Enum(None, _) => {
400                polars_bail!(InvalidOperation: "cannot cast / initialize Enum without categories present");
401            },
402            _ => false,
403        };
404
405        if do_clone {
406            return Ok(slf.into_owned());
407        }
408
409        pub fn cast_dtype(dtype: &DataType) -> Option<DataType> {
410            match dtype {
411                D::Unknown(UnknownKind::Int(v)) => Some(materialize_dyn_int(*v).dtype()),
412                D::Unknown(UnknownKind::Float) => Some(DataType::Float64),
413                D::Unknown(UnknownKind::Str) => Some(DataType::String),
414                // Best leave as is.
415                D::List(inner) => cast_dtype(inner.as_ref()).map(Box::new).map(D::List),
416                #[cfg(feature = "dtype-struct")]
417                D::Struct(fields) => {
418                    // @NOTE: We only allocate if we really need to.
419
420                    let mut field_iter = fields.iter().enumerate();
421                    let mut new_fields = loop {
422                        let (i, field) = field_iter.next()?;
423
424                        if let Some(dtype) = cast_dtype(&field.dtype) {
425                            let mut new_fields = Vec::with_capacity(fields.len());
426                            new_fields.extend(fields.iter().take(i).cloned());
427                            new_fields.push(Field {
428                                name: field.name.clone(),
429                                dtype,
430                            });
431                            break new_fields;
432                        }
433                    };
434
435                    new_fields.extend(fields.iter().skip(new_fields.len()).cloned().map(|field| {
436                        let dtype = cast_dtype(&field.dtype).unwrap_or(field.dtype);
437                        Field {
438                            name: field.name.clone(),
439                            dtype,
440                        }
441                    }));
442
443                    Some(D::Struct(new_fields))
444                },
445                _ => None,
446            }
447        }
448
449        let mut casted = cast_dtype(dtype);
450        if dtype.is_list() && dtype.inner_dtype().is_some_and(|dt| dt.is_null()) {
451            if let Some(from_inner_dtype) = slf.dtype().inner_dtype() {
452                casted = Some(DataType::List(Box::new(from_inner_dtype.clone())));
453            }
454        }
455        let dtype = match casted {
456            None => dtype,
457            Some(ref dtype) => dtype,
458        };
459
460        // Always allow casting all nulls to other all nulls.
461        let len = slf.len();
462        if slf.null_count() == len {
463            return Ok(Series::full_null(slf.name().clone(), len, dtype));
464        }
465
466        let new_options = match options {
467            // Strictness is handled on this level to improve error messages.
468            CastOptions::Strict => CastOptions::NonStrict,
469            opt => opt,
470        };
471
472        let out = slf.0.cast(dtype, new_options)?;
473        if options.is_strict() {
474            handle_casting_failures(slf.as_ref(), &out)?;
475        }
476        Ok(out)
477    }
478
479    /// Cast from physical to logical types without any checks on the validity of the cast.
480    ///
481    /// # Safety
482    ///
483    /// This can lead to invalid memory access in downstream code.
484    pub unsafe fn cast_unchecked(&self, dtype: &DataType) -> PolarsResult<Self> {
485        match self.dtype() {
486            #[cfg(feature = "dtype-struct")]
487            DataType::Struct(_) => self.struct_().unwrap().cast_unchecked(dtype),
488            DataType::List(_) => self.list().unwrap().cast_unchecked(dtype),
489            dt if dt.is_primitive_numeric() => {
490                with_match_physical_numeric_polars_type!(dt, |$T| {
491                    let ca: &ChunkedArray<$T> = self.as_ref().as_ref().as_ref();
492                        ca.cast_unchecked(dtype)
493                })
494            },
495            DataType::Binary => self.binary().unwrap().cast_unchecked(dtype),
496            _ => self.cast_with_options(dtype, CastOptions::Overflowing),
497        }
498    }
499
500    /// Convert a non-logical series back into a logical series without casting.
501    ///
502    /// # Safety
503    ///
504    /// This can lead to invalid memory access in downstream code.
505    pub unsafe fn from_physical_unchecked(&self, dtype: &DataType) -> PolarsResult<Self> {
506        debug_assert!(!self.dtype().is_logical());
507
508        if self.dtype() == dtype {
509            return Ok(self.clone());
510        }
511
512        use DataType as D;
513        match (self.dtype(), dtype) {
514            #[cfg(feature = "dtype-decimal")]
515            (D::Int128, D::Decimal(precision, scale)) => {
516                self.clone().into_decimal(*precision, scale.unwrap())
517            },
518
519            #[cfg(feature = "dtype-categorical")]
520            (D::UInt32, D::Categorical(revmap, ordering)) => match revmap {
521                Some(revmap) => Ok(unsafe {
522                    CategoricalChunked::from_cats_and_rev_map_unchecked(
523                        self.u32().unwrap().clone(),
524                        revmap.clone(),
525                        false,
526                        *ordering,
527                    )
528                }
529                .into_series()),
530                // In the streaming engine this is `None` and the global string cache is turned on
531                // for the duration of the query.
532                None => Ok(unsafe {
533                    CategoricalChunked::from_global_indices_unchecked(
534                        self.u32().unwrap().clone(),
535                        *ordering,
536                    )
537                    .into_series()
538                }),
539            },
540            #[cfg(feature = "dtype-categorical")]
541            (D::UInt32, D::Enum(revmap, ordering)) => Ok(unsafe {
542                CategoricalChunked::from_cats_and_rev_map_unchecked(
543                    self.u32().unwrap().clone(),
544                    revmap.as_ref().unwrap().clone(),
545                    true,
546                    *ordering,
547                )
548            }
549            .into_series()),
550
551            (D::Int32, D::Date) => feature_gated!("dtype-time", Ok(self.clone().into_date())),
552            (D::Int64, D::Datetime(tu, tz)) => feature_gated!(
553                "dtype-datetime",
554                Ok(self.clone().into_datetime(*tu, tz.clone()))
555            ),
556            (D::Int64, D::Duration(tu)) => {
557                feature_gated!("dtype-duration", Ok(self.clone().into_duration(*tu)))
558            },
559            (D::Int64, D::Time) => feature_gated!("dtype-time", Ok(self.clone().into_time())),
560
561            (D::List(_), D::List(to)) => unsafe {
562                self.list()
563                    .unwrap()
564                    .from_physical_unchecked(to.as_ref().clone())
565                    .map(|ca| ca.into_series())
566            },
567            #[cfg(feature = "dtype-array")]
568            (D::Array(_, lw), D::Array(to, rw)) if lw == rw => unsafe {
569                self.array()
570                    .unwrap()
571                    .from_physical_unchecked(to.as_ref().clone())
572                    .map(|ca| ca.into_series())
573            },
574            #[cfg(feature = "dtype-struct")]
575            (D::Struct(_), D::Struct(to)) => unsafe {
576                self.struct_()
577                    .unwrap()
578                    .from_physical_unchecked(to.as_slice())
579                    .map(|ca| ca.into_series())
580            },
581
582            _ => panic!("invalid from_physical({dtype:?}) for {:?}", self.dtype()),
583        }
584    }
585
586    /// Cast numerical types to f64, and keep floats as is.
587    pub fn to_float(&self) -> PolarsResult<Series> {
588        match self.dtype() {
589            DataType::Float32 | DataType::Float64 => Ok(self.clone()),
590            _ => self.cast_with_options(&DataType::Float64, CastOptions::Overflowing),
591        }
592    }
593
594    /// Compute the sum of all values in this Series.
595    /// Returns `Some(0)` if the array is empty, and `None` if the array only
596    /// contains null values.
597    ///
598    /// If the [`DataType`] is one of `{Int8, UInt8, Int16, UInt16}` the `Series` is
599    /// first cast to `Int64` to prevent overflow issues.
600    pub fn sum<T>(&self) -> PolarsResult<T>
601    where
602        T: NumCast,
603    {
604        let sum = self.sum_reduce()?;
605        let sum = sum.value().extract().unwrap();
606        Ok(sum)
607    }
608
609    /// Returns the minimum value in the array, according to the natural order.
610    /// Returns an option because the array is nullable.
611    pub fn min<T>(&self) -> PolarsResult<Option<T>>
612    where
613        T: NumCast,
614    {
615        let min = self.min_reduce()?;
616        let min = min.value().extract::<T>();
617        Ok(min)
618    }
619
620    /// Returns the maximum value in the array, according to the natural order.
621    /// Returns an option because the array is nullable.
622    pub fn max<T>(&self) -> PolarsResult<Option<T>>
623    where
624        T: NumCast,
625    {
626        let max = self.max_reduce()?;
627        let max = max.value().extract::<T>();
628        Ok(max)
629    }
630
631    /// Explode a list Series. This expands every item to a new row..
632    pub fn explode(&self, skip_empty: bool) -> PolarsResult<Series> {
633        match self.dtype() {
634            DataType::List(_) => self.list().unwrap().explode(skip_empty),
635            #[cfg(feature = "dtype-array")]
636            DataType::Array(_, _) => self.array().unwrap().explode(skip_empty),
637            _ => Ok(self.clone()),
638        }
639    }
640
641    /// Check if numeric value is NaN (note this is different than missing/ null)
642    pub fn is_nan(&self) -> PolarsResult<BooleanChunked> {
643        match self.dtype() {
644            DataType::Float32 => Ok(self.f32().unwrap().is_nan()),
645            DataType::Float64 => Ok(self.f64().unwrap().is_nan()),
646            DataType::Null => Ok(BooleanChunked::full_null(self.name().clone(), self.len())),
647            dt if dt.is_primitive_numeric() => {
648                let arr = BooleanArray::full(self.len(), false, ArrowDataType::Boolean)
649                    .with_validity(self.rechunk_validity());
650                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
651            },
652            _ => polars_bail!(opq = is_nan, self.dtype()),
653        }
654    }
655
656    /// Check if numeric value is NaN (note this is different than missing/null)
657    pub fn is_not_nan(&self) -> PolarsResult<BooleanChunked> {
658        match self.dtype() {
659            DataType::Float32 => Ok(self.f32().unwrap().is_not_nan()),
660            DataType::Float64 => Ok(self.f64().unwrap().is_not_nan()),
661            dt if dt.is_primitive_numeric() => {
662                let arr = BooleanArray::full(self.len(), true, ArrowDataType::Boolean)
663                    .with_validity(self.rechunk_validity());
664                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
665            },
666            _ => polars_bail!(opq = is_not_nan, self.dtype()),
667        }
668    }
669
670    /// Check if numeric value is finite
671    pub fn is_finite(&self) -> PolarsResult<BooleanChunked> {
672        match self.dtype() {
673            DataType::Float32 => Ok(self.f32().unwrap().is_finite()),
674            DataType::Float64 => Ok(self.f64().unwrap().is_finite()),
675            DataType::Null => Ok(BooleanChunked::full_null(self.name().clone(), self.len())),
676            dt if dt.is_primitive_numeric() => {
677                let arr = BooleanArray::full(self.len(), true, ArrowDataType::Boolean)
678                    .with_validity(self.rechunk_validity());
679                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
680            },
681            _ => polars_bail!(opq = is_finite, self.dtype()),
682        }
683    }
684
685    /// Check if numeric value is infinite
686    pub fn is_infinite(&self) -> PolarsResult<BooleanChunked> {
687        match self.dtype() {
688            DataType::Float32 => Ok(self.f32().unwrap().is_infinite()),
689            DataType::Float64 => Ok(self.f64().unwrap().is_infinite()),
690            DataType::Null => Ok(BooleanChunked::full_null(self.name().clone(), self.len())),
691            dt if dt.is_primitive_numeric() => {
692                let arr = BooleanArray::full(self.len(), false, ArrowDataType::Boolean)
693                    .with_validity(self.rechunk_validity());
694                Ok(BooleanChunked::with_chunk(self.name().clone(), arr))
695            },
696            _ => polars_bail!(opq = is_infinite, self.dtype()),
697        }
698    }
699
700    /// Create a new ChunkedArray with values from self where the mask evaluates `true` and values
701    /// from `other` where the mask evaluates `false`. This function automatically broadcasts unit
702    /// length inputs.
703    #[cfg(feature = "zip_with")]
704    pub fn zip_with(&self, mask: &BooleanChunked, other: &Series) -> PolarsResult<Series> {
705        let (lhs, rhs) = coerce_lhs_rhs(self, other)?;
706        lhs.zip_with_same_type(mask, rhs.as_ref())
707    }
708
709    /// Converts a Series to their physical representation, if they have one,
710    /// otherwise the series is left unchanged.
711    ///
712    /// * Date -> Int32
713    /// * Datetime -> Int64
714    /// * Duration -> Int64
715    /// * Decimal -> Int128
716    /// * Time -> Int64
717    /// * Categorical -> UInt32
718    /// * List(inner) -> List(physical of inner)
719    /// * Array(inner) -> Array(physical of inner)
720    /// * Struct -> Struct with physical repr of each struct column
721    pub fn to_physical_repr(&self) -> Cow<Series> {
722        use DataType::*;
723        match self.dtype() {
724            // NOTE: Don't use cast here, as it might rechunk (if all nulls)
725            // which is not allowed in a phys repr.
726            #[cfg(feature = "dtype-date")]
727            Date => Cow::Owned(self.date().unwrap().phys.clone().into_series()),
728            #[cfg(feature = "dtype-datetime")]
729            Datetime(_, _) => Cow::Owned(self.datetime().unwrap().phys.clone().into_series()),
730            #[cfg(feature = "dtype-duration")]
731            Duration(_) => Cow::Owned(self.duration().unwrap().phys.clone().into_series()),
732            #[cfg(feature = "dtype-time")]
733            Time => Cow::Owned(self.time().unwrap().phys.clone().into_series()),
734            #[cfg(feature = "dtype-categorical")]
735            Categorical(_, _) | Enum(_, _) => {
736                let ca = self.categorical().unwrap();
737                Cow::Owned(ca.physical().clone().into_series())
738            },
739            #[cfg(feature = "dtype-decimal")]
740            Decimal(_, _) => Cow::Owned(self.decimal().unwrap().phys.clone().into_series()),
741            List(_) => match self.list().unwrap().to_physical_repr() {
742                Cow::Borrowed(_) => Cow::Borrowed(self),
743                Cow::Owned(ca) => Cow::Owned(ca.into_series()),
744            },
745            #[cfg(feature = "dtype-array")]
746            Array(_, _) => match self.array().unwrap().to_physical_repr() {
747                Cow::Borrowed(_) => Cow::Borrowed(self),
748                Cow::Owned(ca) => Cow::Owned(ca.into_series()),
749            },
750            #[cfg(feature = "dtype-struct")]
751            Struct(_) => match self.struct_().unwrap().to_physical_repr() {
752                Cow::Borrowed(_) => Cow::Borrowed(self),
753                Cow::Owned(ca) => Cow::Owned(ca.into_series()),
754            },
755            _ => Cow::Borrowed(self),
756        }
757    }
758
759    /// Traverse and collect every nth element in a new array.
760    pub fn gather_every(&self, n: usize, offset: usize) -> PolarsResult<Series> {
761        polars_ensure!(n > 0, ComputeError: "cannot perform gather every for `n=0`");
762        let idx = ((offset as IdxSize)..self.len() as IdxSize)
763            .step_by(n)
764            .collect_ca(PlSmallStr::EMPTY);
765        // SAFETY: we stay in-bounds.
766        Ok(unsafe { self.take_unchecked(&idx) })
767    }
768
769    #[cfg(feature = "dot_product")]
770    pub fn dot(&self, other: &Series) -> PolarsResult<f64> {
771        std::ops::Mul::mul(self, other)?.sum::<f64>()
772    }
773
774    /// Get the sum of the Series as a new Series of length 1.
775    /// Returns a Series with a single zeroed entry if self is an empty numeric series.
776    ///
777    /// If the [`DataType`] is one of `{Int8, UInt8, Int16, UInt16}` the `Series` is
778    /// first cast to `Int64` to prevent overflow issues.
779    pub fn sum_reduce(&self) -> PolarsResult<Scalar> {
780        use DataType::*;
781        match self.dtype() {
782            Int8 | UInt8 | Int16 | UInt16 => self.cast(&Int64).unwrap().sum_reduce(),
783            _ => self.0.sum_reduce(),
784        }
785    }
786
787    /// Get the product of an array.
788    ///
789    /// If the [`DataType`] is one of `{Int8, UInt8, Int16, UInt16}` the `Series` is
790    /// first cast to `Int64` to prevent overflow issues.
791    pub fn product(&self) -> PolarsResult<Scalar> {
792        #[cfg(feature = "product")]
793        {
794            use DataType::*;
795            match self.dtype() {
796                Boolean => self.cast(&DataType::Int64).unwrap().product(),
797                Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 => {
798                    let s = self.cast(&Int64).unwrap();
799                    s.product()
800                },
801                Int64 => Ok(self.i64().unwrap().prod_reduce()),
802                UInt64 => Ok(self.u64().unwrap().prod_reduce()),
803                #[cfg(feature = "dtype-i128")]
804                Int128 => Ok(self.i128().unwrap().prod_reduce()),
805                Float32 => Ok(self.f32().unwrap().prod_reduce()),
806                Float64 => Ok(self.f64().unwrap().prod_reduce()),
807                dt => {
808                    polars_bail!(InvalidOperation: "`product` operation not supported for dtype `{dt}`")
809                },
810            }
811        }
812        #[cfg(not(feature = "product"))]
813        {
814            panic!("activate 'product' feature")
815        }
816    }
817
818    /// Cast throws an error if conversion had overflows
819    pub fn strict_cast(&self, dtype: &DataType) -> PolarsResult<Series> {
820        self.cast_with_options(dtype, CastOptions::Strict)
821    }
822
823    #[cfg(feature = "dtype-decimal")]
824    pub(crate) fn into_decimal(
825        self,
826        precision: Option<usize>,
827        scale: usize,
828    ) -> PolarsResult<Series> {
829        match self.dtype() {
830            DataType::Int128 => Ok(self
831                .i128()
832                .unwrap()
833                .clone()
834                .into_decimal(precision, scale)?
835                .into_series()),
836            DataType::Decimal(cur_prec, cur_scale)
837                if (cur_prec.is_none() || precision.is_none() || *cur_prec == precision)
838                    && *cur_scale == Some(scale) =>
839            {
840                Ok(self)
841            },
842            dt => panic!("into_decimal({precision:?}, {scale}) not implemented for {dt:?}"),
843        }
844    }
845
846    #[cfg(feature = "dtype-time")]
847    pub(crate) fn into_time(self) -> Series {
848        match self.dtype() {
849            DataType::Int64 => self.i64().unwrap().clone().into_time().into_series(),
850            DataType::Time => self
851                .time()
852                .unwrap()
853                .as_ref()
854                .clone()
855                .into_time()
856                .into_series(),
857            dt => panic!("date not implemented for {dt:?}"),
858        }
859    }
860
861    pub(crate) fn into_date(self) -> Series {
862        #[cfg(not(feature = "dtype-date"))]
863        {
864            panic!("activate feature dtype-date")
865        }
866        #[cfg(feature = "dtype-date")]
867        match self.dtype() {
868            DataType::Int32 => self.i32().unwrap().clone().into_date().into_series(),
869            DataType::Date => self
870                .date()
871                .unwrap()
872                .as_ref()
873                .clone()
874                .into_date()
875                .into_series(),
876            dt => panic!("date not implemented for {dt:?}"),
877        }
878    }
879
880    #[allow(unused_variables)]
881    pub(crate) fn into_datetime(self, timeunit: TimeUnit, tz: Option<TimeZone>) -> Series {
882        #[cfg(not(feature = "dtype-datetime"))]
883        {
884            panic!("activate feature dtype-datetime")
885        }
886
887        #[cfg(feature = "dtype-datetime")]
888        match self.dtype() {
889            DataType::Int64 => self
890                .i64()
891                .unwrap()
892                .clone()
893                .into_datetime(timeunit, tz)
894                .into_series(),
895            DataType::Datetime(_, _) => self
896                .datetime()
897                .unwrap()
898                .as_ref()
899                .clone()
900                .into_datetime(timeunit, tz)
901                .into_series(),
902            dt => panic!("into_datetime not implemented for {dt:?}"),
903        }
904    }
905
906    #[allow(unused_variables)]
907    pub(crate) fn into_duration(self, timeunit: TimeUnit) -> Series {
908        #[cfg(not(feature = "dtype-duration"))]
909        {
910            panic!("activate feature dtype-duration")
911        }
912        #[cfg(feature = "dtype-duration")]
913        match self.dtype() {
914            DataType::Int64 => self
915                .i64()
916                .unwrap()
917                .clone()
918                .into_duration(timeunit)
919                .into_series(),
920            DataType::Duration(_) => self
921                .duration()
922                .unwrap()
923                .as_ref()
924                .clone()
925                .into_duration(timeunit)
926                .into_series(),
927            dt => panic!("into_duration not implemented for {dt:?}"),
928        }
929    }
930
931    // used for formatting
932    pub fn str_value(&self, index: usize) -> PolarsResult<Cow<str>> {
933        Ok(self.0.get(index)?.str_value())
934    }
935    /// Get the head of the Series.
936    pub fn head(&self, length: Option<usize>) -> Series {
937        let len = length.unwrap_or(HEAD_DEFAULT_LENGTH);
938        self.slice(0, std::cmp::min(len, self.len()))
939    }
940
941    /// Get the tail of the Series.
942    pub fn tail(&self, length: Option<usize>) -> Series {
943        let len = length.unwrap_or(TAIL_DEFAULT_LENGTH);
944        let len = std::cmp::min(len, self.len());
945        self.slice(-(len as i64), len)
946    }
947
948    pub fn mean_reduce(&self) -> Scalar {
949        crate::scalar::reduce::mean_reduce(self.mean(), self.dtype().clone())
950    }
951
952    /// Compute the unique elements, but maintain order. This requires more work
953    /// than a naive [`Series::unique`](SeriesTrait::unique).
954    pub fn unique_stable(&self) -> PolarsResult<Series> {
955        let idx = self.arg_unique()?;
956        // SAFETY: Indices are in bounds.
957        unsafe { Ok(self.take_unchecked(&idx)) }
958    }
959
960    pub fn try_idx(&self) -> Option<&IdxCa> {
961        #[cfg(feature = "bigidx")]
962        {
963            self.try_u64()
964        }
965        #[cfg(not(feature = "bigidx"))]
966        {
967            self.try_u32()
968        }
969    }
970
971    pub fn idx(&self) -> PolarsResult<&IdxCa> {
972        #[cfg(feature = "bigidx")]
973        {
974            self.u64()
975        }
976        #[cfg(not(feature = "bigidx"))]
977        {
978            self.u32()
979        }
980    }
981
982    /// Returns an estimation of the total (heap) allocated size of the `Series` in bytes.
983    ///
984    /// # Implementation
985    /// This estimation is the sum of the size of its buffers, validity, including nested arrays.
986    /// Multiple arrays may share buffers and bitmaps. Therefore, the size of 2 arrays is not the
987    /// sum of the sizes computed from this function. In particular, [`StructArray`]'s size is an upper bound.
988    ///
989    /// When an array is sliced, its allocated size remains constant because the buffer unchanged.
990    /// However, this function will yield a smaller number. This is because this function returns
991    /// the visible size of the buffer, not its total capacity.
992    ///
993    /// FFI buffers are included in this estimation.
994    pub fn estimated_size(&self) -> usize {
995        let mut size = 0;
996        match self.dtype() {
997            #[cfg(feature = "dtype-categorical")]
998            DataType::Categorical(Some(rv), _) | DataType::Enum(Some(rv), _) => match &**rv {
999                RevMapping::Local(arr, _) => size += estimated_bytes_size(arr),
1000                RevMapping::Global(map, arr, _) => {
1001                    size += map.capacity() * size_of::<u32>() * 2 + estimated_bytes_size(arr);
1002                },
1003            },
1004            #[cfg(feature = "object")]
1005            DataType::Object(_) => {
1006                let ArrowDataType::FixedSizeBinary(size) = self.chunks()[0].dtype() else {
1007                    unreachable!()
1008                };
1009                // This is only the pointer size in python. So will be a huge underestimation.
1010                return self.len() * *size;
1011            },
1012            _ => {},
1013        }
1014
1015        size += self
1016            .chunks()
1017            .iter()
1018            .map(|arr| estimated_bytes_size(&**arr))
1019            .sum::<usize>();
1020
1021        size
1022    }
1023
1024    /// Packs every element into a list.
1025    pub fn as_list(&self) -> ListChunked {
1026        let s = self.rechunk();
1027        // don't  use `to_arrow` as we need the physical types
1028        let values = s.chunks()[0].clone();
1029        let offsets = (0i64..(s.len() as i64 + 1)).collect::<Vec<_>>();
1030        let offsets = unsafe { Offsets::new_unchecked(offsets) };
1031
1032        let dtype = LargeListArray::default_datatype(
1033            s.dtype().to_physical().to_arrow(CompatLevel::newest()),
1034        );
1035        let new_arr = LargeListArray::new(dtype, offsets.into(), values, None);
1036        let mut out = ListChunked::with_chunk(s.name().clone(), new_arr);
1037        out.set_inner_dtype(s.dtype().clone());
1038        out
1039    }
1040}
1041
1042impl Deref for Series {
1043    type Target = dyn SeriesTrait;
1044
1045    fn deref(&self) -> &Self::Target {
1046        self.0.as_ref()
1047    }
1048}
1049
1050impl<'a> AsRef<(dyn SeriesTrait + 'a)> for Series {
1051    fn as_ref(&self) -> &(dyn SeriesTrait + 'a) {
1052        self.0.as_ref()
1053    }
1054}
1055
1056impl Default for Series {
1057    fn default() -> Self {
1058        Int64Chunked::default().into_series()
1059    }
1060}
1061
1062impl<T> AsRef<ChunkedArray<T>> for dyn SeriesTrait + '_
1063where
1064    T: 'static + PolarsDataType<IsLogical = FalseT>,
1065{
1066    fn as_ref(&self) -> &ChunkedArray<T> {
1067        // @NOTE: SeriesTrait `as_any` returns a std::any::Any for the underlying ChunkedArray /
1068        // Logical (so not the SeriesWrap).
1069        let Some(ca) = self.as_any().downcast_ref::<ChunkedArray<T>>() else {
1070            panic!(
1071                "implementation error, cannot get ref {:?} from {:?}",
1072                T::get_static_dtype(),
1073                self.dtype()
1074            );
1075        };
1076
1077        ca
1078    }
1079}
1080
1081impl<T> AsMut<ChunkedArray<T>> for dyn SeriesTrait + '_
1082where
1083    T: 'static + PolarsDataType<IsLogical = FalseT>,
1084{
1085    fn as_mut(&mut self) -> &mut ChunkedArray<T> {
1086        if !self.as_any_mut().is::<ChunkedArray<T>>() {
1087            panic!(
1088                "implementation error, cannot get ref {:?} from {:?}",
1089                T::get_static_dtype(),
1090                self.dtype()
1091            );
1092        }
1093
1094        // @NOTE: SeriesTrait `as_any` returns a std::any::Any for the underlying ChunkedArray /
1095        // Logical (so not the SeriesWrap).
1096        self.as_any_mut().downcast_mut::<ChunkedArray<T>>().unwrap()
1097    }
1098}
1099
1100#[cfg(test)]
1101mod test {
1102    use crate::prelude::*;
1103    use crate::series::*;
1104
1105    #[test]
1106    fn cast() {
1107        let ar = UInt32Chunked::new("a".into(), &[1, 2]);
1108        let s = ar.into_series();
1109        let s2 = s.cast(&DataType::Int64).unwrap();
1110
1111        assert!(s2.i64().is_ok());
1112        let s2 = s.cast(&DataType::Float32).unwrap();
1113        assert!(s2.f32().is_ok());
1114    }
1115
1116    #[test]
1117    fn new_series() {
1118        let _ = Series::new("boolean series".into(), &vec![true, false, true]);
1119        let _ = Series::new("int series".into(), &[1, 2, 3]);
1120        let ca = Int32Chunked::new("a".into(), &[1, 2, 3]);
1121        let _ = ca.into_series();
1122    }
1123
1124    #[test]
1125    #[cfg(feature = "dtype-date")]
1126    fn roundtrip_list_logical_20311() {
1127        let list = ListChunked::from_chunk_iter(
1128            PlSmallStr::from_static("a"),
1129            [ListArray::new(
1130                ArrowDataType::LargeList(Box::new(ArrowField::new(
1131                    PlSmallStr::from_static("item"),
1132                    ArrowDataType::Int32,
1133                    true,
1134                ))),
1135                unsafe { Offsets::new_unchecked(vec![0, 1]) }.into(),
1136                PrimitiveArray::new(ArrowDataType::Int32, vec![1i32].into(), None).to_boxed(),
1137                None,
1138            )],
1139        );
1140        let list = unsafe { list.from_physical_unchecked(DataType::Date) }.unwrap();
1141        assert_eq!(list.dtype(), &DataType::List(Box::new(DataType::Date)));
1142    }
1143
1144    #[test]
1145    #[cfg(feature = "dtype-struct")]
1146    fn new_series_from_empty_structs() {
1147        let dtype = DataType::Struct(vec![]);
1148        let empties = vec![AnyValue::StructOwned(Box::new((vec![], vec![]))); 3];
1149        let s = Series::from_any_values_and_dtype("".into(), &empties, &dtype, false).unwrap();
1150        assert_eq!(s.len(), 3);
1151    }
1152    #[test]
1153    fn new_series_from_arrow_primitive_array() {
1154        let array = UInt32Array::from_slice([1, 2, 3, 4, 5]);
1155        let array_ref: ArrayRef = Box::new(array);
1156
1157        let _ = Series::try_new("foo".into(), array_ref).unwrap();
1158    }
1159
1160    #[test]
1161    fn series_append() {
1162        let mut s1 = Series::new("a".into(), &[1, 2]);
1163        let s2 = Series::new("b".into(), &[3]);
1164        s1.append(&s2).unwrap();
1165        assert_eq!(s1.len(), 3);
1166
1167        // add wrong type
1168        let s2 = Series::new("b".into(), &[3.0]);
1169        assert!(s1.append(&s2).is_err())
1170    }
1171
1172    #[test]
1173    #[cfg(feature = "dtype-decimal")]
1174    fn series_append_decimal() {
1175        let s1 = Series::new("a".into(), &[1.1, 2.3])
1176            .cast(&DataType::Decimal(None, Some(2)))
1177            .unwrap();
1178        let s2 = Series::new("b".into(), &[3])
1179            .cast(&DataType::Decimal(None, Some(0)))
1180            .unwrap();
1181
1182        {
1183            let mut s1 = s1.clone();
1184            s1.append(&s2).unwrap();
1185            assert_eq!(s1.len(), 3);
1186            assert_eq!(s1.get(2).unwrap(), AnyValue::Decimal(300, 2));
1187        }
1188
1189        {
1190            let mut s2 = s2.clone();
1191            s2.extend(&s1).unwrap();
1192            assert_eq!(s2.get(2).unwrap(), AnyValue::Decimal(2, 0));
1193        }
1194    }
1195
1196    #[test]
1197    fn series_slice_works() {
1198        let series = Series::new("a".into(), &[1i64, 2, 3, 4, 5]);
1199
1200        let slice_1 = series.slice(-3, 3);
1201        let slice_2 = series.slice(-5, 5);
1202        let slice_3 = series.slice(0, 5);
1203
1204        assert_eq!(slice_1.get(0).unwrap(), AnyValue::Int64(3));
1205        assert_eq!(slice_2.get(0).unwrap(), AnyValue::Int64(1));
1206        assert_eq!(slice_3.get(0).unwrap(), AnyValue::Int64(1));
1207    }
1208
1209    #[test]
1210    fn out_of_range_slice_does_not_panic() {
1211        let series = Series::new("a".into(), &[1i64, 2, 3, 4, 5]);
1212
1213        let _ = series.slice(-3, 4);
1214        let _ = series.slice(-6, 2);
1215        let _ = series.slice(4, 2);
1216    }
1217}
polars_core/series/mod.rs

polars_core/series/
mod.rs
