1#![allow(unsafe_op_in_unsafe_fn)]
3use std::borrow::Cow;
4use std::sync::Arc;
5
6use num_traits::Signed;
7use polars_arrow::array::*;
8use polars_arrow::bitmap::Bitmap;
9use polars_arrow::compute::concatenate::concatenate_unchecked;
10use polars_arrow::compute::utils::combine_validities_and;
11use polars_compute::filter::filter_with_bitmap;
12use polars_utils::broadcast::BroadcastLength;
13
14use crate::prelude::{ChunkTakeUnchecked, *};
15
16pub mod ops;
17#[macro_use]
18pub mod arithmetic;
19pub mod builder;
20pub mod cast;
21pub mod collect;
22pub mod comparison;
23pub mod flags;
24pub mod float;
25pub mod iterator;
26#[cfg(feature = "ndarray")]
27pub(crate) mod ndarray;
28
29pub mod arg_min_max;
30#[cfg(feature = "dtype-array")]
31pub(crate) mod array;
32mod binary;
33mod binary_offset;
34mod bitwise;
35#[cfg(feature = "object")]
36mod drop;
37mod from;
38mod from_iterator;
39pub mod from_iterator_par;
40pub(crate) mod list;
41pub(crate) mod logical;
42#[cfg(feature = "object")]
43pub mod object;
44#[cfg(feature = "random")]
45mod random;
46#[cfg(feature = "dtype-struct")]
47mod struct_;
48#[cfg(any(
49 feature = "temporal",
50 feature = "dtype-datetime",
51 feature = "dtype-date"
52))]
53pub mod temporal;
54mod to_vec;
55mod trusted_len;
56pub(crate) use arg_min_max::*;
57#[cfg(feature = "dtype-struct")]
58pub use struct_::StructChunked;
59
60use self::flags::{StatisticsFlags, StatisticsFlagsIM};
61use crate::series::IsSorted;
62use crate::utils::{first_non_null, first_null, last_non_null};
63
64pub type ChunkLenIter<'a> = std::iter::Map<std::slice::Iter<'a, ArrayRef>, fn(&ArrayRef) -> usize>;
65
66pub struct ChunkedArray<T: PolarsDataType> {
141 pub(crate) field: Arc<Field>,
142 pub(crate) chunks: Vec<ArrayRef>,
143
144 pub(crate) flags: StatisticsFlagsIM,
145
146 length: usize,
147 null_count: usize,
148 _pd: std::marker::PhantomData<T>,
149}
150
151impl<T: PolarsDataType> ChunkedArray<T> {
152 fn should_rechunk(&self) -> bool {
153 self.chunks.len() > 1 && self.chunks.len() > self.len() / 3
154 }
155
156 fn optional_rechunk(mut self) -> Self {
157 if self.should_rechunk() {
159 self.rechunk_mut()
160 }
161 self
162 }
163
164 pub(crate) fn as_any(&self) -> &dyn std::any::Any {
165 self
166 }
167
168 pub fn unpack_series_matching_type<'a>(
170 &self,
171 series: &'a Series,
172 ) -> PolarsResult<&'a ChunkedArray<T>> {
173 polars_ensure!(
174 self.dtype() == series.dtype(),
175 SchemaMismatch: "cannot unpack series of type `{}` into `{}`",
176 series.dtype(),
177 self.dtype(),
178 );
179
180 Ok(unsafe { self.unpack_series_matching_physical_type(series) })
182 }
183
184 fn new_with_compute_len(field: Arc<Field>, chunks: Vec<ArrayRef>) -> Self {
189 unsafe {
190 let mut chunked_arr = Self::new_with_dims(field, chunks, 0, 0);
191 chunked_arr.compute_len();
192 chunked_arr
193 }
194 }
195
196 pub unsafe fn new_with_dims(
200 field: Arc<Field>,
201 chunks: Vec<ArrayRef>,
202 length: usize,
203 null_count: usize,
204 ) -> Self {
205 Self {
206 field,
207 chunks,
208 flags: StatisticsFlagsIM::empty(),
209
210 _pd: Default::default(),
211 length,
212 null_count,
213 }
214 }
215
216 pub(crate) fn is_sorted_ascending_flag(&self) -> bool {
217 self.get_flags().is_sorted_ascending()
218 }
219
220 pub(crate) fn is_sorted_descending_flag(&self) -> bool {
221 self.get_flags().is_sorted_descending()
222 }
223
224 pub(crate) fn is_sorted_any(&self) -> bool {
226 self.get_flags().is_sorted_any()
227 }
228
229 pub fn unset_fast_explode_list(&mut self) {
230 self.set_fast_explode_list(false)
231 }
232
233 pub fn set_fast_explode_list(&mut self, value: bool) {
234 let mut flags = self.flags.get_mut();
235 flags.set(StatisticsFlags::CAN_FAST_EXPLODE_LIST, value);
236 self.flags.set_mut(flags);
237 }
238
239 pub fn get_fast_explode_list(&self) -> bool {
240 self.get_flags().can_fast_explode_list()
241 }
242
243 #[inline]
244 pub fn get_flags(&self) -> StatisticsFlags {
245 self.flags.get()
246 }
247
248 pub fn set_flags(&mut self, flags: StatisticsFlags) {
250 self.flags = StatisticsFlagsIM::new(flags);
251 }
252
253 pub fn is_sorted_flag(&self) -> IsSorted {
254 self.get_flags().is_sorted()
255 }
256
257 pub fn retain_flags_from<U: PolarsDataType>(
258 &mut self,
259 from: &ChunkedArray<U>,
260 retain_flags: StatisticsFlags,
261 ) {
262 let flags = from.flags.get();
263 if !flags.is_empty() {
265 self.set_flags(flags & retain_flags)
266 }
267 }
268
269 pub fn set_sorted_flag(&mut self, sorted: IsSorted) {
271 let mut flags = self.flags.get_mut();
272 flags.set_sorted(sorted);
273 self.flags.set_mut(flags);
274 }
275
276 pub fn with_sorted_flag(&self, sorted: IsSorted) -> Self {
278 let mut out = self.clone();
279 out.set_sorted_flag(sorted);
280 out
281 }
282
283 pub fn first_null(&self) -> Option<usize> {
284 if self.null_count() == 0 {
285 None
286 }
287 else if self.null_count() == self.len() {
289 Some(0)
290 } else if self.is_sorted_any() {
291 let out = if self
292 .chunks
293 .iter()
294 .find(|arr| !arr.is_empty())
295 .unwrap()
296 .is_null(0)
297 {
298 0
300 } else {
301 self.len() - self.null_count()
303 };
304
305 debug_assert!(
306 unsafe { self.get_unchecked(out) }.is_none(),
308 "incorrect sorted flag"
309 );
310
311 Some(out)
312 } else {
313 first_null(self.chunks().iter().map(|arr| arr.as_ref()))
314 }
315 }
316
317 pub fn first_non_null(&self) -> Option<usize> {
319 if self.null_count() == self.len() {
320 None
321 }
322 else if self.null_count() == 0 {
324 Some(0)
325 } else if self.is_sorted_any() {
326 let out = if self
327 .chunks
328 .iter()
329 .find(|arr| !arr.is_empty())
330 .unwrap()
331 .is_null(0)
332 {
333 self.null_count()
335 } else {
336 0
338 };
339
340 debug_assert!(
341 unsafe { self.get_unchecked(out) }.is_some(),
343 "incorrect sorted flag"
344 );
345
346 Some(out)
347 } else {
348 first_non_null(self.chunks().iter().map(|arr| arr.as_ref()))
349 }
350 }
351
352 pub fn last_non_null(&self) -> Option<usize> {
354 if self.null_count() == self.len() {
355 None
356 }
357 else if self.null_count() == 0 {
359 Some(self.len() - 1)
360 } else if self.is_sorted_any() {
361 let out = if self
362 .chunks
363 .iter()
364 .find(|arr| !arr.is_empty())
365 .unwrap()
366 .is_null(0)
367 {
368 self.len() - 1
370 } else {
371 self.len() - self.null_count() - 1
373 };
374
375 debug_assert!(
376 unsafe { self.get_unchecked(out) }.is_some(),
378 "incorrect sorted flag"
379 );
380
381 Some(out)
382 } else {
383 last_non_null(self.chunks().iter().map(|arr| arr.as_ref()), self.len())
384 }
385 }
386
387 pub fn drop_nulls(&self) -> Self {
388 if self.null_count() == 0 {
389 self.clone()
390 } else {
391 let chunks = self
392 .downcast_iter()
393 .map(|arr| {
394 if arr.null_count() == 0 {
395 arr.to_boxed()
396 } else {
397 filter_with_bitmap(arr, arr.validity().unwrap())
398 }
399 })
400 .collect();
401 unsafe {
402 Self::new_with_dims(
403 self.field.clone(),
404 chunks,
405 self.len() - self.null_count(),
406 0,
407 )
408 }
409 }
410 }
411
412 #[inline]
414 #[allow(clippy::type_complexity)]
415 pub fn iter_validities(
416 &self,
417 ) -> impl ExactSizeIterator<Item = Option<&Bitmap>> + DoubleEndedIterator {
418 fn to_validity(arr: &ArrayRef) -> Option<&Bitmap> {
419 arr.validity()
420 }
421 self.chunks.iter().map(to_validity)
422 }
423
424 #[inline]
425 pub fn has_nulls(&self) -> bool {
427 self.null_count > 0
428 }
429
430 pub fn shrink_to_fit(&mut self) {
432 self.chunks = vec![concatenate_unchecked(self.chunks.as_slice()).unwrap()];
433 }
434
435 pub fn clear(&self) -> Self {
436 let mut ca = unsafe {
438 self.copy_with_chunks(vec![new_empty_array(
439 self.chunks.first().unwrap().dtype().clone(),
440 )])
441 };
442
443 use StatisticsFlags as F;
444 ca.retain_flags_from(self, F::IS_SORTED_ANY | F::CAN_FAST_EXPLODE_LIST);
445 ca
446 }
447
448 pub(crate) unsafe fn unpack_series_matching_physical_type<'a>(
455 &self,
456 series: &'a Series,
457 ) -> &'a ChunkedArray<T> {
458 let series_trait = &**series;
459 if self.dtype() == series.dtype() {
460 &*(series_trait as *const dyn SeriesTrait as *const ChunkedArray<T>)
461 } else {
462 use DataType::*;
463 match (self.dtype(), series.dtype()) {
464 (Int64, Datetime(_, _)) | (Int64, Duration(_)) | (Int32, Date) => {
465 &*(series_trait as *const dyn SeriesTrait as *const ChunkedArray<T>)
466 },
467 _ => panic!(
468 "cannot unpack series {:?} into matching type {:?}",
469 series,
470 self.dtype()
471 ),
472 }
473 }
474 }
475
476 pub fn chunk_lengths(&self) -> ChunkLenIter<'_> {
478 self.chunks.iter().map(|chunk| chunk.len())
479 }
480
481 #[inline]
483 pub fn chunks(&self) -> &Vec<ArrayRef> {
484 &self.chunks
485 }
486
487 #[inline]
493 pub unsafe fn chunks_mut(&mut self) -> &mut Vec<ArrayRef> {
494 &mut self.chunks
495 }
496
497 unsafe fn copy_with_chunks(&self, chunks: Vec<ArrayRef>) -> Self {
502 Self::new_with_compute_len(self.field.clone(), chunks)
503 }
504
505 #[inline(always)]
507 pub fn dtype(&self) -> &DataType {
508 self.field.dtype()
509 }
510
511 pub(crate) unsafe fn set_dtype(&mut self, dtype: DataType) {
512 self.field = Arc::new(Field::new(self.name().clone(), dtype))
513 }
514
515 #[inline]
517 pub fn name(&self) -> &PlSmallStr {
518 self.field.name()
519 }
520
521 #[inline(always)]
523 pub fn ref_field(&self) -> &Field {
524 &self.field
525 }
526
527 pub fn rename(&mut self, name: PlSmallStr) {
529 self.field = Arc::new(Field::new(name, self.field.dtype().clone()));
530 }
531
532 pub fn with_name(mut self, name: PlSmallStr) -> Self {
534 self.rename(name);
535 self
536 }
537}
538
539impl<T> ChunkedArray<T>
540where
541 T: PolarsDataType,
542{
543 #[inline]
549 pub fn get(&self, idx: usize) -> Option<T::Physical<'_>> {
550 let (chunk_idx, arr_idx) = self.index_to_chunked_index(idx);
551 assert!(
552 chunk_idx < self.chunks().len(),
553 "index: {} out of bounds for len: {}",
554 idx,
555 self.len()
556 );
557 unsafe {
558 let arr = self.downcast_get_unchecked(chunk_idx);
559 assert!(
560 arr_idx < arr.len(),
561 "index: {} out of bounds for len: {}",
562 idx,
563 self.len()
564 );
565 arr.get_unchecked(arr_idx)
566 }
567 }
568
569 #[inline]
575 pub unsafe fn get_unchecked(&self, idx: usize) -> Option<T::Physical<'_>> {
576 let (chunk_idx, arr_idx) = self.index_to_chunked_index(idx);
577
578 unsafe {
579 self.downcast_get_unchecked(chunk_idx)
581 .get_unchecked(arr_idx)
582 }
583 }
584
585 #[inline]
591 pub unsafe fn value_unchecked(&self, idx: usize) -> T::Physical<'_> {
592 let (chunk_idx, arr_idx) = self.index_to_chunked_index(idx);
593
594 unsafe {
595 self.downcast_get_unchecked(chunk_idx)
597 .value_unchecked(arr_idx)
598 }
599 }
600
601 #[inline]
604 pub fn first(&self) -> Option<T::Physical<'_>> {
605 self.iter().next().unwrap()
606 }
607
608 #[inline]
611 pub fn last(&self) -> Option<T::Physical<'_>> {
612 let arr = self
613 .downcast_iter()
614 .rev()
615 .find(|arr| !arr.is_empty())
616 .unwrap();
617 unsafe { arr.get_unchecked(arr.len() - 1) }
618 }
619
620 pub fn set_validity(&mut self, validity: Option<Bitmap>) {
621 assert!(
622 !self.dtype().is_struct(),
623 "set_outer_validity should be used for struct types"
624 );
625 if let Some(v) = &validity {
626 assert_eq!(self.len(), v.len());
627 }
628 let mut i = 0;
629 for chunk in unsafe { self.chunks_mut() } {
630 *chunk =
631 chunk.with_validity(validity.as_ref().map(|v| v.clone().sliced(i, chunk.len())));
632 i += chunk.len();
633 }
634 self.null_count = validity.map(|v| v.unset_bits()).unwrap_or(0);
635 self.set_fast_explode_list(false);
636 }
637
638 pub fn with_validity(mut self, validity: Option<Bitmap>) -> Self {
639 self.set_validity(validity);
640 self
641 }
642}
643
644impl<T> ChunkedArray<T>
645where
646 T: PolarsDataType,
647 ChunkedArray<T>: ChunkExpandAtIndex<T>,
648{
649 pub fn broadcast_to(&self, length: usize) -> PolarsResult<Cow<'_, Self>> {
653 let len = self.len();
654 if len == length {
655 Ok(Cow::Borrowed(self))
656 } else if len == 1 {
657 Ok(Cow::Owned(self.new_from_index(0, length)))
658 } else {
659 polars_bail!(
660 ShapeMismatch: "can't broadcast Series '{}' of length {len} to length {length}",
661 self.name()
662 );
663 }
664 }
665
666 pub fn broadcast_in_place_to(&mut self, length: usize) -> PolarsResult<()> {
668 if let Cow::Owned(new) = self.broadcast_to(length)? {
669 *self = new;
670 }
671 Ok(())
672 }
673
674 pub fn broadcast_owned_to(mut self, length: usize) -> PolarsResult<Self> {
676 self.broadcast_in_place_to(length)?;
677 Ok(self)
678 }
679}
680
681impl<T> ChunkedArray<T>
682where
683 T: PolarsDataType,
684 ChunkedArray<T>: ChunkTakeUnchecked<[IdxSize]>,
685{
686 pub fn deposit(&self, validity: &Bitmap) -> Self {
690 self.deposit_with(validity, Self::set_validity)
691 }
692
693 pub(crate) fn deposit_with(
695 &self,
696 validity: &Bitmap,
697 set_validity: impl FnOnce(&mut Self, Option<Bitmap>),
698 ) -> Self {
699 let set_bits = validity.set_bits();
700
701 assert_eq!(self.len(), set_bits);
702
703 if set_bits == validity.len() {
704 return self.clone();
705 }
706
707 if set_bits == 0 {
708 return Self::full_null_like(self, validity.len());
709 }
710
711 let mut null_mask = validity.clone();
712
713 let mut gather_idxs = Vec::with_capacity(validity.len());
714 let leading_nulls = null_mask.take_leading_zeros();
715 gather_idxs.extend(std::iter::repeat_n(0, leading_nulls + 1));
716
717 let mut i = 0 as IdxSize;
718 gather_idxs.extend(null_mask.iter().skip(1).map(|v| {
719 i += IdxSize::from(v);
720 i
721 }));
722
723 let mut ca = unsafe { ChunkTakeUnchecked::take_unchecked(self, &gather_idxs) };
724 let validity = combine_validities_and(Some(validity), ca.rechunk_validity().as_ref());
725 set_validity(&mut ca, validity);
726 ca
727 }
728}
729
730impl ListChunked {
731 #[inline]
732 pub fn get_as_series(&self, idx: usize) -> Option<Series> {
733 unsafe {
734 Some(Series::from_chunks_and_dtype_unchecked(
735 self.name().clone(),
736 vec![self.get(idx)?],
737 &self.inner_dtype().to_physical(),
738 ))
739 }
740 }
741
742 pub fn has_empty_lists(&self) -> bool {
743 for arr in self.downcast_iter() {
744 if arr.is_empty() {
745 continue;
746 }
747
748 if match arr.validity() {
749 None => arr.offsets().lengths().any(|l| l == 0),
750 Some(validity) => arr
751 .offsets()
752 .lengths()
753 .enumerate()
754 .any(|(i, l)| l == 0 && unsafe { validity.get_bit_unchecked(i) }),
755 } {
756 return true;
757 }
758 }
759
760 false
761 }
762
763 pub fn has_masked_out_values(&self) -> bool {
764 for arr in self.downcast_iter() {
765 if arr.is_empty() {
766 continue;
767 }
768
769 if *arr.offsets().first() != 0 || *arr.offsets().last() != arr.values().len() as i64 {
770 return true;
771 }
772
773 let Some(validity) = arr.validity() else {
774 continue;
775 };
776 if validity.set_bits() == 0 {
777 continue;
778 }
779
780 for i in (!validity).true_idx_iter() {
782 if arr.offsets().length_at(i) > 0 {
783 return true;
784 }
785 }
786 }
787
788 false
789 }
790}
791
792#[cfg(feature = "dtype-array")]
793impl ArrayChunked {
794 #[inline]
795 pub fn get_as_series(&self, idx: usize) -> Option<Series> {
796 unsafe {
797 Some(Series::from_chunks_and_dtype_unchecked(
798 self.name().clone(),
799 vec![self.get(idx)?],
800 &self.inner_dtype().to_physical(),
801 ))
802 }
803 }
804
805 pub fn from_aligned_values(
806 name: PlSmallStr,
807 inner_dtype: &DataType,
808 width: usize,
809 chunks: Vec<ArrayRef>,
810 length: usize,
811 ) -> Self {
812 let dtype = DataType::Array(Box::new(inner_dtype.clone()), width);
813 let arrow_dtype = inner_dtype
814 .to_physical()
815 .to_arrow(CompatLevel::newest())
816 .to_fixed_size_list(width, true);
817 let field = Arc::new(Field::new(name, dtype));
818 if width == 0 {
819 use polars_arrow::array::builder::{ArrayBuilder, make_builder};
820 let values = make_builder(&inner_dtype.to_arrow(CompatLevel::newest())).freeze();
821 return ArrayChunked::new_with_compute_len(
822 field,
823 vec![FixedSizeListArray::new(arrow_dtype, length, values, None).into_boxed()],
824 );
825 }
826 let mut total_len = 0;
827 let chunks = chunks
828 .iter()
829 .map(|chunk| {
830 debug_assert_eq!(chunk.len() % width, 0);
831 let chunk_len = chunk.len() / width;
832 total_len += chunk_len;
833 FixedSizeListArray::new(arrow_dtype.clone(), chunk_len, chunk.clone(), None)
834 .into_boxed()
835 })
836 .collect();
837 debug_assert_eq!(total_len, length);
838
839 unsafe { Self::new_with_dims(field, chunks, length, 0) }
840 }
841
842 pub fn to_list(&self) -> ListChunked {
846 let inner_dtype = self.inner_dtype();
847 let chunks = self
848 .downcast_iter()
849 .map(|chunk| {
850 use polars_arrow::offset::OffsetsBuffer;
851
852 let inner_dtype = chunk.dtype().inner_dtype().unwrap();
853 let dtype = inner_dtype.clone().to_large_list(true);
854
855 let offsets = (0..=chunk.len())
856 .map(|i| (i * self.width()) as i64)
857 .collect::<Vec<i64>>();
858
859 let offsets = unsafe { OffsetsBuffer::new_unchecked(offsets.into()) };
861
862 ListArray::<i64>::new(
863 dtype,
864 offsets,
865 chunk.values().clone(),
866 chunk.validity().cloned(),
867 )
868 .into_boxed()
869 })
870 .collect();
871
872 let mut ca = unsafe {
874 ListChunked::new_with_dims(
875 Arc::new(Field::new(
876 self.name().clone(),
877 DataType::List(Box::new(inner_dtype.clone())),
878 )),
879 chunks,
880 self.len(),
881 self.null_count(),
882 )
883 };
884 ca.set_fast_explode_list(!self.has_nulls());
885 ca
886 }
887}
888
889impl<T> ChunkedArray<T>
890where
891 T: PolarsDataType,
892{
893 pub fn match_chunks<I>(&self, chunk_id: I) -> Self
897 where
898 I: Iterator<Item = usize>,
899 {
900 debug_assert!(self.chunks.len() == 1);
901 let slice = |ca: &Self| {
903 let array = &ca.chunks[0];
904
905 let mut offset = 0;
906 let chunks = chunk_id
907 .map(|len| {
908 debug_assert!((offset + len) <= array.len());
910 let out = unsafe { array.sliced_unchecked(offset, len) };
911 offset += len;
912 out
913 })
914 .collect();
915
916 debug_assert_eq!(offset, array.len());
917
918 unsafe {
920 Self::from_chunks_and_dtype(self.name().clone(), chunks, self.dtype().clone())
921 }
922 };
923
924 if self.chunks.len() != 1 {
925 let out = self.rechunk();
926 slice(&out)
927 } else {
928 slice(self)
929 }
930 }
931}
932
933impl<T: PolarsDataType> AsRefDataType for ChunkedArray<T> {
934 fn as_ref_dtype(&self) -> &DataType {
935 self.dtype()
936 }
937}
938
939pub(crate) trait AsSinglePtr: AsRefDataType {
940 fn as_single_ptr(&mut self) -> PolarsResult<usize> {
942 polars_bail!(opq = as_single_ptr, self.as_ref_dtype());
943 }
944}
945
946impl<T> AsSinglePtr for ChunkedArray<T>
947where
948 T: PolarsNumericType,
949{
950 fn as_single_ptr(&mut self) -> PolarsResult<usize> {
951 self.rechunk_mut();
952 let a = self.data_views().next().unwrap();
953 let ptr = a.as_ptr();
954 Ok(ptr as usize)
955 }
956}
957
958impl AsSinglePtr for BooleanChunked {}
959impl AsSinglePtr for ListChunked {}
960#[cfg(feature = "dtype-array")]
961impl AsSinglePtr for ArrayChunked {}
962impl AsSinglePtr for StringChunked {}
963impl AsSinglePtr for BinaryChunked {}
964#[cfg(feature = "object")]
965impl<T: PolarsObject> AsSinglePtr for ObjectChunked<T> {}
966
967pub enum ChunkedArrayLayout<'a, T: PolarsDataType> {
968 SingleNoNull(&'a T::Array),
969 Single(&'a T::Array),
970 MultiNoNull(&'a ChunkedArray<T>),
971 Multi(&'a ChunkedArray<T>),
972}
973
974impl<T> ChunkedArray<T>
975where
976 T: PolarsDataType,
977{
978 pub fn layout(&self) -> ChunkedArrayLayout<'_, T> {
979 if self.chunks.len() == 1 {
980 let arr = self.downcast_iter().next().unwrap();
981 return if arr.null_count() == 0 {
982 ChunkedArrayLayout::SingleNoNull(arr)
983 } else {
984 ChunkedArrayLayout::Single(arr)
985 };
986 }
987
988 if self.downcast_iter().all(|a| a.null_count() == 0) {
989 ChunkedArrayLayout::MultiNoNull(self)
990 } else {
991 ChunkedArrayLayout::Multi(self)
992 }
993 }
994}
995
996impl<T> ChunkedArray<T>
997where
998 T: PolarsNumericType,
999{
1000 pub fn cont_slice(&self) -> PolarsResult<&[T::Native]> {
1002 polars_ensure!(
1003 self.chunks.len() == 1 && self.chunks[0].null_count() == 0,
1004 ComputeError: "chunked array is not contiguous"
1005 );
1006 Ok(self.downcast_iter().next().map(|arr| arr.values()).unwrap())
1007 }
1008
1009 pub fn data_views(&self) -> impl DoubleEndedIterator<Item = &[T::Native]> {
1013 self.downcast_iter().map(|arr| arr.values().as_slice())
1014 }
1015
1016 #[allow(clippy::wrong_self_convention)]
1017 pub fn into_no_null_iter(
1018 &self,
1019 ) -> impl '_ + Send + Sync + ExactSizeIterator<Item = T::Native> + DoubleEndedIterator + TrustedLen
1020 {
1021 #[allow(clippy::map_clone)]
1023 unsafe {
1025 self.data_views()
1026 .flatten()
1027 .map(|v| *v)
1028 .trust_my_length(self.len())
1029 }
1030 }
1031}
1032
1033impl<T> ChunkedArray<T>
1034where
1035 T: PolarsIntegerType,
1036 T::Native: Signed,
1037{
1038 pub fn has_negative(&self) -> bool {
1040 self.data_views().any(|values| {
1042 values
1043 .chunks(1024)
1044 .any(|block| block.iter().fold(false, |acc, v| acc | v.is_negative()))
1045 })
1046 }
1047}
1048
1049impl<T: PolarsDataType> Clone for ChunkedArray<T> {
1050 fn clone(&self) -> Self {
1051 ChunkedArray {
1052 field: self.field.clone(),
1053 chunks: self.chunks.clone(),
1054 flags: self.flags.clone(),
1055
1056 _pd: Default::default(),
1057 length: self.length,
1058 null_count: self.null_count,
1059 }
1060 }
1061}
1062
1063impl<T: PolarsDataType> AsRef<ChunkedArray<T>> for ChunkedArray<T> {
1064 fn as_ref(&self) -> &ChunkedArray<T> {
1065 self
1066 }
1067}
1068
1069impl ValueSize for ListChunked {
1070 fn get_values_size(&self) -> usize {
1071 self.chunks
1072 .iter()
1073 .fold(0usize, |acc, arr| acc + arr.get_values_size())
1074 }
1075}
1076
1077#[cfg(feature = "dtype-array")]
1078impl ValueSize for ArrayChunked {
1079 fn get_values_size(&self) -> usize {
1080 self.chunks
1081 .iter()
1082 .fold(0usize, |acc, arr| acc + arr.get_values_size())
1083 }
1084}
1085impl ValueSize for StringChunked {
1086 fn get_values_size(&self) -> usize {
1087 self.chunks
1088 .iter()
1089 .fold(0usize, |acc, arr| acc + arr.get_values_size())
1090 }
1091}
1092
1093impl ValueSize for BinaryOffsetChunked {
1094 fn get_values_size(&self) -> usize {
1095 self.chunks
1096 .iter()
1097 .fold(0usize, |acc, arr| acc + arr.get_values_size())
1098 }
1099}
1100
1101pub(crate) fn align_inner_chunks(
1106 chunk_lens: impl Iterator<Item = usize>,
1107 values: &Series,
1108) -> Series {
1109 let chunk_lens = chunk_lens.collect::<Vec<_>>();
1110
1111 if chunk_lens.len() == values.chunks().len()
1112 && chunk_lens
1113 .iter()
1114 .zip(values.chunks())
1115 .all(|(len, arr)| *len == arr.len())
1116 {
1117 return values.clone();
1118 }
1119
1120 let mut values = values.rechunk();
1121 let chunks = unsafe { values.chunks_mut() };
1122 let mut arr = chunks.pop().unwrap();
1123 chunks.extend(chunk_lens.into_iter().map(|len| {
1124 let chunk;
1125 (chunk, arr) = arr.split_at_boxed(len);
1126 chunk
1127 }));
1128 assert!(arr.is_empty());
1129 values
1130}
1131
1132pub(crate) fn to_primitive<T: PolarsNumericType>(
1133 values: Vec<T::Native>,
1134 validity: Option<Bitmap>,
1135) -> PrimitiveArray<T::Native> {
1136 PrimitiveArray::new(
1137 T::get_static_dtype().to_arrow(CompatLevel::newest()),
1138 values.into(),
1139 validity,
1140 )
1141}
1142
1143pub(crate) fn to_array<T: PolarsNumericType>(
1144 values: Vec<T::Native>,
1145 validity: Option<Bitmap>,
1146) -> ArrayRef {
1147 Box::new(to_primitive::<T>(values, validity))
1148}
1149
1150impl<T: PolarsDataType> Default for ChunkedArray<T> {
1151 fn default() -> Self {
1152 let dtype = T::get_static_dtype();
1153 let arrow_dtype = dtype.to_physical().to_arrow(CompatLevel::newest());
1154 ChunkedArray {
1155 field: Arc::new(Field::new(PlSmallStr::EMPTY, dtype)),
1156 chunks: vec![new_empty_array(arrow_dtype)],
1158 flags: StatisticsFlagsIM::empty(),
1159
1160 _pd: Default::default(),
1161 length: 0,
1162 null_count: 0,
1163 }
1164 }
1165}
1166
1167impl<T: PolarsDataType> BroadcastLength for ChunkedArray<T> {
1168 fn _broadcast_len(&self) -> usize {
1169 self.len()
1170 }
1171
1172 fn _column_name(&self) -> Option<&str> {
1173 Some(self.name())
1174 }
1175}
1176
1177#[cfg(test)]
1178pub(crate) mod test {
1179 use crate::prelude::*;
1180
1181 pub(crate) fn get_chunked_array() -> Int32Chunked {
1182 ChunkedArray::new(PlSmallStr::from_static("a"), &[1, 2, 3])
1183 }
1184
1185 #[test]
1186 fn test_sort() {
1187 let a = Int32Chunked::new(PlSmallStr::from_static("a"), &[1, 9, 3, 2]);
1188 let b = a
1189 .sort(false)
1190 .iter()
1191 .map(|opt| opt.unwrap())
1192 .collect::<Vec<_>>();
1193 assert_eq!(b, [1, 2, 3, 9]);
1194 let a = StringChunked::new(PlSmallStr::from_static("a"), &["b", "a", "c"]);
1195 let a = a.sort(false);
1196 let b = a.iter().collect::<Vec<_>>();
1197 assert_eq!(b, [Some("a"), Some("b"), Some("c")]);
1198 assert!(a.is_sorted_ascending_flag());
1199 }
1200
1201 #[test]
1202 fn arithmetic() {
1203 let a = &Int32Chunked::new(PlSmallStr::from_static("a"), &[1, 100, 6, 40]);
1204 let b = &Int32Chunked::new(PlSmallStr::from_static("b"), &[-1, 2, 3, 4]);
1205
1206 println!("{:?}", a + b);
1209 println!("{:?}", a - b);
1210 println!("{:?}", a * b);
1211 println!("{:?}", a / b);
1212 }
1213
1214 #[test]
1215 fn iter() {
1216 let s1 = get_chunked_array();
1217 assert_eq!(s1.iter().fold(0, |acc, val| { acc + val.unwrap() }), 6)
1219 }
1220
1221 #[test]
1222 fn limit() {
1223 let a = get_chunked_array();
1224 let b = a.limit(2);
1225 println!("{b:?}");
1226 assert_eq!(b.len(), 2)
1227 }
1228
1229 #[test]
1230 fn filter() {
1231 let a = get_chunked_array();
1232 let b = a
1233 .filter(&BooleanChunked::new(
1234 PlSmallStr::from_static("filter"),
1235 &[true, false, false],
1236 ))
1237 .unwrap();
1238 assert_eq!(b.len(), 1);
1239 assert_eq!(b.iter().next(), Some(Some(1)));
1240 }
1241
1242 #[test]
1243 fn aggregates() {
1244 let a = &Int32Chunked::new(PlSmallStr::from_static("a"), &[1, 100, 10, 9]);
1245 assert_eq!(a.max(), Some(100));
1246 assert_eq!(a.min(), Some(1));
1247 assert_eq!(a.sum(), Some(120))
1248 }
1249
1250 #[test]
1251 fn take() {
1252 let a = get_chunked_array();
1253 let new = a.take(&[0 as IdxSize, 1]).unwrap();
1254 assert_eq!(new.len(), 2)
1255 }
1256
1257 #[test]
1258 fn cast() {
1259 let a = get_chunked_array();
1260 let b = a.cast(&DataType::Int64).unwrap();
1261 assert_eq!(b.dtype(), &DataType::Int64)
1262 }
1263
1264 fn assert_slice_equal<T>(ca: &ChunkedArray<T>, eq: &[T::Native])
1265 where
1266 T: PolarsNumericType,
1267 {
1268 assert_eq!(ca.iter().map(|opt| opt.unwrap()).collect::<Vec<_>>(), eq)
1269 }
1270
1271 #[test]
1272 fn slice() {
1273 let mut first = UInt32Chunked::new(PlSmallStr::from_static("first"), &[0, 1, 2]);
1274 let second = UInt32Chunked::new(PlSmallStr::from_static("second"), &[3, 4, 5]);
1275 first.append(&second).unwrap();
1276 assert_slice_equal(&first.slice(Slice::head(3)), &[0, 1, 2]);
1277 assert_slice_equal(&first.slice(Slice::head(4)), &[0, 1, 2, 3]);
1278 assert_slice_equal(&first.slice((1usize, 4)), &[1, 2, 3, 4]);
1279 assert_slice_equal(&first.slice((3usize, 2)), &[3, 4]);
1280 assert_slice_equal(&first.slice((3usize, 3)), &[3, 4, 5]);
1281 assert_slice_equal(&first.slice((-3i64, 3)), &[3, 4, 5]);
1282 assert_slice_equal(&first.slice((-6i64, 6)), &[0, 1, 2, 3, 4, 5]);
1283
1284 assert_eq!(first.slice((-7i64, 2)).len(), 1);
1285 assert_eq!(first.slice((-3i64, 4)).len(), 3);
1286 assert_eq!(first.slice((3usize, 4)).len(), 3);
1287 assert_eq!(first.slice((10usize, 4)).len(), 0);
1288 }
1289
1290 #[test]
1291 fn sorting() {
1292 let s = UInt32Chunked::new(PlSmallStr::EMPTY, &[9, 2, 4]);
1293 let sorted = s.sort(false);
1294 assert_slice_equal(&sorted, &[2, 4, 9]);
1295 let sorted = s.sort(true);
1296 assert_slice_equal(&sorted, &[9, 4, 2]);
1297
1298 let s: StringChunked = ["b", "a", "z"].iter().collect();
1299 let sorted = s.sort(false);
1300 assert_eq!(
1301 sorted.iter().collect::<Vec<_>>(),
1302 &[Some("a"), Some("b"), Some("z")]
1303 );
1304 let sorted = s.sort(true);
1305 assert_eq!(
1306 sorted.iter().collect::<Vec<_>>(),
1307 &[Some("z"), Some("b"), Some("a")]
1308 );
1309 let s: StringChunked = [Some("b"), None, Some("z")].iter().copied().collect();
1310 let sorted = s.sort(false);
1311 assert_eq!(
1312 sorted.iter().collect::<Vec<_>>(),
1313 &[None, Some("b"), Some("z")]
1314 );
1315 }
1316
1317 #[test]
1318 fn reverse() {
1319 let s = UInt32Chunked::new(PlSmallStr::EMPTY, &[1, 2, 3]);
1320 assert_slice_equal(&s.reverse(), &[3, 2, 1]);
1322 let s = UInt32Chunked::new(PlSmallStr::EMPTY, &[Some(1), None, Some(3)]);
1324 assert_eq!(Vec::from(&s.reverse()), &[Some(3), None, Some(1)]);
1325 let s = BooleanChunked::new(PlSmallStr::EMPTY, &[true, false]);
1326 assert_eq!(Vec::from(&s.reverse()), &[Some(false), Some(true)]);
1327
1328 let s = StringChunked::new(PlSmallStr::EMPTY, &["a", "b", "c"]);
1329 assert_eq!(Vec::from(&s.reverse()), &[Some("c"), Some("b"), Some("a")]);
1330
1331 let s = StringChunked::new(PlSmallStr::EMPTY, &[Some("a"), None, Some("c")]);
1332 assert_eq!(Vec::from(&s.reverse()), &[Some("c"), None, Some("a")]);
1333 }
1334
1335 #[test]
1336 #[cfg(feature = "dtype-categorical")]
1337 fn test_iter_categorical() {
1338 let ca = StringChunked::new(
1339 PlSmallStr::EMPTY,
1340 &[Some("foo"), None, Some("bar"), Some("ham")],
1341 );
1342 let cats = Categories::new(
1343 PlSmallStr::EMPTY,
1344 PlSmallStr::EMPTY,
1345 CategoricalPhysical::U32,
1346 );
1347 let ca = ca.cast(&DataType::from_categories(cats)).unwrap();
1348 let ca = ca.cat32().unwrap();
1349 let v: Vec<_> = ca.physical().iter().collect();
1350 assert_eq!(v, &[Some(0), None, Some(1), Some(2)]);
1351 }
1352
1353 #[test]
1354 #[ignore]
1355 fn test_shrink_to_fit() {
1356 let mut builder = StringChunkedBuilder::new(PlSmallStr::from_static("foo"), 2048);
1357 builder.append_value("foo");
1358 let mut arr = builder.finish();
1359 let before = arr
1360 .chunks()
1361 .iter()
1362 .map(|arr| polars_arrow::compute::aggregate::estimated_bytes_size(arr.as_ref()))
1363 .sum::<usize>();
1364 arr.shrink_to_fit();
1365 let after = arr
1366 .chunks()
1367 .iter()
1368 .map(|arr| polars_arrow::compute::aggregate::estimated_bytes_size(arr.as_ref()))
1369 .sum::<usize>();
1370 assert!(before > after);
1371 }
1372}