Skip to main content

polars_ops/chunked_array/list/
namespace.rs

1use std::fmt::Write;
2
3use polars_compute::gather::sublist::list::{index_is_oob, sublist_get};
4#[cfg(feature = "diff")]
5use polars_core::series::ops::NullBehavior;
6use polars_core::utils::{CustomIterTools, try_get_supertype};
7
8use super::*;
9use crate::chunked_array::list::min_max::{list_max_function, list_min_function};
10use crate::chunked_array::list::sum_mean::sum_with_nulls;
11#[cfg(feature = "diff")]
12use crate::prelude::diff;
13use crate::prelude::list::sum_mean::{mean_list_numerical, sum_list_numerical};
14use crate::series::ArgAgg;
15use crate::series::concat_list::concat_list;
16
17pub(super) fn has_inner_nulls(ca: &ListChunked) -> bool {
18    for arr in ca.downcast_iter() {
19        if arr.values().null_count() > 0 {
20            return true;
21        }
22    }
23    false
24}
25
26fn cast_rhs(other: &mut [Column], inner_type: &DataType, dtype: &DataType) -> PolarsResult<()> {
27    for s in other.iter_mut() {
28        // make sure that inner types match before we coerce into list
29        if !matches!(s.dtype(), DataType::List(_)) {
30            *s = s.cast(inner_type)?
31        }
32        if !matches!(s.dtype(), DataType::List(_)) && s.dtype() == inner_type {
33            // coerce to list JIT
34            *s = s
35                .reshape_list(&[ReshapeDimension::Infer, ReshapeDimension::new_dimension(1)])
36                .unwrap();
37        }
38        if s.dtype() != dtype {
39            *s = s.cast(dtype).map_err(|e| {
40                polars_err!(
41                    SchemaMismatch:
42                    "cannot concat `{}` into a list of `{}`: {}",
43                    s.dtype(),
44                    dtype,
45                    e
46                )
47            })?;
48        }
49    }
50    Ok(())
51}
52
53pub trait ListNameSpaceImpl: AsList {
54    /// In case the inner dtype [`DataType::String`], the individual items will be joined into a
55    /// single string separated by `separator`.
56    fn lst_join(
57        &self,
58        separator: &StringChunked,
59        ignore_nulls: bool,
60    ) -> PolarsResult<StringChunked> {
61        let ca = self.as_list();
62        match ca.inner_dtype() {
63            DataType::String => match separator.len() {
64                1 => match separator.get(0) {
65                    Some(separator) => self.join_literal(separator, ignore_nulls),
66                    _ => Ok(StringChunked::full_null(ca.name().clone(), ca.len())),
67                },
68                _ => self.join_many(separator, ignore_nulls),
69            },
70            dt => polars_bail!(op = "`lst.join`", got = dt, expected = "String"),
71        }
72    }
73
74    fn join_literal(&self, separator: &str, ignore_nulls: bool) -> PolarsResult<StringChunked> {
75        let ca = self.as_list();
76        // used to amortize heap allocs
77        let mut buf = String::with_capacity(128);
78        let mut builder = StringChunkedBuilder::new(ca.name().clone(), ca.len());
79
80        ca.for_each_amortized(|opt_s| {
81            let opt_val = opt_s.and_then(|s| {
82                // make sure that we don't write values of previous iteration
83                buf.clear();
84                let ca = s.as_ref().str().unwrap();
85
86                if ca.null_count() != 0 && !ignore_nulls {
87                    return None;
88                }
89
90                for arr in ca.downcast_iter() {
91                    for val in arr.non_null_values_iter() {
92                        buf.write_str(val).unwrap();
93                        buf.write_str(separator).unwrap();
94                    }
95                }
96
97                // last value should not have a separator, so slice that off
98                // saturating sub because there might have been nothing written.
99                Some(&buf[..buf.len().saturating_sub(separator.len())])
100            });
101            builder.append_option(opt_val)
102        });
103        Ok(builder.finish())
104    }
105
106    fn join_many(
107        &self,
108        separator: &StringChunked,
109        ignore_nulls: bool,
110    ) -> PolarsResult<StringChunked> {
111        let ca = self.as_list();
112        // used to amortize heap allocs
113        let mut buf = String::with_capacity(128);
114        let mut builder = StringChunkedBuilder::new(ca.name().clone(), ca.len());
115        {
116            ca.amortized_iter()
117                .zip(separator.iter())
118                .for_each(|(opt_s, opt_sep)| match opt_sep {
119                    Some(separator) => {
120                        let opt_val = opt_s.and_then(|s| {
121                            // make sure that we don't write values of previous iteration
122                            buf.clear();
123                            let ca = s.as_ref().str().unwrap();
124
125                            if ca.null_count() != 0 && !ignore_nulls {
126                                return None;
127                            }
128
129                            for arr in ca.downcast_iter() {
130                                for val in arr.non_null_values_iter() {
131                                    buf.write_str(val).unwrap();
132                                    buf.write_str(separator).unwrap();
133                                }
134                            }
135
136                            // last value should not have a separator, so slice that off
137                            // saturating sub because there might have been nothing written.
138                            Some(&buf[..buf.len().saturating_sub(separator.len())])
139                        });
140                        builder.append_option(opt_val)
141                    },
142                    _ => builder.append_null(),
143                })
144        }
145        Ok(builder.finish())
146    }
147
148    fn lst_max(&self) -> PolarsResult<Series> {
149        list_max_function(self.as_list())
150    }
151
152    fn lst_min(&self) -> PolarsResult<Series> {
153        list_min_function(self.as_list())
154    }
155
156    fn lst_sum(&self) -> PolarsResult<Series> {
157        let ca = self.as_list();
158
159        if has_inner_nulls(ca) {
160            return sum_with_nulls(ca, ca.inner_dtype());
161        };
162
163        match ca.inner_dtype() {
164            DataType::Boolean => Ok(count_boolean_bits(ca).into_series()),
165            dt if dt.is_primitive_numeric() => Ok(sum_list_numerical(ca, dt)),
166            dt => sum_with_nulls(ca, dt),
167        }
168    }
169
170    fn lst_mean(&self) -> Series {
171        let ca = self.as_list();
172
173        if has_inner_nulls(ca) {
174            return sum_mean::mean_with_nulls(ca);
175        };
176
177        match ca.inner_dtype() {
178            dt if dt.is_primitive_numeric() => mean_list_numerical(ca, dt),
179            _ => sum_mean::mean_with_nulls(ca),
180        }
181    }
182
183    fn lst_median(&self) -> Series {
184        let ca = self.as_list();
185        dispersion::median_with_nulls(ca)
186    }
187
188    fn lst_std(&self, ddof: u8) -> Series {
189        let ca = self.as_list();
190        dispersion::std_with_nulls(ca, ddof)
191    }
192
193    fn lst_var(&self, ddof: u8) -> PolarsResult<Series> {
194        let ca = self.as_list();
195        dispersion::var_with_nulls(ca, ddof)
196    }
197
198    fn same_type(&self, out: ListChunked) -> ListChunked {
199        let ca = self.as_list();
200        let dtype = ca.dtype();
201        if out.dtype() != dtype {
202            out.cast(ca.dtype()).unwrap().list().unwrap().clone()
203        } else {
204            out
205        }
206    }
207
208    fn lst_sort(&self, options: SortOptions) -> PolarsResult<ListChunked> {
209        let ca = self.as_list();
210        // SAFETY: `sort_with`` doesn't change the dtype
211        let out = unsafe { ca.try_apply_amortized_same_type(|s| s.as_ref().sort_with(options))? };
212        Ok(self.same_type(out))
213    }
214
215    fn lst_arg_min(&self) -> IdxCa {
216        let ca = self.as_list();
217        ca.apply_amortized_generic(|opt_s| {
218            opt_s.and_then(|s| s.as_ref().arg_min().map(|idx| idx as IdxSize))
219        })
220    }
221
222    fn lst_arg_max(&self) -> IdxCa {
223        let ca = self.as_list();
224        ca.apply_amortized_generic(|opt_s| {
225            opt_s.and_then(|s| s.as_ref().arg_max().map(|idx| idx as IdxSize))
226        })
227    }
228
229    #[cfg(feature = "diff")]
230    fn lst_diff(&self, n: i64, null_behavior: NullBehavior) -> PolarsResult<ListChunked> {
231        let ca = self.as_list();
232        ca.try_apply_amortized(|s| diff(s.as_ref(), n, null_behavior))
233    }
234
235    fn lst_shift(&self, periods: &Column) -> PolarsResult<ListChunked> {
236        let ca = self.as_list();
237        let periods_s = periods.cast(&DataType::Int64)?;
238        let periods = periods_s.i64()?;
239
240        polars_ensure!(
241            ca.len() == periods.len() || ca.len() == 1 || periods.len() == 1,
242            length_mismatch = "list.shift",
243            ca.len(),
244            periods.len()
245        );
246
247        let target_len = periods.len();
248        if ca.len() == 1 && target_len > 1 {
249            let single_list = ca.get_as_series(0);
250            let out = shift_broadcast_list(
251                single_list,
252                periods,
253                target_len,
254                ca.name().clone(),
255                ca.inner_dtype(),
256            );
257            return Ok(self.same_type(out));
258        }
259
260        let out = match periods.len() {
261            1 => {
262                if let Some(periods) = periods.get(0) {
263                    // SAFETY: `shift` doesn't change the dtype
264                    unsafe { ca.apply_amortized_same_type(|s| s.as_ref().shift(periods)) }
265                } else {
266                    ListChunked::full_null_with_dtype(ca.name().clone(), ca.len(), ca.inner_dtype())
267                }
268            },
269            _ => ca.zip_and_apply_amortized(periods, |opt_s, opt_periods| {
270                match (opt_s, opt_periods) {
271                    (Some(s), Some(periods)) => Some(s.as_ref().shift(periods)),
272                    _ => None,
273                }
274            }),
275        };
276        Ok(self.same_type(out))
277    }
278
279    fn lst_lengths(&self) -> IdxCa {
280        let ca = self.as_list();
281
282        let ca_validity = ca.rechunk_validity();
283
284        if ca_validity.as_ref().is_some_and(|x| x.set_bits() == 0) {
285            return IdxCa::full_null(ca.name().clone(), ca.len());
286        }
287
288        let mut lengths = Vec::with_capacity(ca.len());
289        ca.downcast_iter().for_each(|arr| {
290            let offsets = arr.offsets().as_slice();
291            let mut last = offsets[0];
292            for o in &offsets[1..] {
293                lengths.push((*o - last) as IdxSize);
294                last = *o;
295            }
296        });
297
298        let arr = IdxArr::from_vec(lengths).with_validity(ca_validity);
299        IdxCa::with_chunk(ca.name().clone(), arr)
300    }
301
302    /// Get the value by index in the sublists.
303    /// So index `0` would return the first item of every sublist
304    /// and index `-1` would return the last item of every sublist
305    /// if an index is out of bounds, it will return a `None`.
306    fn lst_get(&self, idx: i64, null_on_oob: bool) -> PolarsResult<Series> {
307        let ca = self.as_list();
308        if !null_on_oob && ca.downcast_iter().any(|arr| index_is_oob(arr, idx)) {
309            polars_bail!(ComputeError: "get index is out of bounds");
310        }
311
312        let chunks = ca
313            .downcast_iter()
314            .map(|arr| sublist_get(arr, idx))
315            .collect::<Vec<_>>();
316
317        let s = Series::try_from((ca.name().clone(), chunks)).unwrap();
318        // SAFETY: every element in list has dtype equal to its inner type
319        unsafe { s.from_physical_unchecked(ca.inner_dtype()) }
320    }
321
322    #[cfg(feature = "list_gather")]
323    fn lst_gather_every(&self, n: &IdxCa, offset: &IdxCa) -> PolarsResult<Series> {
324        let list_ca = self.as_list();
325        let out = match (n.len(), offset.len()) {
326            (1, 1) => match (n.get(0), offset.get(0)) {
327                (Some(n), Some(offset)) => unsafe {
328                    // SAFETY: `gather_every` doesn't change the dtype
329                    list_ca.try_apply_amortized_same_type(|s| {
330                        s.as_ref().gather_every(n as usize, offset as usize)
331                    })?
332                },
333                _ => ListChunked::full_null_with_dtype(
334                    list_ca.name().clone(),
335                    list_ca.len(),
336                    list_ca.inner_dtype(),
337                ),
338            },
339            (1, len_offset) if len_offset == list_ca.len() => {
340                if let Some(n) = n.get(0) {
341                    list_ca.try_zip_and_apply_amortized(offset, |opt_s, opt_offset| {
342                        match (opt_s, opt_offset) {
343                            (Some(s), Some(offset)) => {
344                                Ok(Some(s.as_ref().gather_every(n as usize, offset as usize)?))
345                            },
346                            _ => Ok(None),
347                        }
348                    })?
349                } else {
350                    ListChunked::full_null_with_dtype(
351                        list_ca.name().clone(),
352                        list_ca.len(),
353                        list_ca.inner_dtype(),
354                    )
355                }
356            },
357            (len_n, 1) if len_n == list_ca.len() => {
358                if let Some(offset) = offset.get(0) {
359                    list_ca.try_zip_and_apply_amortized(n, |opt_s, opt_n| match (opt_s, opt_n) {
360                        (Some(s), Some(n)) => {
361                            Ok(Some(s.as_ref().gather_every(n as usize, offset as usize)?))
362                        },
363                        _ => Ok(None),
364                    })?
365                } else {
366                    ListChunked::full_null_with_dtype(
367                        list_ca.name().clone(),
368                        list_ca.len(),
369                        list_ca.inner_dtype(),
370                    )
371                }
372            },
373            (len_n, len_offset) if len_n == len_offset && len_n == list_ca.len() => list_ca
374                .try_binary_zip_and_apply_amortized(
375                    n,
376                    offset,
377                    |opt_s, opt_n, opt_offset| match (opt_s, opt_n, opt_offset) {
378                        (Some(s), Some(n), Some(offset)) => {
379                            Ok(Some(s.as_ref().gather_every(n as usize, offset as usize)?))
380                        },
381                        _ => Ok(None),
382                    },
383                )?,
384            _ => {
385                polars_bail!(
386                    ComputeError: "The lengths of `n` and `offset` should be 1 or equal to the length of list (got `n`: {}, `offset`: {}, list: {}).",
387                    n.len(), offset.len(), list_ca.len()
388                )
389            },
390        };
391        Ok(out.into_series())
392    }
393
394    #[cfg(feature = "list_drop_nulls")]
395    fn lst_drop_nulls(&self) -> ListChunked {
396        let list_ca = self.as_list();
397
398        // SAFETY: `drop_nulls` doesn't change the dtype
399        unsafe { list_ca.apply_amortized_same_type(|s| s.as_ref().drop_nulls()) }
400    }
401
402    #[cfg(feature = "list_sample")]
403    fn lst_sample_n(
404        &self,
405        n: &Series,
406        with_replacement: bool,
407        shuffle: Option<bool>,
408        seed: Option<u64>,
409    ) -> PolarsResult<ListChunked> {
410        let ca = self.as_list();
411
412        let n_s = n.strict_cast(&IDX_DTYPE)?;
413        let n = n_s.idx()?;
414
415        polars_ensure!(
416            ca.len() == n.len() || ca.len() == 1 || n.len() == 1,
417            length_mismatch = "list.sample(n)",
418            ca.len(),
419            n.len()
420        );
421
422        let target_len = n.len();
423        if ca.len() == 1 && target_len > 1 {
424            let single_list = ca.get_as_series(0);
425            let out = sample_n_broadcast_list(
426                single_list,
427                n,
428                with_replacement,
429                shuffle,
430                seed,
431                target_len,
432                ca.name().clone(),
433                ca.inner_dtype(),
434            )?;
435            return Ok(self.same_type(out));
436        }
437
438        let out = match n.len() {
439            1 => {
440                if let Some(n) = n.get(0) {
441                    unsafe {
442                        // SAFETY: `sample_n` doesn't change the dtype
443                        ca.try_apply_amortized_same_type(|s| {
444                            s.as_ref()
445                                .sample_n(n as usize, with_replacement, shuffle, seed)
446                        })
447                    }
448                } else {
449                    Ok(ListChunked::full_null_with_dtype(
450                        ca.name().clone(),
451                        ca.len(),
452                        ca.inner_dtype(),
453                    ))
454                }
455            },
456            _ => ca.try_zip_and_apply_amortized(n, |opt_s, opt_n| match (opt_s, opt_n) {
457                (Some(s), Some(n)) => s
458                    .as_ref()
459                    .sample_n(n as usize, with_replacement, shuffle, seed)
460                    .map(Some),
461                _ => Ok(None),
462            }),
463        };
464        out.map(|ok| self.same_type(ok))
465    }
466
467    #[cfg(feature = "list_sample")]
468    fn lst_sample_fraction(
469        &self,
470        fraction: &Series,
471        with_replacement: bool,
472        shuffle: Option<bool>,
473        seed: Option<u64>,
474    ) -> PolarsResult<ListChunked> {
475        let ca = self.as_list();
476
477        let fraction_s = fraction.cast(&DataType::Float64)?;
478        let fraction = fraction_s.f64()?;
479
480        if !with_replacement {
481            for frac in fraction.iter().flatten() {
482                polars_ensure!(
483                    (0.0..=1.0).contains(&frac),
484                    ComputeError: "fraction must be between 0.0 and 1.0, got: {}", frac
485                )
486            }
487        }
488
489        polars_ensure!(
490            ca.len() == fraction.len() || ca.len() == 1 || fraction.len() == 1,
491            length_mismatch = "list.sample(fraction)",
492            ca.len(),
493            fraction.len()
494        );
495
496        let target_len = fraction.len();
497        if ca.len() == 1 && target_len > 1 {
498            let single_list = ca.get_as_series(0);
499            let out = sample_frac_broadcast_list(
500                single_list,
501                fraction,
502                with_replacement,
503                shuffle,
504                seed,
505                target_len,
506                ca.name().clone(),
507                ca.inner_dtype(),
508            )?;
509            return Ok(self.same_type(out));
510        }
511
512        let out = match fraction.len() {
513            1 => {
514                if let Some(fraction) = fraction.get(0) {
515                    unsafe {
516                        // SAFETY: `sample_n` doesn't change the dtype
517                        ca.try_apply_amortized_same_type(|s| {
518                            let n = (s.as_ref().len() as f64 * fraction) as usize;
519                            s.as_ref().sample_n(n, with_replacement, shuffle, seed)
520                        })
521                    }
522                } else {
523                    Ok(ListChunked::full_null_with_dtype(
524                        ca.name().clone(),
525                        ca.len(),
526                        ca.inner_dtype(),
527                    ))
528                }
529            },
530            _ => ca.try_zip_and_apply_amortized(fraction, |opt_s, opt_n| match (opt_s, opt_n) {
531                (Some(s), Some(fraction)) => {
532                    let n = (s.as_ref().len() as f64 * fraction) as usize;
533                    s.as_ref()
534                        .sample_n(n, with_replacement, shuffle, seed)
535                        .map(Some)
536                },
537                _ => Ok(None),
538            }),
539        };
540        out.map(|ok| self.same_type(ok))
541    }
542
543    fn lst_concat(&self, other: &[Column]) -> PolarsResult<ListChunked> {
544        let ca = self.as_list();
545        if other.is_empty() {
546            return Ok(ca.clone());
547        }
548
549        let mut other = other.to_vec();
550        let mut inner_super_type = ca.inner_dtype().clone();
551
552        for s in &other {
553            match s.dtype() {
554                DataType::List(inner_type) => {
555                    inner_super_type = try_get_supertype(&inner_super_type, inner_type)?;
556                },
557                dt => {
558                    inner_super_type = try_get_supertype(&inner_super_type, dt)?;
559                },
560            }
561        }
562
563        // cast lhs and rhs
564        let dtype = &DataType::List(Box::new(inner_super_type.clone()));
565        let ca = ca.cast(dtype)?;
566        cast_rhs(&mut other, &inner_super_type, dtype)?;
567
568        other.insert(0, ca.into_column());
569        let out = concat_list(&other, dtype)?;
570        Ok(out.list()?.clone())
571    }
572}
573
574impl ListNameSpaceImpl for ListChunked {}
575
576pub fn slice_broadcast_list(
577    single_list: Option<Series>,
578    offsets: &Int64Chunked,
579    lengths: &Int64Chunked,
580    target_len: usize,
581    name: PlSmallStr,
582    inner_dtype: &DataType,
583) -> ListChunked {
584    debug_assert!(target_len == offsets.len().max(lengths.len()));
585
586    let Some(single_list) = single_list else {
587        return ListChunked::full_null_with_dtype(name, target_len, inner_dtype);
588    };
589
590    let iter = (0..target_len).map(|index| {
591        let opt_offset = offsets.get(if offsets.len() == 1 { 0 } else { index });
592        let opt_length = lengths.get(if lengths.len() == 1 { 0 } else { index });
593        let offset = opt_offset?;
594        Some(match Slice::new(offset, opt_length.map(|l| l as usize)) {
595            Some(slice) => single_list.slice(slice),
596            None => single_list.clone(),
597        })
598    });
599
600    let mut out: ListChunked = iter.collect_trusted();
601    out.rename(name);
602    out
603}
604
605fn shift_broadcast_list(
606    single_list: Option<Series>,
607    periods: &Int64Chunked,
608    target_len: usize,
609    name: PlSmallStr,
610    inner_dtype: &DataType,
611) -> ListChunked {
612    debug_assert!(target_len == periods.len());
613
614    let Some(single_list) = single_list else {
615        return ListChunked::full_null_with_dtype(name, target_len, inner_dtype);
616    };
617
618    let iter = (0..target_len).map(|index| {
619        let opt_period = periods.get(index);
620        opt_period.map(|period| single_list.shift(period))
621    });
622
623    let mut out: ListChunked = iter.collect_trusted();
624    out.rename(name);
625    out
626}
627
628#[cfg(feature = "list_sample")]
629#[allow(clippy::too_many_arguments)]
630fn sample_n_broadcast_list(
631    single_list: Option<Series>,
632    n: &IdxCa,
633    with_replacement: bool,
634    shuffle: Option<bool>,
635    seed: Option<u64>,
636    target_len: usize,
637    name: PlSmallStr,
638    inner_dtype: &DataType,
639) -> PolarsResult<ListChunked> {
640    debug_assert!(target_len == n.len());
641
642    let Some(single_list) = single_list else {
643        return Ok(ListChunked::full_null_with_dtype(
644            name,
645            target_len,
646            inner_dtype,
647        ));
648    };
649
650    let mut out: ListChunked = (0..target_len)
651        .map(|index| -> PolarsResult<Option<Series>> {
652            match n.get(index) {
653                Some(n_val) => single_list
654                    .sample_n(n_val as usize, with_replacement, shuffle, seed)
655                    .map(Some),
656                None => Ok(None),
657            }
658        })
659        .collect::<PolarsResult<_>>()?;
660
661    out.rename(name);
662    Ok(out)
663}
664
665#[cfg(feature = "list_sample")]
666#[allow(clippy::too_many_arguments)]
667fn sample_frac_broadcast_list(
668    single_list: Option<Series>,
669    fraction: &Float64Chunked,
670    with_replacement: bool,
671    shuffle: Option<bool>,
672    seed: Option<u64>,
673    target_len: usize,
674    name: PlSmallStr,
675    inner_dtype: &DataType,
676) -> PolarsResult<ListChunked> {
677    debug_assert!(target_len == fraction.len());
678
679    let Some(single_list) = single_list else {
680        return Ok(ListChunked::full_null_with_dtype(
681            name,
682            target_len,
683            inner_dtype,
684        ));
685    };
686
687    let mut out: ListChunked = (0..target_len)
688        .map(|index| -> PolarsResult<Option<Series>> {
689            match fraction.get(index) {
690                Some(frac_val) => single_list
691                    .sample_frac(frac_val, with_replacement, shuffle, seed)
692                    .map(Some),
693                None => Ok(None),
694            }
695        })
696        .collect::<PolarsResult<_>>()?;
697
698    out.rename(name);
699    Ok(out)
700}