1use std::fmt::Write;
2
3use polars_compute::gather::sublist::list::{index_is_oob, sublist_get};
4#[cfg(feature = "diff")]
5use polars_core::series::ops::NullBehavior;
6use polars_core::utils::{CustomIterTools, try_get_supertype};
7
8use super::*;
9use crate::chunked_array::list::min_max::{list_max_function, list_min_function};
10use crate::chunked_array::list::sum_mean::sum_with_nulls;
11#[cfg(feature = "diff")]
12use crate::prelude::diff;
13use crate::prelude::list::sum_mean::{mean_list_numerical, sum_list_numerical};
14use crate::series::ArgAgg;
15use crate::series::concat_list::concat_list;
16
17pub(super) fn has_inner_nulls(ca: &ListChunked) -> bool {
18 for arr in ca.downcast_iter() {
19 if arr.values().null_count() > 0 {
20 return true;
21 }
22 }
23 false
24}
25
26fn cast_rhs(other: &mut [Column], inner_type: &DataType, dtype: &DataType) -> PolarsResult<()> {
27 for s in other.iter_mut() {
28 if !matches!(s.dtype(), DataType::List(_)) {
30 *s = s.cast(inner_type)?
31 }
32 if !matches!(s.dtype(), DataType::List(_)) && s.dtype() == inner_type {
33 *s = s
35 .reshape_list(&[ReshapeDimension::Infer, ReshapeDimension::new_dimension(1)])
36 .unwrap();
37 }
38 if s.dtype() != dtype {
39 *s = s.cast(dtype).map_err(|e| {
40 polars_err!(
41 SchemaMismatch:
42 "cannot concat `{}` into a list of `{}`: {}",
43 s.dtype(),
44 dtype,
45 e
46 )
47 })?;
48 }
49 }
50 Ok(())
51}
52
53pub trait ListNameSpaceImpl: AsList {
54 fn lst_join(
57 &self,
58 separator: &StringChunked,
59 ignore_nulls: bool,
60 ) -> PolarsResult<StringChunked> {
61 let ca = self.as_list();
62 match ca.inner_dtype() {
63 DataType::String => match separator.len() {
64 1 => match separator.get(0) {
65 Some(separator) => self.join_literal(separator, ignore_nulls),
66 _ => Ok(StringChunked::full_null(ca.name().clone(), ca.len())),
67 },
68 _ => self.join_many(separator, ignore_nulls),
69 },
70 dt => polars_bail!(op = "`lst.join`", got = dt, expected = "String"),
71 }
72 }
73
74 fn join_literal(&self, separator: &str, ignore_nulls: bool) -> PolarsResult<StringChunked> {
75 let ca = self.as_list();
76 let mut buf = String::with_capacity(128);
78 let mut builder = StringChunkedBuilder::new(ca.name().clone(), ca.len());
79
80 ca.for_each_amortized(|opt_s| {
81 let opt_val = opt_s.and_then(|s| {
82 buf.clear();
84 let ca = s.as_ref().str().unwrap();
85
86 if ca.null_count() != 0 && !ignore_nulls {
87 return None;
88 }
89
90 for arr in ca.downcast_iter() {
91 for val in arr.non_null_values_iter() {
92 buf.write_str(val).unwrap();
93 buf.write_str(separator).unwrap();
94 }
95 }
96
97 Some(&buf[..buf.len().saturating_sub(separator.len())])
100 });
101 builder.append_option(opt_val)
102 });
103 Ok(builder.finish())
104 }
105
106 fn join_many(
107 &self,
108 separator: &StringChunked,
109 ignore_nulls: bool,
110 ) -> PolarsResult<StringChunked> {
111 let ca = self.as_list();
112 let mut buf = String::with_capacity(128);
114 let mut builder = StringChunkedBuilder::new(ca.name().clone(), ca.len());
115 {
116 ca.amortized_iter()
117 .zip(separator.iter())
118 .for_each(|(opt_s, opt_sep)| match opt_sep {
119 Some(separator) => {
120 let opt_val = opt_s.and_then(|s| {
121 buf.clear();
123 let ca = s.as_ref().str().unwrap();
124
125 if ca.null_count() != 0 && !ignore_nulls {
126 return None;
127 }
128
129 for arr in ca.downcast_iter() {
130 for val in arr.non_null_values_iter() {
131 buf.write_str(val).unwrap();
132 buf.write_str(separator).unwrap();
133 }
134 }
135
136 Some(&buf[..buf.len().saturating_sub(separator.len())])
139 });
140 builder.append_option(opt_val)
141 },
142 _ => builder.append_null(),
143 })
144 }
145 Ok(builder.finish())
146 }
147
148 fn lst_max(&self) -> PolarsResult<Series> {
149 list_max_function(self.as_list())
150 }
151
152 fn lst_min(&self) -> PolarsResult<Series> {
153 list_min_function(self.as_list())
154 }
155
156 fn lst_sum(&self) -> PolarsResult<Series> {
157 let ca = self.as_list();
158
159 if has_inner_nulls(ca) {
160 return sum_with_nulls(ca, ca.inner_dtype());
161 };
162
163 match ca.inner_dtype() {
164 DataType::Boolean => Ok(count_boolean_bits(ca).into_series()),
165 dt if dt.is_primitive_numeric() => Ok(sum_list_numerical(ca, dt)),
166 dt => sum_with_nulls(ca, dt),
167 }
168 }
169
170 fn lst_mean(&self) -> Series {
171 let ca = self.as_list();
172
173 if has_inner_nulls(ca) {
174 return sum_mean::mean_with_nulls(ca);
175 };
176
177 match ca.inner_dtype() {
178 dt if dt.is_primitive_numeric() => mean_list_numerical(ca, dt),
179 _ => sum_mean::mean_with_nulls(ca),
180 }
181 }
182
183 fn lst_median(&self) -> Series {
184 let ca = self.as_list();
185 dispersion::median_with_nulls(ca)
186 }
187
188 fn lst_std(&self, ddof: u8) -> Series {
189 let ca = self.as_list();
190 dispersion::std_with_nulls(ca, ddof)
191 }
192
193 fn lst_var(&self, ddof: u8) -> PolarsResult<Series> {
194 let ca = self.as_list();
195 dispersion::var_with_nulls(ca, ddof)
196 }
197
198 fn same_type(&self, out: ListChunked) -> ListChunked {
199 let ca = self.as_list();
200 let dtype = ca.dtype();
201 if out.dtype() != dtype {
202 out.cast(ca.dtype()).unwrap().list().unwrap().clone()
203 } else {
204 out
205 }
206 }
207
208 fn lst_sort(&self, options: SortOptions) -> PolarsResult<ListChunked> {
209 let ca = self.as_list();
210 let out = unsafe { ca.try_apply_amortized_same_type(|s| s.as_ref().sort_with(options))? };
212 Ok(self.same_type(out))
213 }
214
215 fn lst_arg_min(&self) -> IdxCa {
216 let ca = self.as_list();
217 ca.apply_amortized_generic(|opt_s| {
218 opt_s.and_then(|s| s.as_ref().arg_min().map(|idx| idx as IdxSize))
219 })
220 }
221
222 fn lst_arg_max(&self) -> IdxCa {
223 let ca = self.as_list();
224 ca.apply_amortized_generic(|opt_s| {
225 opt_s.and_then(|s| s.as_ref().arg_max().map(|idx| idx as IdxSize))
226 })
227 }
228
229 #[cfg(feature = "diff")]
230 fn lst_diff(&self, n: i64, null_behavior: NullBehavior) -> PolarsResult<ListChunked> {
231 let ca = self.as_list();
232 ca.try_apply_amortized(|s| diff(s.as_ref(), n, null_behavior))
233 }
234
235 fn lst_shift(&self, periods: &Column) -> PolarsResult<ListChunked> {
236 let ca = self.as_list();
237 let periods_s = periods.cast(&DataType::Int64)?;
238 let periods = periods_s.i64()?;
239
240 polars_ensure!(
241 ca.len() == periods.len() || ca.len() == 1 || periods.len() == 1,
242 length_mismatch = "list.shift",
243 ca.len(),
244 periods.len()
245 );
246
247 let target_len = periods.len();
248 if ca.len() == 1 && target_len > 1 {
249 let single_list = ca.get_as_series(0);
250 let out = shift_broadcast_list(
251 single_list,
252 periods,
253 target_len,
254 ca.name().clone(),
255 ca.inner_dtype(),
256 );
257 return Ok(self.same_type(out));
258 }
259
260 let out = match periods.len() {
261 1 => {
262 if let Some(periods) = periods.get(0) {
263 unsafe { ca.apply_amortized_same_type(|s| s.as_ref().shift(periods)) }
265 } else {
266 ListChunked::full_null_with_dtype(ca.name().clone(), ca.len(), ca.inner_dtype())
267 }
268 },
269 _ => ca.zip_and_apply_amortized(periods, |opt_s, opt_periods| {
270 match (opt_s, opt_periods) {
271 (Some(s), Some(periods)) => Some(s.as_ref().shift(periods)),
272 _ => None,
273 }
274 }),
275 };
276 Ok(self.same_type(out))
277 }
278
279 fn lst_lengths(&self) -> IdxCa {
280 let ca = self.as_list();
281
282 let ca_validity = ca.rechunk_validity();
283
284 if ca_validity.as_ref().is_some_and(|x| x.set_bits() == 0) {
285 return IdxCa::full_null(ca.name().clone(), ca.len());
286 }
287
288 let mut lengths = Vec::with_capacity(ca.len());
289 ca.downcast_iter().for_each(|arr| {
290 let offsets = arr.offsets().as_slice();
291 let mut last = offsets[0];
292 for o in &offsets[1..] {
293 lengths.push((*o - last) as IdxSize);
294 last = *o;
295 }
296 });
297
298 let arr = IdxArr::from_vec(lengths).with_validity(ca_validity);
299 IdxCa::with_chunk(ca.name().clone(), arr)
300 }
301
302 fn lst_get(&self, idx: i64, null_on_oob: bool) -> PolarsResult<Series> {
307 let ca = self.as_list();
308 if !null_on_oob && ca.downcast_iter().any(|arr| index_is_oob(arr, idx)) {
309 polars_bail!(ComputeError: "get index is out of bounds");
310 }
311
312 let chunks = ca
313 .downcast_iter()
314 .map(|arr| sublist_get(arr, idx))
315 .collect::<Vec<_>>();
316
317 let s = Series::try_from((ca.name().clone(), chunks)).unwrap();
318 unsafe { s.from_physical_unchecked(ca.inner_dtype()) }
320 }
321
322 #[cfg(feature = "list_gather")]
323 fn lst_gather_every(&self, n: &IdxCa, offset: &IdxCa) -> PolarsResult<Series> {
324 let list_ca = self.as_list();
325 let out = match (n.len(), offset.len()) {
326 (1, 1) => match (n.get(0), offset.get(0)) {
327 (Some(n), Some(offset)) => unsafe {
328 list_ca.try_apply_amortized_same_type(|s| {
330 s.as_ref().gather_every(n as usize, offset as usize)
331 })?
332 },
333 _ => ListChunked::full_null_with_dtype(
334 list_ca.name().clone(),
335 list_ca.len(),
336 list_ca.inner_dtype(),
337 ),
338 },
339 (1, len_offset) if len_offset == list_ca.len() => {
340 if let Some(n) = n.get(0) {
341 list_ca.try_zip_and_apply_amortized(offset, |opt_s, opt_offset| {
342 match (opt_s, opt_offset) {
343 (Some(s), Some(offset)) => {
344 Ok(Some(s.as_ref().gather_every(n as usize, offset as usize)?))
345 },
346 _ => Ok(None),
347 }
348 })?
349 } else {
350 ListChunked::full_null_with_dtype(
351 list_ca.name().clone(),
352 list_ca.len(),
353 list_ca.inner_dtype(),
354 )
355 }
356 },
357 (len_n, 1) if len_n == list_ca.len() => {
358 if let Some(offset) = offset.get(0) {
359 list_ca.try_zip_and_apply_amortized(n, |opt_s, opt_n| match (opt_s, opt_n) {
360 (Some(s), Some(n)) => {
361 Ok(Some(s.as_ref().gather_every(n as usize, offset as usize)?))
362 },
363 _ => Ok(None),
364 })?
365 } else {
366 ListChunked::full_null_with_dtype(
367 list_ca.name().clone(),
368 list_ca.len(),
369 list_ca.inner_dtype(),
370 )
371 }
372 },
373 (len_n, len_offset) if len_n == len_offset && len_n == list_ca.len() => list_ca
374 .try_binary_zip_and_apply_amortized(
375 n,
376 offset,
377 |opt_s, opt_n, opt_offset| match (opt_s, opt_n, opt_offset) {
378 (Some(s), Some(n), Some(offset)) => {
379 Ok(Some(s.as_ref().gather_every(n as usize, offset as usize)?))
380 },
381 _ => Ok(None),
382 },
383 )?,
384 _ => {
385 polars_bail!(
386 ComputeError: "The lengths of `n` and `offset` should be 1 or equal to the length of list (got `n`: {}, `offset`: {}, list: {}).",
387 n.len(), offset.len(), list_ca.len()
388 )
389 },
390 };
391 Ok(out.into_series())
392 }
393
394 #[cfg(feature = "list_drop_nulls")]
395 fn lst_drop_nulls(&self) -> ListChunked {
396 let list_ca = self.as_list();
397
398 unsafe { list_ca.apply_amortized_same_type(|s| s.as_ref().drop_nulls()) }
400 }
401
402 #[cfg(feature = "list_sample")]
403 fn lst_sample_n(
404 &self,
405 n: &Series,
406 with_replacement: bool,
407 shuffle: Option<bool>,
408 seed: Option<u64>,
409 ) -> PolarsResult<ListChunked> {
410 let ca = self.as_list();
411
412 let n_s = n.strict_cast(&IDX_DTYPE)?;
413 let n = n_s.idx()?;
414
415 polars_ensure!(
416 ca.len() == n.len() || ca.len() == 1 || n.len() == 1,
417 length_mismatch = "list.sample(n)",
418 ca.len(),
419 n.len()
420 );
421
422 let target_len = n.len();
423 if ca.len() == 1 && target_len > 1 {
424 let single_list = ca.get_as_series(0);
425 let out = sample_n_broadcast_list(
426 single_list,
427 n,
428 with_replacement,
429 shuffle,
430 seed,
431 target_len,
432 ca.name().clone(),
433 ca.inner_dtype(),
434 )?;
435 return Ok(self.same_type(out));
436 }
437
438 let out = match n.len() {
439 1 => {
440 if let Some(n) = n.get(0) {
441 unsafe {
442 ca.try_apply_amortized_same_type(|s| {
444 s.as_ref()
445 .sample_n(n as usize, with_replacement, shuffle, seed)
446 })
447 }
448 } else {
449 Ok(ListChunked::full_null_with_dtype(
450 ca.name().clone(),
451 ca.len(),
452 ca.inner_dtype(),
453 ))
454 }
455 },
456 _ => ca.try_zip_and_apply_amortized(n, |opt_s, opt_n| match (opt_s, opt_n) {
457 (Some(s), Some(n)) => s
458 .as_ref()
459 .sample_n(n as usize, with_replacement, shuffle, seed)
460 .map(Some),
461 _ => Ok(None),
462 }),
463 };
464 out.map(|ok| self.same_type(ok))
465 }
466
467 #[cfg(feature = "list_sample")]
468 fn lst_sample_fraction(
469 &self,
470 fraction: &Series,
471 with_replacement: bool,
472 shuffle: Option<bool>,
473 seed: Option<u64>,
474 ) -> PolarsResult<ListChunked> {
475 let ca = self.as_list();
476
477 let fraction_s = fraction.cast(&DataType::Float64)?;
478 let fraction = fraction_s.f64()?;
479
480 if !with_replacement {
481 for frac in fraction.iter().flatten() {
482 polars_ensure!(
483 (0.0..=1.0).contains(&frac),
484 ComputeError: "fraction must be between 0.0 and 1.0, got: {}", frac
485 )
486 }
487 }
488
489 polars_ensure!(
490 ca.len() == fraction.len() || ca.len() == 1 || fraction.len() == 1,
491 length_mismatch = "list.sample(fraction)",
492 ca.len(),
493 fraction.len()
494 );
495
496 let target_len = fraction.len();
497 if ca.len() == 1 && target_len > 1 {
498 let single_list = ca.get_as_series(0);
499 let out = sample_frac_broadcast_list(
500 single_list,
501 fraction,
502 with_replacement,
503 shuffle,
504 seed,
505 target_len,
506 ca.name().clone(),
507 ca.inner_dtype(),
508 )?;
509 return Ok(self.same_type(out));
510 }
511
512 let out = match fraction.len() {
513 1 => {
514 if let Some(fraction) = fraction.get(0) {
515 unsafe {
516 ca.try_apply_amortized_same_type(|s| {
518 let n = (s.as_ref().len() as f64 * fraction) as usize;
519 s.as_ref().sample_n(n, with_replacement, shuffle, seed)
520 })
521 }
522 } else {
523 Ok(ListChunked::full_null_with_dtype(
524 ca.name().clone(),
525 ca.len(),
526 ca.inner_dtype(),
527 ))
528 }
529 },
530 _ => ca.try_zip_and_apply_amortized(fraction, |opt_s, opt_n| match (opt_s, opt_n) {
531 (Some(s), Some(fraction)) => {
532 let n = (s.as_ref().len() as f64 * fraction) as usize;
533 s.as_ref()
534 .sample_n(n, with_replacement, shuffle, seed)
535 .map(Some)
536 },
537 _ => Ok(None),
538 }),
539 };
540 out.map(|ok| self.same_type(ok))
541 }
542
543 fn lst_concat(&self, other: &[Column]) -> PolarsResult<ListChunked> {
544 let ca = self.as_list();
545 if other.is_empty() {
546 return Ok(ca.clone());
547 }
548
549 let mut other = other.to_vec();
550 let mut inner_super_type = ca.inner_dtype().clone();
551
552 for s in &other {
553 match s.dtype() {
554 DataType::List(inner_type) => {
555 inner_super_type = try_get_supertype(&inner_super_type, inner_type)?;
556 },
557 dt => {
558 inner_super_type = try_get_supertype(&inner_super_type, dt)?;
559 },
560 }
561 }
562
563 let dtype = &DataType::List(Box::new(inner_super_type.clone()));
565 let ca = ca.cast(dtype)?;
566 cast_rhs(&mut other, &inner_super_type, dtype)?;
567
568 other.insert(0, ca.into_column());
569 let out = concat_list(&other, dtype)?;
570 Ok(out.list()?.clone())
571 }
572}
573
574impl ListNameSpaceImpl for ListChunked {}
575
576pub fn slice_broadcast_list(
577 single_list: Option<Series>,
578 offsets: &Int64Chunked,
579 lengths: &Int64Chunked,
580 target_len: usize,
581 name: PlSmallStr,
582 inner_dtype: &DataType,
583) -> ListChunked {
584 debug_assert!(target_len == offsets.len().max(lengths.len()));
585
586 let Some(single_list) = single_list else {
587 return ListChunked::full_null_with_dtype(name, target_len, inner_dtype);
588 };
589
590 let iter = (0..target_len).map(|index| {
591 let opt_offset = offsets.get(if offsets.len() == 1 { 0 } else { index });
592 let opt_length = lengths.get(if lengths.len() == 1 { 0 } else { index });
593 let offset = opt_offset?;
594 Some(match Slice::new(offset, opt_length.map(|l| l as usize)) {
595 Some(slice) => single_list.slice(slice),
596 None => single_list.clone(),
597 })
598 });
599
600 let mut out: ListChunked = iter.collect_trusted();
601 out.rename(name);
602 out
603}
604
605fn shift_broadcast_list(
606 single_list: Option<Series>,
607 periods: &Int64Chunked,
608 target_len: usize,
609 name: PlSmallStr,
610 inner_dtype: &DataType,
611) -> ListChunked {
612 debug_assert!(target_len == periods.len());
613
614 let Some(single_list) = single_list else {
615 return ListChunked::full_null_with_dtype(name, target_len, inner_dtype);
616 };
617
618 let iter = (0..target_len).map(|index| {
619 let opt_period = periods.get(index);
620 opt_period.map(|period| single_list.shift(period))
621 });
622
623 let mut out: ListChunked = iter.collect_trusted();
624 out.rename(name);
625 out
626}
627
628#[cfg(feature = "list_sample")]
629#[allow(clippy::too_many_arguments)]
630fn sample_n_broadcast_list(
631 single_list: Option<Series>,
632 n: &IdxCa,
633 with_replacement: bool,
634 shuffle: Option<bool>,
635 seed: Option<u64>,
636 target_len: usize,
637 name: PlSmallStr,
638 inner_dtype: &DataType,
639) -> PolarsResult<ListChunked> {
640 debug_assert!(target_len == n.len());
641
642 let Some(single_list) = single_list else {
643 return Ok(ListChunked::full_null_with_dtype(
644 name,
645 target_len,
646 inner_dtype,
647 ));
648 };
649
650 let mut out: ListChunked = (0..target_len)
651 .map(|index| -> PolarsResult<Option<Series>> {
652 match n.get(index) {
653 Some(n_val) => single_list
654 .sample_n(n_val as usize, with_replacement, shuffle, seed)
655 .map(Some),
656 None => Ok(None),
657 }
658 })
659 .collect::<PolarsResult<_>>()?;
660
661 out.rename(name);
662 Ok(out)
663}
664
665#[cfg(feature = "list_sample")]
666#[allow(clippy::too_many_arguments)]
667fn sample_frac_broadcast_list(
668 single_list: Option<Series>,
669 fraction: &Float64Chunked,
670 with_replacement: bool,
671 shuffle: Option<bool>,
672 seed: Option<u64>,
673 target_len: usize,
674 name: PlSmallStr,
675 inner_dtype: &DataType,
676) -> PolarsResult<ListChunked> {
677 debug_assert!(target_len == fraction.len());
678
679 let Some(single_list) = single_list else {
680 return Ok(ListChunked::full_null_with_dtype(
681 name,
682 target_len,
683 inner_dtype,
684 ));
685 };
686
687 let mut out: ListChunked = (0..target_len)
688 .map(|index| -> PolarsResult<Option<Series>> {
689 match fraction.get(index) {
690 Some(frac_val) => single_list
691 .sample_frac(frac_val, with_replacement, shuffle, seed)
692 .map(Some),
693 None => Ok(None),
694 }
695 })
696 .collect::<PolarsResult<_>>()?;
697
698 out.rename(name);
699 Ok(out)
700}