Skip to main content

polars_ops/series/ops/
concat_list.rs

1use polars_arrow::compute::utils::combine_validities_and;
2use polars_compute::horizontal_concat_list::horizontal_concat_list_unchecked;
3use polars_core::prelude::{Column, DataType, IntoColumn, ListChunked};
4use polars_error::{PolarsContext, PolarsResult};
5use polars_utils::broadcast::broadcast_len;
6
7/// Note: The caller must ensure all columns in `args` have the type `dtype`.
8///
9/// # Panics
10/// Panics if
11/// * `args` is empty
12/// * `dtype` is not a `DataType::List`
13pub fn concat_list(args: &[Column], dtype: &DataType) -> PolarsResult<Column> {
14    let DataType::List(inner_dtype) = dtype else {
15        panic!("{}", dtype);
16    };
17
18    let inner_dtype = inner_dtype.as_ref();
19
20    let output_height = broadcast_len(args.iter()).context("concat_list")?;
21    // If there is a `List` column with a single NULL, the output will be entirely NULL.
22    let mut return_all_null = false;
23    // Indicates whether all `arrays` have unit length
24    let mut all_unit_len = true;
25    let mut validities = Vec::with_capacity(args.len());
26
27    let arrays = args
28        .iter()
29        .map(|c| {
30            let len = c.len();
31
32            all_unit_len &= len == 1;
33
34            // Don't expand scalars to height, this is handled by the `horizontal_concat_list` kernel.
35            let s = c.as_materialized_series_maintain_scalar();
36            debug_assert_eq!(s.dtype(), dtype);
37
38            let arr = s.list().unwrap().rechunk();
39            let validity = arr.rechunk_validity();
40
41            return_all_null |= len == 1 && validity.as_ref().is_some_and(|x| !x.get_bit(0));
42
43            // Ignore unit-length validities. If they are non-valid then `return_all_null` will
44            // cause an early return.
45            if let Some(v) = validity.filter(|_| len > 1) {
46                validities.push(v)
47            }
48
49            arr.downcast_as_array().clone()
50        })
51        .collect::<Vec<_>>();
52
53    if return_all_null || output_height == 0 {
54        return Ok(ListChunked::full_null_with_dtype(
55            args[0].name().clone(),
56            output_height,
57            inner_dtype,
58        )
59        .into_column());
60    }
61
62    // Combine validities
63    let outer_validity = validities.into_iter().fold(None, |a, b| {
64        debug_assert_eq!(b.len(), output_height);
65        combine_validities_and(a.as_ref(), Some(&b))
66    });
67
68    // At this point the output height and all arrays should have non-zero length
69    let out = if all_unit_len {
70        // Fast-path for all scalars
71        let arr = unsafe { horizontal_concat_list_unchecked(&arrays, 1, outer_validity) }?;
72
73        let mut out = ListChunked::with_chunk(args[0].name().clone(), arr);
74        unsafe { out.to_logical(inner_dtype.clone()) };
75
76        return Ok(out.into_column().new_from_index(0, output_height));
77    } else {
78        let arr =
79            unsafe { horizontal_concat_list_unchecked(&arrays, output_height, outer_validity) }?;
80
81        let mut out = ListChunked::with_chunk(args[0].name().clone(), arr);
82        unsafe { out.to_logical(inner_dtype.clone()) };
83
84        out.into_column()
85    };
86
87    Ok(out)
88}