polars_core/utils/
series.rs

1use polars_compute::find_validity_mismatch::{
2    find_validity_mismatch, find_validity_mismatch_shallow,
3};
4use polars_compute::gather::take_unchecked;
5
6use crate::prelude::*;
7
8/// Check that two dtypes have the same struct fields: same names, same order, nested in
9/// lists and arrays. Field dtypes are not compared.
10pub fn check_is_valid_struct_cast(
11    input_dtype: &DataType,
12    output_dtype: &DataType,
13    output_name: &PlSmallStr,
14) -> PolarsResult<()> {
15    use DataType as D;
16
17    let err = |msg: &str| -> PolarsError {
18        polars_err!(
19            InvalidOperation:
20            "cast from `{}` to `{}` failed in column '{}': {}\n\n\
21            Ensure that any output struct has the same number of fields as the input, and that all struct field names in the output are present in the input.\n\
22            Use `strict=False` to force the cast, and Polars will select the first n fields from the struct.",
23            input_dtype,
24            output_dtype,
25            output_name,
26            msg,
27        )
28    };
29
30    #[allow(clippy::single_match)]
31    match (input_dtype, output_dtype) {
32        #[cfg(feature = "dtype-struct")]
33        (D::Struct(l_fields), D::Struct(r_fields)) => {
34            if l_fields.len() != r_fields.len() {
35                return Err(err(&format!(
36                    "structs do not have the same number of fields: {} vs {}",
37                    l_fields.len(),
38                    r_fields.len(),
39                )));
40            }
41            for (l, r) in Iterator::zip(l_fields.iter(), r_fields.iter()) {
42                if l.name() != r.name() {
43                    return Err(err(&format!(
44                        "structs field name mismatch: {} vs {}",
45                        l.name(),
46                        r.name()
47                    )));
48                }
49                check_is_valid_struct_cast(l.dtype(), r.dtype(), output_name)?;
50            }
51            Ok(())
52        },
53        (D::List(input_dtype), D::List(output_dtype)) => {
54            check_is_valid_struct_cast(input_dtype, output_dtype, output_name)
55        },
56        #[cfg(feature = "dtype-array")]
57        (D::Array(input_dtype, _), D::Array(output_dtype, _)) => {
58            check_is_valid_struct_cast(input_dtype, output_dtype, output_name)
59        },
60        #[cfg(feature = "dtype-array")]
61        (D::List(input_dtype), D::Array(output_dtype, _))
62        | (D::Array(input_dtype, _), D::List(output_dtype)) => {
63            check_is_valid_struct_cast(input_dtype, output_dtype, output_name)
64        },
65        _ => Ok(()),
66    }
67}
68
69/// Return `input_dtype` with its struct fields reordered and completed as in `output_dtype`,
70/// matched by name.
71///
72/// Matched fields keep their `input_dtype` dtype and missing fields get their `output_dtype`
73/// dtype. A struct with fields that are not in `output_dtype` is left as is. A non-strict cast to
74/// the result, followed by a strict cast to `output_dtype`, aligns struct fields by name while
75/// value conversions stay strict.
76pub fn match_struct_fields(input_dtype: &DataType, output_dtype: &DataType) -> DataType {
77    use DataType as D;
78
79    match (input_dtype, output_dtype) {
80        #[cfg(feature = "dtype-struct")]
81        (D::Struct(l_fields), D::Struct(r_fields)) => {
82            if !l_fields
83                .iter()
84                .all(|l| r_fields.iter().any(|r| r.name() == l.name()))
85            {
86                return input_dtype.clone();
87            }
88            let fields =
89                r_fields
90                    .iter()
91                    .map(|r| match l_fields.iter().find(|l| l.name() == r.name()) {
92                        Some(l) => {
93                            Field::new(r.name().clone(), match_struct_fields(l.dtype(), r.dtype()))
94                        },
95                        None => r.clone(),
96                    });
97            D::Struct(fields.collect())
98        },
99        (D::List(l), D::List(r)) => D::List(Box::new(match_struct_fields(l, r))),
100        #[cfg(feature = "dtype-array")]
101        (D::Array(l, width), D::Array(r, _) | D::List(r)) => {
102            D::Array(Box::new(match_struct_fields(l, r)), *width)
103        },
104        #[cfg(feature = "dtype-array")]
105        (D::List(l), D::Array(r, _)) => D::List(Box::new(match_struct_fields(l, r))),
106        _ => input_dtype.clone(),
107    }
108}
109
110pub fn handle_casting_failures(input: &Series, output: &Series) -> PolarsResult<()> {
111    check_is_valid_struct_cast(input.dtype(), output.dtype(), output.name())?;
112
113    let mut idxs = Vec::new();
114
115    #[cfg(feature = "dtype-map")]
116    let maps_involved = input.dtype().contains_map() || output.dtype().contains_map();
117    #[cfg(not(feature = "dtype-map"))]
118    let maps_involved = false;
119
120    if maps_involved {
121        // Map entries are not positionally comparable with a cast's -- which
122        // `find_validity_mismatch` requires -- because casting merges duplicate keys and
123        // drops the entries that no live row owns. Rows still line up, and strictness holds
124        // below, since a Map is nested, so its key and value child casts run with the same
125        // options.
126        find_validity_mismatch_shallow(
127            input.rechunk_validity().as_ref(),
128            output.rechunk_validity().as_ref(),
129            &mut idxs,
130        );
131    } else {
132        input.find_validity_mismatch(output, &mut idxs);
133    }
134
135    if idxs.is_empty() {
136        return Ok(());
137    }
138
139    let num_failures = idxs.len();
140    let failures = input.take_slice(&idxs[..num_failures.min(10)])?;
141
142    let additional_info = match (input.dtype(), output.dtype()) {
143        (DataType::String, DataType::Date | DataType::Datetime(_, _) | DataType::Time) => {
144            "\n\nYou might want to try:\n\
145            - setting `strict=False` to set values that cannot be converted to `null`\n\
146            - using `str.strptime`, `str.to_date`, `str.to_datetime`, or `str.to_time` and providing a format string"
147        },
148        #[cfg(feature = "dtype-categorical")]
149        (DataType::String, DataType::Enum(_, _)) => {
150            "\n\nEnsure that all values in the input column are present in the categories of the enum datatype."
151        },
152        _ if failures.len() < num_failures => {
153            "\n\nDid not show all failed cases as there were too many."
154        },
155        _ => "",
156    };
157
158    polars_bail!(
159        InvalidOperation:
160        "conversion from `{}` to `{}` failed in column '{}' for {} out of {} values: {}{}",
161        input.dtype(),
162        output.dtype(),
163        output.name(),
164        num_failures,
165        input.len(),
166        failures.fmt_list(),
167        additional_info,
168    )
169}
170
171pub fn handle_array_casting_failures(input: &dyn Array, output: &dyn Array) -> PolarsResult<()> {
172    let mut idxs = Vec::new();
173    find_validity_mismatch(input, output, &mut idxs);
174    if idxs.is_empty() {
175        return Ok(());
176    }
177
178    let num_failures = idxs.len();
179    let failures = PrimitiveArray::with_slice(&idxs[..num_failures.min(10)], |idxs| unsafe {
180        take_unchecked(input, &idxs)
181    });
182
183    polars_bail!(
184        InvalidOperation:
185        "conversion from `{}` to `{}` failed for {} out of {} values: {}",
186        DataType::from_arrow(input.dtype(), None),
187        DataType::from_arrow(output.dtype(), None),
188        num_failures,
189        input.len(),
190        Series::try_from((PlSmallStr::EMPTY, failures))?,
191    )
192}
polars_core/utils/series.rs

polars_core/utils/
series.rs
