Skip to main content

polars_core/frame/row/
transpose.rs

1use std::borrow::Cow;
2
3use either::Either;
4
5use super::*;
6
7impl DataFrame {
8    pub(crate) fn transpose_from_dtype(
9        &self,
10        dtype: &DataType,
11        keep_names_as: Option<PlSmallStr>,
12        names_out: &[PlSmallStr],
13    ) -> PolarsResult<DataFrame> {
14        let new_width = self.height();
15        let new_height = self.width();
16        // Allocate space for the transposed columns, putting the "row names" first if needed
17        let mut cols_t = match keep_names_as {
18            None => Vec::<Column>::with_capacity(new_width),
19            Some(name) => {
20                let mut tmp = Vec::<Column>::with_capacity(new_width + 1);
21                tmp.push(
22                    StringChunked::from_iter_values(
23                        name,
24                        self.get_column_names_owned().into_iter(),
25                    )
26                    .into_column(),
27                );
28                tmp
29            },
30        };
31
32        let cols = self.columns();
33        match dtype {
34            #[cfg(feature = "dtype-i8")]
35            DataType::Int8 => numeric_transpose::<Int8Type>(cols, names_out, &mut cols_t),
36            #[cfg(feature = "dtype-i16")]
37            DataType::Int16 => numeric_transpose::<Int16Type>(cols, names_out, &mut cols_t),
38            DataType::Int32 => numeric_transpose::<Int32Type>(cols, names_out, &mut cols_t),
39            DataType::Int64 => numeric_transpose::<Int64Type>(cols, names_out, &mut cols_t),
40            #[cfg(feature = "dtype-u8")]
41            DataType::UInt8 => numeric_transpose::<UInt8Type>(cols, names_out, &mut cols_t),
42            #[cfg(feature = "dtype-u16")]
43            DataType::UInt16 => numeric_transpose::<UInt16Type>(cols, names_out, &mut cols_t),
44            DataType::UInt32 => numeric_transpose::<UInt32Type>(cols, names_out, &mut cols_t),
45            DataType::UInt64 => numeric_transpose::<UInt64Type>(cols, names_out, &mut cols_t),
46            DataType::Float32 => numeric_transpose::<Float32Type>(cols, names_out, &mut cols_t),
47            DataType::Float64 => numeric_transpose::<Float64Type>(cols, names_out, &mut cols_t),
48            #[cfg(feature = "object")]
49            DataType::Object(_) => {
50                // this requires to support `Object` in Series::iter which we don't yet
51                polars_bail!(InvalidOperation: "Object dtype not supported in 'transpose'")
52            },
53            _ => {
54                let phys_dtype = dtype.to_physical();
55                let mut buffers = (0..new_width)
56                    .map(|_| {
57                        let buf: AnyValueBufferTrusted = (&phys_dtype, new_height).into();
58                        buf
59                    })
60                    .collect::<Vec<_>>();
61
62                let columns = self
63                    .materialized_column_iter()
64                    // first cast to supertype before casting to physical to ensure units are correct
65                    .map(|s| Ok(s.cast(dtype)?.to_physical_repr().into_owned()))
66                    .collect::<PolarsResult<Vec<_>>>()?;
67
68                // this is very expensive. A lot of cache misses here.
69                // This is the part that is performance critical.
70                for series in &columns {
71                    polars_ensure!(
72                        series.dtype() == &phys_dtype,
73                        ComputeError: "cannot transpose with supertype: {}", dtype
74                    );
75                    for (av, buf) in series.iter().zip(buffers.iter_mut()) {
76                        // SAFETY: we checked the type and we borrow
77                        unsafe {
78                            buf.add_unchecked_borrowed_physical(&av);
79                        }
80                    }
81                }
82                cols_t.extend(buffers.into_iter().zip(names_out).map(|(buf, name)| {
83                    // SAFETY: the buffer holds the physical values of `dtype`.
84                    let mut s =
85                        unsafe { buf.into_series().from_physical_unchecked(dtype).unwrap() };
86                    s.rename(name.clone());
87                    s.into()
88                }));
89            },
90        };
91
92        DataFrame::new(new_height, cols_t)
93    }
94
95    pub fn transpose(
96        &mut self,
97        keep_names_as: Option<&str>,
98        new_col_names: Option<Either<String, Vec<String>>>,
99    ) -> PolarsResult<DataFrame> {
100        let new_col_names = match new_col_names {
101            None => None,
102            Some(Either::Left(v)) => Some(Either::Left(v.into())),
103            Some(Either::Right(v)) => Some(Either::Right(
104                v.into_iter().map(Into::into).collect::<Vec<_>>(),
105            )),
106        };
107
108        self.transpose_impl(keep_names_as, new_col_names)
109    }
110    /// Transpose a DataFrame. This is a very expensive operation.
111    pub fn transpose_impl(
112        &mut self,
113        keep_names_as: Option<&str>,
114        new_col_names: Option<Either<PlSmallStr, Vec<PlSmallStr>>>,
115    ) -> PolarsResult<DataFrame> {
116        // We must iterate columns as [`AnyValue`], so we must be contiguous.
117        self.rechunk_mut_par();
118
119        let mut df = Cow::Borrowed(self); // Can't use self because we might drop a name column
120        let names_out = match new_col_names {
121            None => (0..self.height())
122                .map(|i| format_pl_smallstr!("column_{i}"))
123                .collect(),
124            Some(cn) => match cn {
125                Either::Left(name) => {
126                    let new_names = self.column(name.as_str()).and_then(|x| x.str())?;
127                    polars_ensure!(new_names.null_count() == 0, ComputeError: "Column with new names can't have null values");
128                    df = Cow::Owned(self.drop(name.as_str())?);
129                    new_names.no_null_iter().map(PlSmallStr::from_str).collect()
130                },
131                Either::Right(names) => {
132                    polars_ensure!(
133                        names.len() == self.height(),
134                        ShapeMismatch: "Length of new column names must be the same as the row count (got {} names for {} rows)",
135                        names.len(), self.height()
136                    );
137                    names
138                },
139            },
140        };
141        if let Some(cn) = keep_names_as {
142            // Check that the column name we're using for the original column names is unique before
143            // wasting time transposing
144            polars_ensure!(names_out.iter().all(|a| a.as_str() != cn), Duplicate: "{} is already in output column names", cn)
145        }
146        let dtype = df.get_supertype().unwrap_or(Ok(DataType::Null))?;
147        df.transpose_from_dtype(&dtype, keep_names_as.map(PlSmallStr::from_str), &names_out)
148    }
149}
150
151#[inline]
152unsafe fn add_value<T: NumericNative>(
153    values_buf_ptr: usize,
154    col_idx: usize,
155    row_idx: usize,
156    value: T,
157) {
158    let vec_ref: &mut Vec<Vec<T>> = &mut *(values_buf_ptr as *mut Vec<Vec<T>>);
159    let column = vec_ref.get_unchecked_mut(col_idx);
160    let el_ptr = column.as_mut_ptr();
161    *el_ptr.add(row_idx) = value;
162}
163
164// This just fills a pre-allocated mutable series vector, which may have a name column.
165// Nothing is returned and the actual DataFrame is constructed above.
166pub(super) fn numeric_transpose<T: PolarsNumericType>(
167    cols: &[Column],
168    names_out: &[PlSmallStr],
169    cols_t: &mut Vec<Column>,
170) {
171    let new_width = cols[0].len();
172    let new_height = cols.len();
173
174    let has_nulls = cols.iter().any(|s| s.null_count() > 0);
175
176    let mut values_buf: Vec<Vec<T::Native>> = (0..new_width)
177        .map(|_| Vec::with_capacity(new_height))
178        .collect();
179    let mut validity_buf: Vec<_> = if has_nulls {
180        // we first use bools instead of bits, because we can access these in parallel without aliasing
181        (0..new_width).map(|_| vec![true; new_height]).collect()
182    } else {
183        (0..new_width).map(|_| vec![]).collect()
184    };
185
186    // work with *mut pointers because we it is UB write to &refs.
187    let values_buf_ptr = &mut values_buf as *mut Vec<Vec<T::Native>> as usize;
188    let validity_buf_ptr = &mut validity_buf as *mut Vec<Vec<bool>> as usize;
189
190    RAYON.install(|| {
191        cols.iter()
192            .map(Column::as_materialized_series)
193            .enumerate()
194            .for_each(|(row_idx, s)| {
195                let s = s.cast(&T::get_static_dtype()).unwrap();
196                let ca = s.unpack::<T>().unwrap();
197
198                // SAFETY:
199                // we access in parallel, but every access is unique, so we don't break aliasing rules
200                // we also ensured we allocated enough memory, so we never reallocate and thus
201                // the pointers remain valid.
202                if has_nulls {
203                    for (col_idx, opt_v) in ca.iter().enumerate() {
204                        match opt_v {
205                            None => unsafe {
206                                let validity_vec: &mut Vec<Vec<bool>> =
207                                    &mut *(validity_buf_ptr as *mut Vec<Vec<bool>>);
208                                let column = validity_vec.get_unchecked_mut(col_idx);
209                                let el_ptr = column.as_mut_ptr();
210                                *el_ptr.add(row_idx) = false;
211                                // we must initialize this memory otherwise downstream code
212                                // might access uninitialized memory when the masked out values
213                                // are changed.
214                                add_value(values_buf_ptr, col_idx, row_idx, T::Native::default());
215                            },
216                            Some(v) => unsafe {
217                                add_value(values_buf_ptr, col_idx, row_idx, v);
218                            },
219                        }
220                    }
221                } else {
222                    for (col_idx, v) in ca.into_no_null_iter().enumerate() {
223                        unsafe {
224                            let column: &mut Vec<Vec<T::Native>> =
225                                &mut *(values_buf_ptr as *mut Vec<Vec<T::Native>>);
226                            let el_ptr = column.get_unchecked_mut(col_idx).as_mut_ptr();
227                            *el_ptr.add(row_idx) = v;
228                        }
229                    }
230                }
231            })
232    });
233
234    let par_iter = values_buf
235        .into_par_iter()
236        .zip(validity_buf)
237        .zip(names_out)
238        .map(|((mut values, validity), name)| {
239            // SAFETY:
240            // all values are written we can now set len
241            unsafe {
242                values.set_len(new_height);
243            }
244
245            let validity = if has_nulls {
246                let validity = Bitmap::from_trusted_len_iter(validity.iter().copied());
247                if validity.unset_bits() > 0 {
248                    Some(validity)
249                } else {
250                    None
251                }
252            } else {
253                None
254            };
255
256            let arr = PrimitiveArray::<T::Native>::new(
257                T::get_static_dtype().to_arrow(CompatLevel::newest()),
258                values.into(),
259                validity,
260            );
261            ChunkedArray::<T>::with_chunk(name.clone(), arr).into_column()
262        });
263    RAYON.install(|| cols_t.par_extend(par_iter));
264}
265
266#[cfg(test)]
267mod test {
268    use super::*;
269
270    #[test]
271    fn test_transpose() -> PolarsResult<()> {
272        let mut df = df![
273            "a" => [1, 2, 3],
274            "b" => [10, 20, 30],
275        ]?;
276
277        let out = df.transpose(None, None)?;
278        let expected = df![
279            "column_0" => [1, 10],
280            "column_1" => [2, 20],
281            "column_2" => [3, 30],
282
283        ]?;
284        assert!(out.equals_missing(&expected));
285
286        let mut df = df![
287            "a" => [Some(1), None, Some(3)],
288            "b" => [Some(10), Some(20), None],
289        ]?;
290        let out = df.transpose(None, None)?;
291        let expected = df![
292            "column_0" => [1, 10],
293            "column_1" => [None, Some(20)],
294            "column_2" => [Some(3), None],
295
296        ]?;
297        assert!(out.equals_missing(&expected));
298
299        let mut df = df![
300            "a" => ["a", "b", "c"],
301            "b" => [Some(10), Some(20), None],
302        ]?;
303        let out = df.transpose(None, None)?;
304        let expected = df![
305            "column_0" => ["a", "10"],
306            "column_1" => ["b", "20"],
307            "column_2" => [Some("c"), None],
308
309        ]?;
310        assert!(out.equals_missing(&expected));
311        Ok(())
312    }
313}