Skip to main content

polars_lazy/frame/
mod.rs

1//! Lazy variant of a [DataFrame].
2#[cfg(feature = "python")]
3mod python;
4
5mod cached_arenas;
6mod err;
7#[cfg(not(target_arch = "wasm32"))]
8mod exitable;
9
10use std::num::NonZeroUsize;
11use std::sync::mpsc::{Receiver, sync_channel};
12use std::sync::{Arc, Mutex};
13
14pub use anonymous_scan::*;
15#[cfg(feature = "csv")]
16pub use csv::*;
17#[cfg(not(target_arch = "wasm32"))]
18pub use exitable::*;
19pub use file_list_reader::*;
20#[cfg(feature = "json")]
21pub use ndjson::*;
22#[cfg(feature = "parquet")]
23pub use parquet::*;
24use polars_compute::rolling::QuantileMethod;
25use polars_core::error::feature_gated;
26#[cfg(feature = "pivot")]
27use polars_core::frame::PivotColumnNaming;
28use polars_core::prelude::*;
29use polars_core::query_result::QueryResult;
30#[cfg(feature = "is_between")]
31use polars_defs::expr::ClosedInterval;
32use polars_defs::join::{
33    JoinArgs, JoinBuildSide, JoinCoalesce, JoinType, JoinValidation, MaintainOrderJoin,
34};
35use polars_defs::time::group_by::{DynamicGroupOptions, RollingGroupOptions};
36use polars_io::RowIndex;
37use polars_mem_engine::scan_predicate::functions::apply_scan_predicate_to_scan_ir;
38use polars_mem_engine::{Executor, create_multiple_physical_plans, create_physical_plan};
39use polars_observer::{PlannedQuery, QueryObserver};
40pub use polars_plan::frame::{AllowedOptimizations, OptFlags};
41use polars_plan::plans::ExecutionHooks;
42use polars_plan::prelude::ir_plan_to_description;
43use polars_utils::pl_str::PlSmallStr;
44use polars_utils::slice_enum::Slice;
45
46use crate::frame::cached_arenas::CachedArena;
47use crate::prelude::*;
48
49pub trait IntoLazy {
50    fn lazy(self) -> LazyFrame;
51}
52
53impl IntoLazy for DataFrame {
54    /// Convert the `DataFrame` into a `LazyFrame`
55    fn lazy(self) -> LazyFrame {
56        let lp = DslBuilder::from_existing_df(self).build();
57        LazyFrame {
58            logical_plan: lp,
59            opt_state: Default::default(),
60            cached_arena: Default::default(),
61        }
62    }
63}
64
65/// Join implementation handed to the optimizer for its hive partition rewrite.
66fn hive_join(
67    left: &DataFrame,
68    right: &DataFrame,
69    left_on: &str,
70    right_on: &str,
71    args: JoinArgs,
72) -> PolarsResult<DataFrame> {
73    use polars_ops::frame::DataFrameJoinOps;
74
75    left.join(right, [left_on], [right_on], args, None)
76}
77
78impl IntoLazy for LazyFrame {
79    fn lazy(self) -> LazyFrame {
80        self
81    }
82}
83
84/// Lazy abstraction over an eager `DataFrame`.
85///
86/// It really is an abstraction over a logical plan. The methods of this struct will incrementally
87/// modify a logical plan until output is requested (via [`collect`](crate::frame::LazyFrame::collect)).
88#[derive(Clone, Default)]
89#[must_use]
90pub struct LazyFrame {
91    pub logical_plan: DslPlan,
92    pub(crate) opt_state: OptFlags,
93    pub(crate) cached_arena: Arc<Mutex<Option<CachedArena>>>,
94}
95
96impl From<DslPlan> for LazyFrame {
97    fn from(plan: DslPlan) -> Self {
98        Self {
99            logical_plan: plan,
100            opt_state: OptFlags::default(),
101            cached_arena: Default::default(),
102        }
103    }
104}
105
106impl LazyFrame {
107    pub(crate) fn from_inner(
108        logical_plan: DslPlan,
109        opt_state: OptFlags,
110        cached_arena: Arc<Mutex<Option<CachedArena>>>,
111    ) -> Self {
112        Self {
113            logical_plan,
114            opt_state,
115            cached_arena,
116        }
117    }
118
119    pub(crate) fn get_plan_builder(self) -> DslBuilder {
120        DslBuilder::from(self.logical_plan)
121    }
122
123    fn get_opt_state(&self) -> OptFlags {
124        self.opt_state
125    }
126
127    pub fn from_logical_plan(logical_plan: DslPlan, opt_state: OptFlags) -> Self {
128        LazyFrame {
129            logical_plan,
130            opt_state,
131            cached_arena: Default::default(),
132        }
133    }
134
135    /// Get current optimizations.
136    pub fn get_current_optimizations(&self) -> OptFlags {
137        self.opt_state
138    }
139
140    /// Set allowed optimizations.
141    pub fn with_optimizations(mut self, opt_state: OptFlags) -> Self {
142        self.opt_state = opt_state;
143        self
144    }
145
146    /// Turn off all optimizations.
147    pub fn without_optimizations(self) -> Self {
148        self.with_optimizations(OptFlags::from_bits_truncate(0) | OptFlags::TYPE_COERCION)
149    }
150
151    /// Toggle projection pushdown optimization.
152    pub fn with_projection_pushdown(mut self, toggle: bool) -> Self {
153        self.opt_state.set(OptFlags::PROJECTION_PUSHDOWN, toggle);
154        self
155    }
156
157    /// Toggle cluster with columns optimization.
158    pub fn with_cluster_with_columns(mut self, toggle: bool) -> Self {
159        self.opt_state.set(OptFlags::CLUSTER_WITH_COLUMNS, toggle);
160        self
161    }
162
163    /// Check if operations are order dependent and unset maintaining_order if
164    /// the order would not be observed.
165    pub fn with_check_order(mut self, toggle: bool) -> Self {
166        self.opt_state.set(OptFlags::CHECK_ORDER_OBSERVE, toggle);
167        self
168    }
169
170    /// Toggle predicate pushdown optimization.
171    pub fn with_predicate_pushdown(mut self, toggle: bool) -> Self {
172        self.opt_state.set(OptFlags::PREDICATE_PUSHDOWN, toggle);
173        self
174    }
175
176    /// Toggle type coercion optimization.
177    pub fn with_type_coercion(mut self, toggle: bool) -> Self {
178        self.opt_state.set(OptFlags::TYPE_COERCION, toggle);
179        self
180    }
181
182    /// Toggle type check optimization.
183    pub fn with_type_check(mut self, toggle: bool) -> Self {
184        self.opt_state.set(OptFlags::TYPE_CHECK, toggle);
185        self
186    }
187
188    /// Toggle expression simplification optimization on or off.
189    pub fn with_simplify_expr(mut self, toggle: bool) -> Self {
190        self.opt_state.set(OptFlags::SIMPLIFY_EXPR, toggle);
191        self
192    }
193
194    /// Toggle common subplan elimination optimization on or off
195    #[cfg(feature = "cse")]
196    pub fn with_comm_subplan_elim(mut self, toggle: bool) -> Self {
197        self.opt_state.set(OptFlags::COMM_SUBPLAN_ELIM, toggle);
198        self
199    }
200
201    /// Toggle common subexpression elimination optimization on or off
202    #[cfg(feature = "cse")]
203    pub fn with_comm_subexpr_elim(mut self, toggle: bool) -> Self {
204        self.opt_state.set(OptFlags::COMM_SUBEXPR_ELIM, toggle);
205        self
206    }
207
208    /// Toggle slice pushdown optimization.
209    pub fn with_slice_pushdown(mut self, toggle: bool) -> Self {
210        self.opt_state.set(OptFlags::SLICE_PUSHDOWN, toggle);
211        self
212    }
213
214    #[cfg(feature = "streaming")]
215    pub fn with_streaming(mut self, toggle: bool) -> Self {
216        self.opt_state.set(OptFlags::STREAMING, toggle);
217        self
218    }
219
220    pub fn with_gpu(mut self, toggle: bool) -> Self {
221        self.opt_state.set(OptFlags::GPU, toggle);
222        self
223    }
224
225    /// Try to estimate the number of rows so that joins can determine which side to keep in memory.
226    pub fn with_row_estimate(mut self, toggle: bool) -> Self {
227        self.opt_state.set(OptFlags::ROW_ESTIMATE, toggle);
228        self
229    }
230
231    /// Run every node eagerly. This turns off multi-node optimizations.
232    pub fn _with_eager(mut self, toggle: bool) -> Self {
233        self.opt_state.set(OptFlags::EAGER, toggle);
234        self
235    }
236
237    /// Return a String describing the naive (un-optimized) logical plan.
238    pub fn describe_plan(&self) -> PolarsResult<String> {
239        Ok(self.clone().to_alp()?.describe())
240    }
241
242    /// Return a String describing the naive (un-optimized) logical plan in tree format.
243    pub fn describe_plan_tree(&self) -> PolarsResult<String> {
244        Ok(self.clone().to_alp()?.describe_tree_format())
245    }
246
247    /// Return a String describing the optimized logical plan.
248    ///
249    /// Returns `Err` if optimizing the logical plan fails.
250    pub fn describe_optimized_plan(&self) -> PolarsResult<String> {
251        Ok(self.clone().to_alp_optimized()?.describe())
252    }
253
254    /// Return a String describing the optimized logical plan in tree format.
255    ///
256    /// Returns `Err` if optimizing the logical plan fails.
257    pub fn describe_optimized_plan_tree(&self) -> PolarsResult<String> {
258        Ok(self.clone().to_alp_optimized()?.describe_tree_format())
259    }
260
261    /// Return a String describing the logical plan.
262    ///
263    /// If `optimized` is `true`, explains the optimized plan. If `optimized` is `false`,
264    /// explains the naive, un-optimized plan.
265    pub fn explain(&self, optimized: bool) -> PolarsResult<String> {
266        if optimized {
267            self.describe_optimized_plan()
268        } else {
269            self.describe_plan()
270        }
271    }
272
273    /// Add a sort operation to the logical plan.
274    ///
275    /// Sorts the LazyFrame by the column name specified using the provided options.
276    ///
277    /// # Example
278    ///
279    /// Sort DataFrame by 'sepal_width' column:
280    /// ```rust
281    /// # use polars_core::prelude::*;
282    /// # use polars_lazy::prelude::*;
283    /// fn sort_by_a(df: DataFrame) -> LazyFrame {
284    ///     df.lazy().sort(["sepal_width"], Default::default())
285    /// }
286    /// ```
287    /// Sort by a single column with specific order:
288    /// ```
289    /// # use polars_core::prelude::*;
290    /// # use polars_lazy::prelude::*;
291    /// fn sort_with_specific_order(df: DataFrame, descending: bool) -> LazyFrame {
292    ///     df.lazy().sort(
293    ///         ["sepal_width"],
294    ///         SortMultipleOptions::new()
295    ///             .with_order_descending(descending)
296    ///     )
297    /// }
298    /// ```
299    /// Sort by multiple columns with specifying order for each column:
300    /// ```
301    /// # use polars_core::prelude::*;
302    /// # use polars_lazy::prelude::*;
303    /// fn sort_by_multiple_columns_with_specific_order(df: DataFrame) -> LazyFrame {
304    ///     df.lazy().sort(
305    ///         ["sepal_width", "sepal_length"],
306    ///         SortMultipleOptions::new()
307    ///             .with_order_descending_multi([false, true])
308    ///     )
309    /// }
310    /// ```
311    /// See [`SortMultipleOptions`] for more options.
312    pub fn sort(self, by: impl IntoVec<PlSmallStr>, sort_options: SortMultipleOptions) -> Self {
313        let opt_state = self.get_opt_state();
314        let lp = self
315            .get_plan_builder()
316            .sort(by.into_vec().into_iter().map(col).collect(), sort_options)
317            .build();
318        Self::from_logical_plan(lp, opt_state)
319    }
320
321    /// Add a sort operation to the logical plan.
322    ///
323    /// Sorts the LazyFrame by the provided list of expressions, which will be turned into
324    /// concrete columns before sorting.
325    ///
326    /// See [`SortMultipleOptions`] for more options.
327    ///
328    /// # Example
329    ///
330    /// ```rust
331    /// use polars_core::prelude::*;
332    /// use polars_lazy::prelude::*;
333    ///
334    /// /// Sort DataFrame by 'sepal_width' column
335    /// fn example(df: DataFrame) -> LazyFrame {
336    ///       df.lazy()
337    ///         .sort_by_exprs(vec![col("sepal_width")], Default::default())
338    /// }
339    /// ```
340    pub fn sort_by_exprs<E: AsRef<[Expr]>>(
341        self,
342        by_exprs: E,
343        sort_options: SortMultipleOptions,
344    ) -> Self {
345        let by_exprs = by_exprs.as_ref().to_vec();
346        if by_exprs.is_empty() {
347            self
348        } else {
349            let opt_state = self.get_opt_state();
350            let lp = self.get_plan_builder().sort(by_exprs, sort_options).build();
351            Self::from_logical_plan(lp, opt_state)
352        }
353    }
354
355    pub fn top_k<E: AsRef<[Expr]>>(
356        self,
357        k: IdxSize,
358        by_exprs: E,
359        sort_options: SortMultipleOptions,
360    ) -> Self {
361        // this will optimize to top-k
362        self.sort_by_exprs(
363            by_exprs,
364            sort_options.with_order_reversed().with_nulls_last(true),
365        )
366        .slice(Slice::head(k as usize))
367    }
368
369    pub fn bottom_k<E: AsRef<[Expr]>>(
370        self,
371        k: IdxSize,
372        by_exprs: E,
373        sort_options: SortMultipleOptions,
374    ) -> Self {
375        // this will optimize to bottom-k
376        self.sort_by_exprs(by_exprs, sort_options.with_nulls_last(true))
377            .slice(Slice::head(k as usize))
378    }
379
380    /// Reverse the `DataFrame` from top to bottom.
381    ///
382    /// Row `i` becomes row `number_of_rows - i - 1`.
383    ///
384    /// # Example
385    ///
386    /// ```rust
387    /// use polars_core::prelude::*;
388    /// use polars_lazy::prelude::*;
389    ///
390    /// fn example(df: DataFrame) -> LazyFrame {
391    ///       df.lazy()
392    ///         .reverse()
393    /// }
394    /// ```
395    pub fn reverse(self) -> Self {
396        self.select(vec![col(PlSmallStr::from_static("*")).reverse()])
397    }
398
399    /// Rename columns in the DataFrame.
400    ///
401    /// `existing` and `new` are iterables of the same length containing the old and
402    /// corresponding new column names. Renaming happens to all `existing` columns
403    /// simultaneously, not iteratively. If `strict` is true, all columns in `existing`
404    /// must be present in the `LazyFrame` when `rename` is called; otherwise, only
405    /// those columns that are actually found will be renamed (others will be ignored).
406    pub fn rename<I, J, T, S>(self, existing: I, new: J, strict: bool) -> Self
407    where
408        I: IntoIterator<Item = T>,
409        J: IntoIterator<Item = S>,
410        T: AsRef<str>,
411        S: AsRef<str>,
412    {
413        let iter = existing.into_iter();
414        let cap = iter.size_hint().0;
415        let mut existing_vec: Vec<PlSmallStr> = Vec::with_capacity(cap);
416        let mut new_vec: Vec<PlSmallStr> = Vec::with_capacity(cap);
417
418        // TODO! should this error if `existing` and `new` have different lengths?
419        // Currently, the longer of the two is truncated.
420        for (existing, new) in iter.zip(new) {
421            let existing = existing.as_ref();
422            let new = new.as_ref();
423            if new != existing {
424                existing_vec.push(existing.into());
425                new_vec.push(new.into());
426            }
427        }
428
429        self.map_private(DslFunction::Rename {
430            existing: existing_vec.into(),
431            new: new_vec.into(),
432            strict,
433        })
434    }
435
436    /// Removes columns from the DataFrame.
437    /// Note that it's better to only select the columns you need
438    /// and let the projection pushdown optimize away the unneeded columns.
439    ///
440    /// Any given columns that are not in the schema will give a [`PolarsError::ColumnNotFound`]
441    /// error while materializing the [`LazyFrame`].
442    pub fn drop(self, columns: Selector) -> Self {
443        let opt_state = self.get_opt_state();
444        let lp = self.get_plan_builder().drop(columns).build();
445        Self::from_logical_plan(lp, opt_state)
446    }
447
448    /// Shift the values by a given period and fill the parts that will be empty due to this operation
449    /// with `Nones`.
450    ///
451    /// See the method on [Series](polars_core::series::SeriesTrait::shift) for more info on the `shift` operation.
452    pub fn shift<E: Into<Expr>>(self, n: E) -> Self {
453        self.select(vec![col(PlSmallStr::from_static("*")).shift(n.into())])
454    }
455
456    /// Shift the values by a given period and fill the parts that will be empty due to this operation
457    /// with the result of the `fill_value` expression.
458    ///
459    /// See the method on [Series](polars_core::series::SeriesTrait::shift) for more info on the `shift` operation.
460    pub fn shift_and_fill<E: Into<Expr>, IE: Into<Expr>>(self, n: E, fill_value: IE) -> Self {
461        self.select(vec![
462            col(PlSmallStr::from_static("*")).shift_and_fill(n.into(), fill_value.into()),
463        ])
464    }
465
466    /// Fill None values in the DataFrame with an expression.
467    pub fn fill_null<E: Into<Expr>>(self, fill_value: E) -> LazyFrame {
468        let opt_state = self.get_opt_state();
469        let lp = self.get_plan_builder().fill_null(fill_value.into()).build();
470        Self::from_logical_plan(lp, opt_state)
471    }
472
473    /// Fill NaN values in the DataFrame with an expression.
474    pub fn fill_nan<E: Into<Expr>>(self, fill_value: E) -> LazyFrame {
475        let opt_state = self.get_opt_state();
476        let lp = self.get_plan_builder().fill_nan(fill_value.into()).build();
477        Self::from_logical_plan(lp, opt_state)
478    }
479
480    /// Caches the result into a new LazyFrame.
481    ///
482    /// This should be used to prevent computations running multiple times.
483    pub fn cache(self) -> Self {
484        let opt_state = self.get_opt_state();
485        let lp = self.get_plan_builder().cache().build();
486        Self::from_logical_plan(lp, opt_state)
487    }
488
489    /// Cast named frame columns, resulting in a new LazyFrame with updated dtypes
490    pub fn cast(self, dtypes: PlHashMap<&str, DataType>, strict: bool) -> Self {
491        let cast_cols: Vec<Expr> = dtypes
492            .into_iter()
493            .map(|(name, dt)| {
494                let name = PlSmallStr::from_str(name);
495
496                if strict {
497                    col(name).strict_cast(dt)
498                } else {
499                    col(name).cast(dt)
500                }
501            })
502            .collect();
503
504        if cast_cols.is_empty() {
505            self
506        } else {
507            self.with_columns(cast_cols)
508        }
509    }
510
511    /// Cast all frame columns to the given dtype, resulting in a new LazyFrame
512    pub fn cast_all(self, dtype: impl Into<DataTypeExpr>, strict: bool) -> Self {
513        self.with_columns(vec![if strict {
514            col(PlSmallStr::from_static("*")).strict_cast(dtype)
515        } else {
516            col(PlSmallStr::from_static("*")).cast(dtype)
517        }])
518    }
519
520    pub fn optimize(
521        self,
522        lp_arena: &mut Arena<IR>,
523        expr_arena: &mut Arena<AExpr>,
524    ) -> PolarsResult<Node> {
525        self.optimize_with_scratch(lp_arena, expr_arena, &mut vec![])
526    }
527
528    pub fn to_alp_optimized(mut self) -> PolarsResult<IRPlan> {
529        let (mut lp_arena, mut expr_arena) = self.get_arenas();
530        let node = self.optimize_with_scratch(&mut lp_arena, &mut expr_arena, &mut vec![])?;
531
532        Ok(IRPlan::new(node, lp_arena, expr_arena))
533    }
534
535    pub fn to_alp(mut self) -> PolarsResult<IRPlan> {
536        let (mut lp_arena, mut expr_arena) = self.get_arenas();
537        let node = to_alp(
538            self.logical_plan,
539            &mut expr_arena,
540            &mut lp_arena,
541            &mut self.opt_state,
542        )?;
543        let plan = IRPlan::new(node, lp_arena, expr_arena);
544        Ok(plan)
545    }
546
547    pub(crate) fn optimize_with_scratch(
548        self,
549        ir_arena: &mut Arena<IR>,
550        expr_arena: &mut Arena<AExpr>,
551        scratch: &mut Vec<Node>,
552    ) -> PolarsResult<Node> {
553        let mut opt_flags = self.opt_state;
554        // Unset CSE
555        // This can be turned on again during ir-conversion.
556        #[allow(clippy::eq_op)]
557        #[cfg(feature = "cse")]
558        if opt_flags.contains(OptFlags::EAGER) {
559            opt_flags &= !(OptFlags::COMM_SUBEXPR_ELIM | OptFlags::COMM_SUBEXPR_ELIM);
560        }
561        let root = to_alp(self.logical_plan, expr_arena, ir_arena, &mut opt_flags)?;
562
563        let lp_top = optimize(
564            root,
565            opt_flags,
566            ir_arena,
567            expr_arena,
568            scratch,
569            ExecutionHooks {
570                apply_scan_predicate_to_scan_ir,
571                hive_join,
572            },
573        )?;
574
575        Ok(lp_top)
576    }
577
578    fn prepare_collect_post_opt<P>(
579        mut self,
580        check_sink: bool,
581        post_opt: P,
582    ) -> PolarsResult<(ExecutionState, Box<dyn Executor>, bool)>
583    where
584        P: FnOnce(Node, &mut Arena<IR>, &mut Arena<AExpr>) -> PolarsResult<()>,
585    {
586        let (mut lp_arena, mut expr_arena) = self.get_arenas();
587
588        let mut scratch = vec![];
589        let lp_top = self.optimize_with_scratch(&mut lp_arena, &mut expr_arena, &mut scratch)?;
590        post_opt(lp_top, &mut lp_arena, &mut expr_arena)?;
591
592        // sink should be replaced
593        let no_file_sink = if check_sink {
594            !matches!(
595                lp_arena.get(lp_top),
596                IR::Sink {
597                    payload: SinkTypeIR::File { .. },
598                    ..
599                }
600            )
601        } else {
602            true
603        };
604        let physical_plan = create_physical_plan(
605            lp_top,
606            &mut lp_arena,
607            &mut expr_arena,
608            BUILD_STREAMING_EXECUTOR,
609        )?;
610
611        let state = ExecutionState::new();
612        Ok((state, physical_plan, no_file_sink))
613    }
614
615    // post_opt: A function that is called after optimization. This can be used to modify the IR jit.
616    //
617    // The plan is optimized as for the GPU engine, the one consumer of such callbacks
618    // (cudf-polars): it does not support every optimization (e.g. struct-field pruning, as it
619    // ignores the field selection of scans).
620    pub fn _collect_post_opt<P>(self, post_opt: P) -> PolarsResult<DataFrame>
621    where
622        P: FnOnce(Node, &mut Arena<IR>, &mut Arena<AExpr>) -> PolarsResult<()>,
623    {
624        let (mut state, mut physical_plan, _) = self
625            .with_gpu(true)
626            .prepare_collect_post_opt(false, post_opt)?;
627        physical_plan.execute(&mut state)
628    }
629
630    #[allow(unused_mut)]
631    fn prepare_collect(
632        self,
633        check_sink: bool,
634    ) -> PolarsResult<(ExecutionState, Box<dyn Executor>, bool)> {
635        self.prepare_collect_post_opt(check_sink, |_, _, _| Ok(()))
636    }
637
638    /// Execute all the lazy operations and collect them into a [`DataFrame`] using a specified
639    /// `engine`.
640    ///
641    /// The query is optimized prior to execution.
642    pub fn collect_with_engine(mut self, engine: Engine) -> PolarsResult<QueryResult> {
643        let engine = match engine {
644            Engine::Streaming => Engine::Streaming,
645            _ if std::env::var("POLARS_FORCE_STREAMING").as_deref() == Ok("1") => Engine::Streaming,
646            Engine::Auto => {
647                if self.opt_state.eager() {
648                    Engine::InMemory
649                } else {
650                    Engine::Streaming
651                }
652            },
653            v => v,
654        };
655
656        if engine != Engine::Streaming
657            && std::env::var("POLARS_AUTO_STREAMING").as_deref() == Ok("1")
658        {
659            feature_gated!("streaming", {
660                if let Some(r) = self.clone()._collect_with_streaming_suppress_todo_panic() {
661                    return r;
662                }
663            })
664        }
665        match engine {
666            Engine::Streaming => {
667                feature_gated!("streaming", self = self.with_streaming(true))
668            },
669            Engine::Gpu => self = self.with_gpu(true),
670            _ => (),
671        }
672
673        let observer = self
674            .opt_state
675            .query_monitoring()
676            .then(polars_observer::new_query_observer)
677            .flatten();
678
679        if let Some(o) = observer.as_ref() {
680            o.on_query_started()
681        }
682
683        let mut ir_plan = self.to_alp_optimized().inspect_err(|err| {
684            if let Some(o) = observer.as_ref() {
685                o.on_query_failed(err)
686            }
687        })?;
688        ir_plan.ensure_root_node_is_sink();
689
690        match engine {
691            Engine::Streaming => feature_gated!("streaming", {
692                polars_stream::run_query(
693                    ir_plan.lp_top,
694                    &mut ir_plan.lp_arena,
695                    &mut ir_plan.expr_arena,
696                    observer,
697                )
698            }),
699            Engine::InMemory | Engine::Gpu => run_in_memory_query(
700                ir_plan.lp_top,
701                &mut ir_plan.lp_arena,
702                &mut ir_plan.expr_arena,
703                engine,
704                observer,
705            ),
706            Engine::Auto => unreachable!(),
707        }
708    }
709
710    pub fn explain_all(plans: Vec<DslPlan>, opt_state: OptFlags) -> PolarsResult<String> {
711        let sink_multiple = LazyFrame {
712            logical_plan: DslPlan::SinkMultiple { inputs: plans },
713            opt_state,
714            cached_arena: Default::default(),
715        };
716        sink_multiple.explain(true)
717    }
718
719    pub fn collect_all_with_engine(
720        plans: Vec<DslPlan>,
721        engine: Engine,
722        opt_state: OptFlags,
723    ) -> PolarsResult<Vec<DataFrame>> {
724        if plans.is_empty() {
725            return Ok(Vec::new());
726        }
727
728        LazyFrame {
729            logical_plan: DslPlan::SinkMultiple { inputs: plans },
730            opt_state,
731            cached_arena: Default::default(),
732        }
733        .collect_with_engine(engine)
734        .map(|r| r.unwrap_multiple())
735    }
736
737    /// Execute all the lazy operations and collect them into a [`DataFrame`].
738    ///
739    /// The query is optimized prior to execution.
740    ///
741    /// # Example
742    ///
743    /// ```rust
744    /// use polars_core::prelude::*;
745    /// use polars_lazy::prelude::*;
746    ///
747    /// fn example(df: DataFrame) -> PolarsResult<DataFrame> {
748    ///     df.lazy()
749    ///       .group_by([col("foo")])
750    ///       .agg([col("bar").sum(), col("ham").mean().alias("avg_ham")])
751    ///       .collect()
752    /// }
753    /// ```
754    pub fn collect(self) -> PolarsResult<DataFrame> {
755        self.collect_with_engine(Engine::Auto).map(|r| match r {
756            QueryResult::Single(df) => df,
757            // TODO: Should return query results
758            QueryResult::Multiple(_) => DataFrame::empty(),
759        })
760    }
761
762    /// Collect the query in batches.
763    ///
764    /// If lazy is true the query will not start until the first poll (or until
765    /// start is called on CollectBatches).
766    #[cfg(feature = "async")]
767    pub fn collect_batches(
768        self,
769        engine: Engine,
770        maintain_order: bool,
771        chunk_size: Option<NonZeroUsize>,
772        lazy: bool,
773    ) -> PolarsResult<CollectBatches> {
774        let (send, recv) = sync_channel(1);
775        let runner_send = send.clone();
776        let ldf = self.sink_batches(
777            PlanCallback::new(move |df| {
778                // Stop if receiver has closed.
779                let send_result = send.send(Ok(df));
780                Ok(send_result.is_err())
781            }),
782            maintain_order,
783            chunk_size,
784        )?;
785        let runner = move || {
786            // We use spawn_blocking here as it has a high blocking thread pool limit.
787            polars_core::runtime::ASYNC.spawn_blocking(move || {
788                if let Err(e) = ldf.collect_with_engine(engine) {
789                    runner_send.send(Err(e)).ok();
790                }
791            });
792        };
793
794        let mut collect_batches = CollectBatches {
795            recv,
796            runner: Some(Box::new(runner)),
797        };
798        if !lazy {
799            collect_batches.start();
800        }
801        Ok(collect_batches)
802    }
803
804    pub fn sink_batches(
805        mut self,
806        function: PlanCallback<DataFrame, bool>,
807        maintain_order: bool,
808        chunk_size: Option<NonZeroUsize>,
809    ) -> PolarsResult<Self> {
810        use polars_plan::prelude::sink::CallbackSinkType;
811
812        polars_ensure!(
813            !matches!(self.logical_plan, DslPlan::Sink { .. }),
814            InvalidOperation: "cannot create a sink on top of another sink"
815        );
816
817        self.logical_plan = DslPlan::Sink {
818            input: Arc::new(self.logical_plan),
819            payload: SinkType::Callback(CallbackSinkType {
820                function,
821                maintain_order,
822                chunk_size,
823            }),
824        };
825
826        Ok(self)
827    }
828
829    /// Collect with the streaming engine. Returns `None` if the streaming engine panics with a todo!.
830    #[cfg(feature = "streaming")]
831    fn _collect_with_streaming_suppress_todo_panic(
832        mut self,
833    ) -> Option<PolarsResult<polars_core::query_result::QueryResult>> {
834        self.opt_state |= OptFlags::STREAMING;
835        let mut ir_plan = match self.to_alp_optimized() {
836            Ok(v) => v,
837            Err(e) => return Some(Err(e)),
838        };
839
840        ir_plan.ensure_root_node_is_sink();
841
842        let f = || {
843            polars_stream::run_query(
844                ir_plan.lp_top,
845                &mut ir_plan.lp_arena,
846                &mut ir_plan.expr_arena,
847                None,
848            )
849        };
850
851        match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
852            Ok(v) => Some(v),
853            Err(e) => {
854                // Fallback to normal engine if error is due to not being implemented
855                // and auto_streaming is set, otherwise propagate error.
856                if e.downcast_ref::<&str>()
857                    .is_some_and(|s| s.starts_with("not yet implemented"))
858                {
859                    if polars_core::config::verbose() {
860                        eprintln!(
861                            "caught unimplemented error in new streaming engine, falling back to normal engine"
862                        );
863                    }
864                    None
865                } else {
866                    std::panic::resume_unwind(e)
867                }
868            },
869        }
870    }
871
872    pub fn sink(
873        mut self,
874        sink_type: SinkDestination,
875        file_format: FileWriteFormat,
876        unified_sink_args: UnifiedSinkArgs,
877    ) -> PolarsResult<Self> {
878        polars_ensure!(
879            !matches!(self.logical_plan, DslPlan::Sink { .. }),
880            InvalidOperation: "cannot create a sink on top of another sink"
881        );
882
883        self.logical_plan = DslPlan::Sink {
884            input: Arc::new(self.logical_plan),
885            payload: match sink_type {
886                SinkDestination::File { target } => SinkType::File(FileSinkOptions {
887                    target,
888                    file_format,
889                    unified_sink_args,
890                }),
891                SinkDestination::Partitioned {
892                    base_path,
893                    file_path_provider,
894                    partition_strategy,
895                    max_rows_per_file,
896                    approximate_bytes_per_file,
897                } => SinkType::Partitioned(PartitionedSinkOptions {
898                    base_path,
899                    file_path_provider,
900                    partition_strategy,
901                    file_format,
902                    unified_sink_args,
903                    max_rows_per_file,
904                    approximate_bytes_per_file,
905                }),
906            },
907        };
908        Ok(self)
909    }
910
911    /// Filter frame rows that match a predicate expression.
912    ///
913    /// The expression must yield boolean values (note that rows where the
914    /// predicate resolves to `null` are *not* included in the resulting frame).
915    ///
916    /// # Example
917    ///
918    /// ```rust
919    /// use polars_core::prelude::*;
920    /// use polars_lazy::prelude::*;
921    ///
922    /// fn example(df: DataFrame) -> LazyFrame {
923    ///       df.lazy()
924    ///         .filter(col("sepal_width").is_not_null())
925    ///         .select([col("sepal_width"), col("sepal_length")])
926    /// }
927    /// ```
928    pub fn filter(self, predicate: Expr) -> Self {
929        let opt_state = self.get_opt_state();
930        let lp = self.get_plan_builder().filter(predicate).build();
931        Self::from_logical_plan(lp, opt_state)
932    }
933
934    /// Remove frame rows that match a predicate expression.
935    ///
936    /// The expression must yield boolean values (note that rows where the
937    /// predicate resolves to `null` are *not* removed from the resulting frame).
938    ///
939    /// # Example
940    ///
941    /// ```rust
942    /// use polars_core::prelude::*;
943    /// use polars_lazy::prelude::*;
944    ///
945    /// fn example(df: DataFrame) -> LazyFrame {
946    ///       df.lazy()
947    ///         .remove(col("sepal_width").is_null())
948    ///         .select([col("sepal_width"), col("sepal_length")])
949    /// }
950    /// ```
951    pub fn remove(self, predicate: Expr) -> Self {
952        self.filter(predicate.neq_missing(lit(true)))
953    }
954
955    /// Select (and optionally rename, with [`alias`](crate::dsl::Expr::alias)) columns from the query.
956    ///
957    /// Columns can be selected with [`col`];
958    /// If you want to select all columns use `col(PlSmallStr::from_static("*"))`.
959    ///
960    /// # Example
961    ///
962    /// ```rust
963    /// use polars_core::prelude::*;
964    /// use polars_lazy::prelude::*;
965    ///
966    /// /// This function selects column "foo" and column "bar".
967    /// /// Column "bar" is renamed to "ham".
968    /// fn example(df: DataFrame) -> LazyFrame {
969    ///       df.lazy()
970    ///         .select([col("foo"),
971    ///                   col("bar").alias("ham")])
972    /// }
973    ///
974    /// /// This function selects all columns except "foo"
975    /// fn exclude_a_column(df: DataFrame) -> LazyFrame {
976    ///       df.lazy()
977    ///         .select([all().exclude_cols(["foo"]).as_expr()])
978    /// }
979    /// ```
980    pub fn select<E: AsRef<[Expr]>>(self, exprs: E) -> Self {
981        let exprs = exprs.as_ref().to_vec();
982        self.select_impl(
983            exprs,
984            ProjectionOptions {
985                run_parallel: true,
986                duplicate_check: true,
987                should_broadcast: true,
988                maintain_dataframe_height: false,
989            },
990        )
991    }
992
993    pub fn select_seq<E: AsRef<[Expr]>>(self, exprs: E) -> Self {
994        let exprs = exprs.as_ref().to_vec();
995        self.select_impl(
996            exprs,
997            ProjectionOptions {
998                run_parallel: false,
999                duplicate_check: true,
1000                should_broadcast: true,
1001                maintain_dataframe_height: false,
1002            },
1003        )
1004    }
1005
1006    fn select_impl(self, exprs: Vec<Expr>, options: ProjectionOptions) -> Self {
1007        let opt_state = self.get_opt_state();
1008        let lp = self.get_plan_builder().project(exprs, options).build();
1009        Self::from_logical_plan(lp, opt_state)
1010    }
1011
1012    /// Performs a "group-by" on a `LazyFrame`, producing a [`LazyGroupBy`], which can subsequently be aggregated.
1013    ///
1014    /// Takes a list of expressions to group on.
1015    ///
1016    /// # Example
1017    ///
1018    /// ```rust
1019    /// use polars_core::prelude::*;
1020    /// use polars_lazy::prelude::*;
1021    ///
1022    /// fn example(df: DataFrame) -> LazyFrame {
1023    ///       df.lazy()
1024    ///        .group_by([col("date")])
1025    ///        .agg([
1026    ///            col("rain").min().alias("min_rain"),
1027    ///            col("rain").sum().alias("sum_rain"),
1028    ///            col("rain").quantile(lit(0.5), QuantileMethod::Nearest).alias("median_rain"),
1029    ///        ])
1030    /// }
1031    /// ```
1032    pub fn group_by<E: AsRef<[IE]>, IE: Into<Expr> + Clone>(self, by: E) -> LazyGroupBy {
1033        let keys = by
1034            .as_ref()
1035            .iter()
1036            .map(|e| e.clone().into())
1037            .collect::<Vec<_>>();
1038        let opt_state = self.get_opt_state();
1039
1040        #[cfg(feature = "dynamic_group_by")]
1041        {
1042            LazyGroupBy {
1043                logical_plan: self.logical_plan,
1044                opt_state,
1045                keys,
1046                predicates: vec![],
1047                maintain_order: false,
1048                dynamic_options: None,
1049                rolling_options: None,
1050            }
1051        }
1052
1053        #[cfg(not(feature = "dynamic_group_by"))]
1054        {
1055            LazyGroupBy {
1056                logical_plan: self.logical_plan,
1057                opt_state,
1058                keys,
1059                predicates: vec![],
1060                maintain_order: false,
1061            }
1062        }
1063    }
1064
1065    /// Create rolling groups based on a time column.
1066    ///
1067    /// Also works for index values of type UInt32, UInt64, Int32, or Int64.
1068    ///
1069    /// Different from a [`group_by_dynamic`][`Self::group_by_dynamic`], the windows are now determined by the
1070    /// individual values and are not of constant intervals. For constant intervals use
1071    /// *group_by_dynamic*
1072    #[cfg(feature = "dynamic_group_by")]
1073    pub fn rolling<E: AsRef<[Expr]>>(
1074        mut self,
1075        index_column: Expr,
1076        group_by: E,
1077        mut options: RollingGroupOptions,
1078    ) -> LazyGroupBy {
1079        if let Expr::Column(name) = index_column {
1080            options.index_column = name;
1081        } else {
1082            let output_field = index_column
1083                .to_field(&self.collect_schema().unwrap())
1084                .unwrap();
1085            return self.with_column(index_column).rolling(
1086                Expr::Column(output_field.name().clone()),
1087                group_by,
1088                options,
1089            );
1090        }
1091        let opt_state = self.get_opt_state();
1092        LazyGroupBy {
1093            logical_plan: self.logical_plan,
1094            opt_state,
1095            predicates: vec![],
1096            keys: group_by.as_ref().to_vec(),
1097            maintain_order: true,
1098            dynamic_options: None,
1099            rolling_options: Some(options),
1100        }
1101    }
1102
1103    /// Group based on a time value (or index value of type Int32, Int64).
1104    ///
1105    /// Time windows are calculated and rows are assigned to windows. Different from a
1106    /// normal group_by is that a row can be member of multiple groups. The time/index
1107    /// window could be seen as a rolling window, with a window size determined by
1108    /// dates/times/values instead of slots in the DataFrame.
1109    ///
1110    /// A window is defined by:
1111    ///
1112    /// - every: interval of the window
1113    /// - period: length of the window
1114    /// - offset: offset of the window
1115    ///
1116    /// The `group_by` argument should be empty `[]` if you don't want to combine this
1117    /// with a ordinary group_by on these keys.
1118    #[cfg(feature = "dynamic_group_by")]
1119    pub fn group_by_dynamic<E: AsRef<[Expr]>>(
1120        mut self,
1121        index_column: Expr,
1122        group_by: E,
1123        mut options: DynamicGroupOptions,
1124    ) -> LazyGroupBy {
1125        if let Expr::Column(name) = index_column {
1126            options.index_column = name;
1127        } else {
1128            let output_field = index_column
1129                .to_field(&self.collect_schema().unwrap())
1130                .unwrap();
1131            return self.with_column(index_column).group_by_dynamic(
1132                Expr::Column(output_field.name().clone()),
1133                group_by,
1134                options,
1135            );
1136        }
1137        let opt_state = self.get_opt_state();
1138        LazyGroupBy {
1139            logical_plan: self.logical_plan,
1140            opt_state,
1141            predicates: vec![],
1142            keys: group_by.as_ref().to_vec(),
1143            maintain_order: true,
1144            dynamic_options: Some(options),
1145            rolling_options: None,
1146        }
1147    }
1148
1149    /// Similar to [`group_by`][`Self::group_by`], but order of the DataFrame is maintained.
1150    pub fn group_by_stable<E: AsRef<[IE]>, IE: Into<Expr> + Clone>(self, by: E) -> LazyGroupBy {
1151        let keys = by
1152            .as_ref()
1153            .iter()
1154            .map(|e| e.clone().into())
1155            .collect::<Vec<_>>();
1156        let opt_state = self.get_opt_state();
1157
1158        #[cfg(feature = "dynamic_group_by")]
1159        {
1160            LazyGroupBy {
1161                logical_plan: self.logical_plan,
1162                opt_state,
1163                keys,
1164                predicates: vec![],
1165                maintain_order: true,
1166                dynamic_options: None,
1167                rolling_options: None,
1168            }
1169        }
1170
1171        #[cfg(not(feature = "dynamic_group_by"))]
1172        {
1173            LazyGroupBy {
1174                logical_plan: self.logical_plan,
1175                opt_state,
1176                keys,
1177                predicates: vec![],
1178                maintain_order: true,
1179            }
1180        }
1181    }
1182
1183    /// Left anti join this query with another lazy query.
1184    ///
1185    /// Matches on the values of the expressions `left_on` and `right_on`. For more
1186    /// flexible join logic, see [`join`](LazyFrame::join) or
1187    /// [`join_builder`](LazyFrame::join_builder).
1188    ///
1189    /// # Example
1190    ///
1191    /// ```rust
1192    /// use polars_core::prelude::*;
1193    /// use polars_lazy::prelude::*;
1194    /// fn anti_join_dataframes(ldf: LazyFrame, other: LazyFrame) -> PolarsResult<LazyFrame> {
1195    ///         ldf
1196    ///         .anti_join(other, col("foo"), col("bar").cast(DataType::String))
1197    /// }
1198    /// ```
1199    #[cfg(feature = "semi_anti_join")]
1200    pub fn anti_join<E: Into<Expr>>(
1201        self,
1202        other: LazyFrame,
1203        left_on: E,
1204        right_on: E,
1205    ) -> PolarsResult<LazyFrame> {
1206        self.join(
1207            other,
1208            [left_on.into()],
1209            [right_on.into()],
1210            JoinArgs::new(JoinType::Anti),
1211        )
1212    }
1213
1214    /// Creates the Cartesian product from both frames, preserving the order of the left keys.
1215    #[cfg(feature = "cross_join")]
1216    pub fn cross_join(self, other: LazyFrame, suffix: Option<PlSmallStr>) -> LazyFrame {
1217        self.join(
1218            other,
1219            vec![],
1220            vec![],
1221            JoinArgs::new(JoinType::Cross).with_suffix(suffix),
1222        )
1223        .unwrap()
1224    }
1225
1226    /// Left outer join this query with another lazy query.
1227    ///
1228    /// Matches on the values of the expressions `left_on` and `right_on`. For more
1229    /// flexible join logic, see [`join`](LazyFrame::join) or
1230    /// [`join_builder`](LazyFrame::join_builder).
1231    ///
1232    /// # Example
1233    ///
1234    /// ```rust
1235    /// use polars_core::prelude::*;
1236    /// use polars_lazy::prelude::*;
1237    /// fn left_join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
1238    ///         ldf
1239    ///         .left_join(other, col("foo"), col("bar"))
1240    /// }
1241    /// ```
1242    pub fn left_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1243        self.join(
1244            other,
1245            [left_on.into()],
1246            [right_on.into()],
1247            JoinArgs::new(JoinType::Left),
1248        )
1249        .unwrap()
1250    }
1251
1252    /// Inner join this query with another lazy query.
1253    ///
1254    /// Matches on the values of the expressions `left_on` and `right_on`. For more
1255    /// flexible join logic, see [`join`](LazyFrame::join) or
1256    /// [`join_builder`](LazyFrame::join_builder).
1257    ///
1258    /// # Example
1259    ///
1260    /// ```rust
1261    /// use polars_core::prelude::*;
1262    /// use polars_lazy::prelude::*;
1263    /// fn inner_join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
1264    ///         ldf
1265    ///         .inner_join(other, col("foo"), col("bar").cast(DataType::String))
1266    /// }
1267    /// ```
1268    pub fn inner_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1269        self.join(
1270            other,
1271            [left_on.into()],
1272            [right_on.into()],
1273            JoinArgs::new(JoinType::Inner),
1274        )
1275        .unwrap()
1276    }
1277
1278    /// Full outer join this query with another lazy query.
1279    ///
1280    /// Matches on the values of the expressions `left_on` and `right_on`. For more
1281    /// flexible join logic, see [`join`](LazyFrame::join) or
1282    /// [`join_builder`](LazyFrame::join_builder).
1283    ///
1284    /// # Example
1285    ///
1286    /// ```rust
1287    /// use polars_core::prelude::*;
1288    /// use polars_lazy::prelude::*;
1289    /// fn full_join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
1290    ///         ldf
1291    ///         .full_join(other, col("foo"), col("bar"))
1292    /// }
1293    /// ```
1294    pub fn full_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1295        self.join(
1296            other,
1297            [left_on.into()],
1298            [right_on.into()],
1299            JoinArgs::new(JoinType::Full),
1300        )
1301        .unwrap()
1302    }
1303
1304    /// Left semi join this query with another lazy query.
1305    ///
1306    /// Matches on the values of the expressions `left_on` and `right_on`. For more
1307    /// flexible join logic, see [`join`](LazyFrame::join) or
1308    /// [`join_builder`](LazyFrame::join_builder).
1309    ///
1310    /// # Example
1311    ///
1312    /// ```rust
1313    /// use polars_core::prelude::*;
1314    /// use polars_lazy::prelude::*;
1315    /// fn semi_join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
1316    ///         ldf
1317    ///         .semi_join(other, col("foo"), col("bar").cast(DataType::String))
1318    /// }
1319    /// ```
1320    #[cfg(feature = "semi_anti_join")]
1321    pub fn semi_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1322        self.join(
1323            other,
1324            [left_on.into()],
1325            [right_on.into()],
1326            JoinArgs::new(JoinType::Semi),
1327        )
1328        .unwrap()
1329    }
1330
1331    /// Generic function to join two LazyFrames.
1332    ///
1333    /// `join` can join on multiple columns, given as two list of expressions, and with a
1334    /// [`JoinType`] specified by `how`. Non-joined column names in the right DataFrame
1335    /// that already exist in this DataFrame are suffixed with `"_right"`. For control
1336    /// over how columns are renamed and parallelization options, use
1337    /// [`join_builder`](LazyFrame::join_builder).
1338    ///
1339    /// Any provided `args.slice` parameter is not considered, but set by the internal optimizer.
1340    ///
1341    /// # Example
1342    ///
1343    /// ```rust
1344    /// use polars_core::prelude::*;
1345    /// use polars_lazy::prelude::*;
1346    ///
1347    /// fn example(ldf: LazyFrame, other: LazyFrame) -> PolarsResult<LazyFrame> {
1348    ///         ldf
1349    ///         .join(other, [col("foo"), col("bar")], [col("foo"), col("bar")], JoinArgs::new(JoinType::Inner))
1350    /// }
1351    /// ```
1352    pub fn join<E: AsRef<[Expr]>>(
1353        self,
1354        other: LazyFrame,
1355        left_on: E,
1356        right_on: E,
1357        args: JoinArgs,
1358    ) -> PolarsResult<LazyFrame> {
1359        let left_on = left_on.as_ref().to_vec();
1360        let right_on = right_on.as_ref().to_vec();
1361
1362        self._join_impl(other, left_on, right_on, args)
1363    }
1364
1365    fn _join_impl(
1366        self,
1367        other: LazyFrame,
1368        left_on: Vec<Expr>,
1369        right_on: Vec<Expr>,
1370        args: JoinArgs,
1371    ) -> PolarsResult<LazyFrame> {
1372        let JoinArgs {
1373            how,
1374            validation,
1375            suffix,
1376            slice,
1377            nulls_equal,
1378            coalesce,
1379            maintain_order,
1380            build_side,
1381        } = args;
1382
1383        if slice.is_some() {
1384            panic!("impl error: slice is not handled")
1385        }
1386
1387        let mut builder = self
1388            .join_builder()
1389            .with(other)
1390            .left_on(left_on)
1391            .right_on(right_on)
1392            .how(how)
1393            .validate(validation)
1394            .join_nulls(nulls_equal)
1395            .coalesce(coalesce)
1396            .maintain_order(maintain_order)
1397            .build_side(build_side);
1398
1399        if let Some(suffix) = suffix {
1400            builder = builder.suffix(suffix);
1401        }
1402
1403        // Note: args.slice is set by the optimizer
1404        builder.finish()
1405    }
1406
1407    /// Consume `self` and return a [`JoinBuilder`] to customize a join on this LazyFrame.
1408    ///
1409    /// After the `JoinBuilder` has been created and set up, calling
1410    /// [`finish()`](JoinBuilder::finish) on it will give back the `LazyFrame`
1411    /// representing the `join` operation.
1412    pub fn join_builder(self) -> JoinBuilder {
1413        JoinBuilder::new(self)
1414    }
1415
1416    /// Gathers rows from this DataFrame based on the indices in idxs.
1417    ///
1418    /// idxs must only have a single column of indices.
1419    pub fn gather(self, idxs: LazyFrame, null_on_oob: bool) -> LazyFrame {
1420        let opt_state = self.get_opt_state();
1421        let lp = self
1422            .get_plan_builder()
1423            .gather(idxs.logical_plan, null_on_oob)
1424            .build();
1425        Self::from_logical_plan(lp, opt_state)
1426    }
1427
1428    /// Add or replace a column, given as an expression, to a DataFrame.
1429    ///
1430    /// # Example
1431    ///
1432    /// ```rust
1433    /// use polars_core::prelude::*;
1434    /// use polars_lazy::prelude::*;
1435    /// fn add_column(df: DataFrame) -> LazyFrame {
1436    ///     df.lazy()
1437    ///         .with_column(
1438    ///             when(col("sepal_length").lt(lit(5.0)))
1439    ///             .then(lit(10))
1440    ///             .otherwise(lit(1))
1441    ///             .alias("new_column_name"),
1442    ///         )
1443    /// }
1444    /// ```
1445    pub fn with_column(self, expr: Expr) -> LazyFrame {
1446        let opt_state = self.get_opt_state();
1447        let lp = self
1448            .get_plan_builder()
1449            .with_columns(
1450                vec![expr],
1451                ProjectionOptions {
1452                    run_parallel: false,
1453                    duplicate_check: true,
1454                    should_broadcast: true,
1455                    maintain_dataframe_height: false,
1456                },
1457            )
1458            .build();
1459        Self::from_logical_plan(lp, opt_state)
1460    }
1461
1462    /// Add or replace multiple columns, given as expressions, to a DataFrame.
1463    ///
1464    /// # Example
1465    ///
1466    /// ```rust
1467    /// use polars_core::prelude::*;
1468    /// use polars_lazy::prelude::*;
1469    /// fn add_columns(df: DataFrame) -> LazyFrame {
1470    ///     df.lazy()
1471    ///         .with_columns(
1472    ///             vec![lit(10).alias("foo"), lit(100).alias("bar")]
1473    ///          )
1474    /// }
1475    /// ```
1476    pub fn with_columns<E: AsRef<[Expr]>>(self, exprs: E) -> LazyFrame {
1477        let exprs = exprs.as_ref().to_vec();
1478        self.with_columns_impl(
1479            exprs,
1480            ProjectionOptions {
1481                run_parallel: true,
1482                duplicate_check: true,
1483                should_broadcast: true,
1484                maintain_dataframe_height: false,
1485            },
1486        )
1487    }
1488
1489    /// Add or replace multiple columns to a DataFrame, but evaluate them sequentially.
1490    pub fn with_columns_seq<E: AsRef<[Expr]>>(self, exprs: E) -> LazyFrame {
1491        let exprs = exprs.as_ref().to_vec();
1492        self.with_columns_impl(
1493            exprs,
1494            ProjectionOptions {
1495                run_parallel: false,
1496                duplicate_check: true,
1497                should_broadcast: true,
1498                maintain_dataframe_height: false,
1499            },
1500        )
1501    }
1502
1503    /// Match or evolve to a certain schema.
1504    pub fn match_to_schema(
1505        self,
1506        schema: SchemaRef,
1507        per_column: Arc<[MatchToSchemaPerColumn]>,
1508        extra_columns: ExtraColumnsPolicy,
1509    ) -> LazyFrame {
1510        let opt_state = self.get_opt_state();
1511        let lp = self
1512            .get_plan_builder()
1513            .match_to_schema(schema, per_column, extra_columns)
1514            .build();
1515        Self::from_logical_plan(lp, opt_state)
1516    }
1517
1518    pub fn pipe_with_schema(
1519        self,
1520        callback: PlanCallback<(Vec<DslPlan>, Vec<SchemaRef>), DslPlan>,
1521    ) -> Self {
1522        let opt_state = self.get_opt_state();
1523        let lp = self
1524            .get_plan_builder()
1525            .pipe_with_schema(vec![], callback)
1526            .build();
1527        Self::from_logical_plan(lp, opt_state)
1528    }
1529
1530    pub fn pipe_with_schemas(
1531        self,
1532        others: Vec<LazyFrame>,
1533        callback: PlanCallback<(Vec<DslPlan>, Vec<SchemaRef>), DslPlan>,
1534    ) -> Self {
1535        let opt_state = self.get_opt_state();
1536        let lp = self
1537            .get_plan_builder()
1538            .pipe_with_schema(
1539                others.into_iter().map(|lf| lf.logical_plan).collect(),
1540                callback,
1541            )
1542            .build();
1543        Self::from_logical_plan(lp, opt_state)
1544    }
1545
1546    fn with_columns_impl(self, exprs: Vec<Expr>, options: ProjectionOptions) -> LazyFrame {
1547        let opt_state = self.get_opt_state();
1548        let lp = self.get_plan_builder().with_columns(exprs, options).build();
1549        Self::from_logical_plan(lp, opt_state)
1550    }
1551
1552    /// Aggregate all the columns as their maximum values.
1553    ///
1554    /// Aggregated columns will have the same names as the original columns.
1555    pub fn max(self) -> Self {
1556        self.map_private(DslFunction::Stats(StatsFunction::Max))
1557    }
1558
1559    /// Aggregate all the columns as their minimum values.
1560    ///
1561    /// Aggregated columns will have the same names as the original columns.
1562    pub fn min(self) -> Self {
1563        self.map_private(DslFunction::Stats(StatsFunction::Min))
1564    }
1565
1566    /// Aggregate all the columns as their sum values.
1567    ///
1568    /// Aggregated columns will have the same names as the original columns.
1569    ///
1570    /// - Boolean columns will sum to a `u32` containing the number of `true`s.
1571    /// - For integer columns, the ordinary checks for overflow are performed:
1572    ///   if running in `debug` mode, overflows will panic, whereas in `release` mode overflows will
1573    ///   silently wrap.
1574    /// - String columns will sum to None.
1575    pub fn sum(self) -> Self {
1576        self.map_private(DslFunction::Stats(StatsFunction::Sum))
1577    }
1578
1579    /// Aggregate all the columns as their mean values.
1580    ///
1581    /// - Boolean and integer columns are converted to `f64` before computing the mean.
1582    /// - String columns will have a mean of None.
1583    pub fn mean(self) -> Self {
1584        self.map_private(DslFunction::Stats(StatsFunction::Mean))
1585    }
1586
1587    /// Aggregate all the columns as their median values.
1588    ///
1589    /// - Boolean and integer results are converted to `f64`. However, they are still
1590    ///   susceptible to overflow before this conversion occurs.
1591    /// - String columns will sum to None.
1592    pub fn median(self) -> Self {
1593        self.map_private(DslFunction::Stats(StatsFunction::Median))
1594    }
1595
1596    /// Aggregate all the columns as their quantile values.
1597    pub fn quantile(self, quantile: Expr, method: QuantileMethod) -> Self {
1598        self.map_private(DslFunction::Stats(StatsFunction::Quantile {
1599            quantile,
1600            method,
1601        }))
1602    }
1603
1604    /// Aggregate all the columns as their standard deviation values.
1605    ///
1606    /// `ddof` is the "Delta Degrees of Freedom"; `N - ddof` will be the denominator when
1607    /// computing the variance, where `N` is the number of rows.
1608    /// > In standard statistical practice, `ddof=1` provides an unbiased estimator of the
1609    /// > variance of a hypothetical infinite population. `ddof=0` provides a maximum
1610    /// > likelihood estimate of the variance for normally distributed variables. The
1611    /// > standard deviation computed in this function is the square root of the estimated
1612    /// > variance, so even with `ddof=1`, it will not be an unbiased estimate of the
1613    /// > standard deviation per se.
1614    ///
1615    /// Source: [Numpy](https://numpy.org/doc/stable/reference/generated/numpy.std.html#)
1616    pub fn std(self, ddof: u8) -> Self {
1617        self.map_private(DslFunction::Stats(StatsFunction::Std { ddof }))
1618    }
1619
1620    /// Aggregate all the columns as their variance values.
1621    ///
1622    /// `ddof` is the "Delta Degrees of Freedom"; `N - ddof` will be the denominator when
1623    /// computing the variance, where `N` is the number of rows.
1624    /// > In standard statistical practice, `ddof=1` provides an unbiased estimator of the
1625    /// > variance of a hypothetical infinite population. `ddof=0` provides a maximum
1626    /// > likelihood estimate of the variance for normally distributed variables.
1627    ///
1628    /// Source: [Numpy](https://numpy.org/doc/stable/reference/generated/numpy.var.html#)
1629    pub fn var(self, ddof: u8) -> Self {
1630        self.map_private(DslFunction::Stats(StatsFunction::Var { ddof }))
1631    }
1632
1633    /// Apply explode operation. [See eager explode](polars_core::frame::DataFrame::explode).
1634    pub fn explode(self, columns: Selector, options: ExplodeOptions) -> LazyFrame {
1635        self.explode_impl(columns, options, false)
1636    }
1637
1638    /// Apply explode operation. [See eager explode](polars_core::frame::DataFrame::explode).
1639    fn explode_impl(
1640        self,
1641        columns: Selector,
1642        options: ExplodeOptions,
1643        allow_empty: bool,
1644    ) -> LazyFrame {
1645        let opt_state = self.get_opt_state();
1646        let lp = self
1647            .get_plan_builder()
1648            .explode(columns, options, allow_empty)
1649            .build();
1650        Self::from_logical_plan(lp, opt_state)
1651    }
1652
1653    /// Aggregate all the columns as the sum of their null value count.
1654    pub fn null_count(self) -> LazyFrame {
1655        self.select(vec![col(PlSmallStr::from_static("*")).null_count()])
1656    }
1657
1658    /// Drop non-unique rows and maintain the order of kept rows.
1659    ///
1660    /// `subset` is an optional `Vec` of column names to consider for uniqueness; if
1661    /// `None`, all columns are considered.
1662    pub fn unique_stable(
1663        self,
1664        subset: Option<Selector>,
1665        keep_strategy: UniqueKeepStrategy,
1666    ) -> LazyFrame {
1667        let subset = subset.map(|s| vec![Expr::Selector(s)]);
1668        self.unique_stable_generic(subset, keep_strategy)
1669    }
1670
1671    pub fn unique_stable_generic(
1672        self,
1673        subset: Option<Vec<Expr>>,
1674        keep_strategy: UniqueKeepStrategy,
1675    ) -> LazyFrame {
1676        let opt_state = self.get_opt_state();
1677        let options = DistinctOptionsDSL {
1678            subset,
1679            maintain_order: true,
1680            keep_strategy,
1681        };
1682        let lp = self.get_plan_builder().distinct(options).build();
1683        Self::from_logical_plan(lp, opt_state)
1684    }
1685
1686    /// Drop non-unique rows without maintaining the order of kept rows.
1687    ///
1688    /// The order of the kept rows may change; to maintain the original row order, use
1689    /// [`unique_stable`](LazyFrame::unique_stable).
1690    ///
1691    /// `subset` is an optional `Vec` of column names to consider for uniqueness; if None,
1692    /// all columns are considered.
1693    pub fn unique(self, subset: Option<Selector>, keep_strategy: UniqueKeepStrategy) -> LazyFrame {
1694        let subset = subset.map(|s| vec![Expr::Selector(s)]);
1695        self.unique_generic(subset, keep_strategy)
1696    }
1697
1698    pub fn unique_generic(
1699        self,
1700        subset: Option<Vec<Expr>>,
1701        keep_strategy: UniqueKeepStrategy,
1702    ) -> LazyFrame {
1703        let opt_state = self.get_opt_state();
1704        let options = DistinctOptionsDSL {
1705            subset,
1706            maintain_order: false,
1707            keep_strategy,
1708        };
1709        let lp = self.get_plan_builder().distinct(options).build();
1710        Self::from_logical_plan(lp, opt_state)
1711    }
1712
1713    /// Drop rows containing one or more NaN values.
1714    ///
1715    /// `subset` is an optional `Vec` of column names to consider for NaNs; if None, all
1716    /// floating point columns are considered.
1717    pub fn drop_nans(self, subset: Option<Selector>) -> LazyFrame {
1718        let opt_state = self.get_opt_state();
1719        let lp = self.get_plan_builder().drop_nans(subset).build();
1720        Self::from_logical_plan(lp, opt_state)
1721    }
1722
1723    /// Drop rows containing one or more None values.
1724    ///
1725    /// `subset` is an optional `Vec` of column names to consider for nulls; if None, all
1726    /// columns are considered.
1727    pub fn drop_nulls(self, subset: Option<Selector>) -> LazyFrame {
1728        let opt_state = self.get_opt_state();
1729        let lp = self.get_plan_builder().drop_nulls(subset).build();
1730        Self::from_logical_plan(lp, opt_state)
1731    }
1732
1733    /// Slice the DataFrame.
1734    ///
1735    /// The slice is given as a [`Slice`] or as an `(offset, len)` tuple. A negative `offset`
1736    /// counts from the end of the DataFrame: `lf.slice((-5i64, 3))` gets three rows, starting at
1737    /// the row fifth from the end. [`Slice::skip`] gives a slice that runs to the end, or `None`
1738    /// for an offset of 0.
1739    ///
1740    /// If the slice extends beyond the end of the DataFrame, only the rows up to the end are
1741    /// returned.
1742    pub fn slice(self, slice: impl Into<Slice>) -> LazyFrame {
1743        let opt_state = self.get_opt_state();
1744        let lp = self.get_plan_builder().slice(slice.into()).build();
1745        Self::from_logical_plan(lp, opt_state)
1746    }
1747
1748    /// Remove all the rows of the LazyFrame.
1749    pub fn clear(self) -> LazyFrame {
1750        self.slice(Slice::EMPTY)
1751    }
1752
1753    /// Get the first row.
1754    pub fn first(self) -> LazyFrame {
1755        self.slice(Slice::head(1))
1756    }
1757
1758    /// Get the last row.
1759    pub fn last(self) -> LazyFrame {
1760        self.slice(Slice::tail(1))
1761    }
1762
1763    /// Get the last `n` rows.
1764    pub fn tail(self, n: IdxSize) -> LazyFrame {
1765        self.slice(Slice::tail(n as usize))
1766    }
1767
1768    #[cfg(feature = "pivot")]
1769    #[expect(clippy::too_many_arguments)]
1770    pub fn pivot(
1771        self,
1772        on: Selector,
1773        on_columns: Arc<DataFrame>,
1774        index: Selector,
1775        values: Selector,
1776        agg: Expr,
1777        maintain_order: bool,
1778        separator: PlSmallStr,
1779        column_naming: PivotColumnNaming,
1780    ) -> LazyFrame {
1781        let opt_state = self.get_opt_state();
1782        let lp = self
1783            .get_plan_builder()
1784            .pivot(
1785                on,
1786                on_columns,
1787                index,
1788                values,
1789                agg,
1790                maintain_order,
1791                separator,
1792                column_naming,
1793            )
1794            .build();
1795        Self::from_logical_plan(lp, opt_state)
1796    }
1797
1798    /// Unpivot the DataFrame from wide to long format.
1799    ///
1800    /// See [`UnpivotArgsIR`] for information on how to unpivot a DataFrame.
1801    #[cfg(feature = "pivot")]
1802    pub fn unpivot(self, args: UnpivotArgsDSL) -> LazyFrame {
1803        let opt_state = self.get_opt_state();
1804        let lp = self.get_plan_builder().unpivot(args).build();
1805        Self::from_logical_plan(lp, opt_state)
1806    }
1807
1808    /// Limit the DataFrame to the first `n` rows.
1809    pub fn limit(self, n: IdxSize) -> LazyFrame {
1810        self.slice(Slice::head(n as usize))
1811    }
1812
1813    /// Apply a function/closure once the logical plan get executed.
1814    ///
1815    /// The function has access to the whole materialized DataFrame at the time it is
1816    /// called.
1817    ///
1818    /// To apply specific functions to specific columns, use [`Expr::map`] in conjunction
1819    /// with `LazyFrame::with_column` or `with_columns`.
1820    ///
1821    /// ## Warning
1822    /// This can blow up in your face if the schema is changed due to the operation. The
1823    /// optimizer relies on a correct schema.
1824    ///
1825    /// You can toggle certain optimizations off.
1826    pub fn map<F>(
1827        self,
1828        function: F,
1829        optimizations: AllowedOptimizations,
1830        schema: Option<Arc<dyn UdfSchema>>,
1831        name: Option<&'static str>,
1832    ) -> LazyFrame
1833    where
1834        F: 'static + Fn(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
1835    {
1836        let opt_state = self.get_opt_state();
1837        let lp = self
1838            .get_plan_builder()
1839            .map(
1840                function,
1841                optimizations,
1842                schema,
1843                PlSmallStr::from_static(name.unwrap_or("ANONYMOUS UDF")),
1844            )
1845            .build();
1846        Self::from_logical_plan(lp, opt_state)
1847    }
1848
1849    #[cfg(feature = "python")]
1850    pub fn map_python(
1851        self,
1852        function: polars_utils::python_function::PythonFunction,
1853        optimizations: AllowedOptimizations,
1854        schema: Option<SchemaRef>,
1855        validate_output: bool,
1856    ) -> LazyFrame {
1857        let opt_state = self.get_opt_state();
1858        let lp = self
1859            .get_plan_builder()
1860            .map_python(function, optimizations, schema, validate_output)
1861            .build();
1862        Self::from_logical_plan(lp, opt_state)
1863    }
1864
1865    pub(crate) fn map_private(self, function: DslFunction) -> LazyFrame {
1866        let opt_state = self.get_opt_state();
1867        let lp = self.get_plan_builder().map_private(function).build();
1868        Self::from_logical_plan(lp, opt_state)
1869    }
1870
1871    /// Add a new column at index 0 that counts the rows.
1872    ///
1873    /// `name` is the name of the new column. `offset` is where to start counting from; if
1874    /// `None`, it is set to `0`.
1875    ///
1876    /// # Warning
1877    /// This can have a negative effect on query performance. This may for instance block
1878    /// predicate pushdown optimization.
1879    pub fn with_row_index<S>(self, name: S, offset: Option<IdxSize>) -> LazyFrame
1880    where
1881        S: Into<PlSmallStr>,
1882    {
1883        let name = name.into();
1884
1885        match &self.logical_plan {
1886            v @ DslPlan::Scan {
1887                scan_type,
1888                unified_scan_args,
1889                ..
1890            } if unified_scan_args.row_index.is_none()
1891                && !matches!(
1892                    &**scan_type,
1893                    FileScanDsl::Anonymous { .. } | FileScanDsl::ExpandedPaths { .. }
1894                ) =>
1895            {
1896                let DslPlan::Scan {
1897                    sources,
1898                    mut unified_scan_args,
1899                    scan_type,
1900                    cached_ir: _,
1901                } = v.clone()
1902                else {
1903                    unreachable!()
1904                };
1905
1906                unified_scan_args.row_index = Some(RowIndex {
1907                    name,
1908                    offset: offset.unwrap_or(0),
1909                });
1910
1911                DslPlan::Scan {
1912                    sources,
1913                    unified_scan_args,
1914                    scan_type,
1915                    cached_ir: Default::default(),
1916                }
1917                .into()
1918            },
1919            _ => self.map_private(DslFunction::RowIndex { name, offset }),
1920        }
1921    }
1922
1923    /// Return the number of non-null elements for each column.
1924    pub fn count(self) -> LazyFrame {
1925        self.select(vec![col(PlSmallStr::from_static("*")).count()])
1926    }
1927
1928    /// Unnest the given `Struct` columns: the fields of the `Struct` type will be
1929    /// inserted as columns.
1930    #[cfg(feature = "dtype-struct")]
1931    pub fn unnest(self, cols: Selector, separator: Option<PlSmallStr>) -> Self {
1932        self.map_private(DslFunction::Unnest {
1933            columns: cols,
1934            separator,
1935        })
1936    }
1937
1938    #[cfg(feature = "merge_sorted")]
1939    pub fn merge_sorted<I, S>(
1940        self,
1941        other: LazyFrame,
1942        key: I,
1943        maintain_order: bool,
1944    ) -> PolarsResult<LazyFrame>
1945    where
1946        I: IntoIterator<Item = S>,
1947        S: Into<PlSmallStr>,
1948    {
1949        let key: Arc<[PlSmallStr]> = key.into_iter().map(Into::into).collect();
1950
1951        polars_ensure!(
1952            !key.is_empty(),
1953            ComputeError: "merge_sorted requires at least one key column"
1954        );
1955
1956        let lp = DslPlan::MergeSorted {
1957            input_left: Arc::new(self.logical_plan),
1958            input_right: Arc::new(other.logical_plan),
1959            key,
1960            maintain_order,
1961        };
1962        Ok(LazyFrame::from_logical_plan(lp, self.opt_state))
1963    }
1964
1965    pub fn hint(self, hint: HintIR) -> PolarsResult<LazyFrame> {
1966        let lp = DslPlan::MapFunction {
1967            input: Arc::new(self.logical_plan),
1968            function: DslFunction::Hint(hint),
1969        };
1970        Ok(LazyFrame::from_logical_plan(lp, self.opt_state))
1971    }
1972}
1973
1974/// Utility struct for lazy group_by operation.
1975#[derive(Clone)]
1976pub struct LazyGroupBy {
1977    pub logical_plan: DslPlan,
1978    opt_state: OptFlags,
1979    keys: Vec<Expr>,
1980    predicates: Vec<Expr>,
1981    maintain_order: bool,
1982    #[cfg(feature = "dynamic_group_by")]
1983    dynamic_options: Option<DynamicGroupOptions>,
1984    #[cfg(feature = "dynamic_group_by")]
1985    rolling_options: Option<RollingGroupOptions>,
1986}
1987
1988impl From<LazyGroupBy> for LazyFrame {
1989    fn from(lgb: LazyGroupBy) -> Self {
1990        Self {
1991            logical_plan: lgb.logical_plan,
1992            opt_state: lgb.opt_state,
1993            cached_arena: Default::default(),
1994        }
1995    }
1996}
1997
1998impl LazyGroupBy {
1999    /// Filter groups with a predicate after aggregation.
2000    ///
2001    /// Similarly to the [LazyGroupBy::agg] method, the predicate must run an aggregation as it
2002    /// is evaluated on the groups.
2003    /// This method can be chained in which case all predicates must evaluate to `true` for a
2004    /// group to be kept.
2005    ///
2006    /// # Example
2007    ///
2008    /// ```rust
2009    /// use polars_core::prelude::*;
2010    /// use polars_lazy::prelude::*;
2011    ///
2012    /// fn example(df: DataFrame) -> LazyFrame {
2013    ///       df.lazy()
2014    ///        .group_by_stable([col("date")])
2015    ///        .having(col("rain").sum().gt(lit(10)))
2016    ///        .agg([col("rain").min().alias("min_rain")])
2017    /// }
2018    /// ```
2019    pub fn having(mut self, predicate: Expr) -> Self {
2020        self.predicates.push(predicate);
2021        self
2022    }
2023
2024    /// Group by and aggregate.
2025    ///
2026    /// Select a column with [col] and choose an aggregation.
2027    /// If you want to aggregate all columns use `col(PlSmallStr::from_static("*"))`.
2028    ///
2029    /// # Example
2030    ///
2031    /// ```rust
2032    /// use polars_core::prelude::*;
2033    /// use polars_lazy::prelude::*;
2034    ///
2035    /// fn example(df: DataFrame) -> LazyFrame {
2036    ///       df.lazy()
2037    ///        .group_by_stable([col("date")])
2038    ///        .agg([
2039    ///            col("rain").min().alias("min_rain"),
2040    ///            col("rain").sum().alias("sum_rain"),
2041    ///            col("rain").quantile(lit(0.5), QuantileMethod::Nearest).alias("median_rain"),
2042    ///        ])
2043    /// }
2044    /// ```
2045    pub fn agg<E: AsRef<[Expr]>>(self, aggs: E) -> LazyFrame {
2046        #[cfg(feature = "dynamic_group_by")]
2047        let lp = DslBuilder::from(self.logical_plan)
2048            .group_by(
2049                self.keys,
2050                self.predicates,
2051                aggs,
2052                None,
2053                self.maintain_order,
2054                self.dynamic_options,
2055                self.rolling_options,
2056            )
2057            .build();
2058
2059        #[cfg(not(feature = "dynamic_group_by"))]
2060        let lp = DslBuilder::from(self.logical_plan)
2061            .group_by(self.keys, self.predicates, aggs, None, self.maintain_order)
2062            .build();
2063        LazyFrame::from_logical_plan(lp, self.opt_state)
2064    }
2065
2066    /// Return first n rows of each group
2067    pub fn head(self, n: Option<usize>) -> LazyFrame {
2068        let keys = self
2069            .keys
2070            .iter()
2071            .filter_map(|expr| expr_output_name(expr).ok())
2072            .collect::<Vec<_>>();
2073
2074        self.agg([all().as_expr().head(n)]).explode_impl(
2075            all() - by_name(keys.iter().cloned(), false, false),
2076            ExplodeOptions {
2077                empty_as_null: true,
2078                keep_nulls: true,
2079            },
2080            true,
2081        )
2082    }
2083
2084    /// Return last n rows of each group
2085    pub fn tail(self, n: Option<usize>) -> LazyFrame {
2086        let keys = self
2087            .keys
2088            .iter()
2089            .filter_map(|expr| expr_output_name(expr).ok())
2090            .collect::<Vec<_>>();
2091
2092        self.agg([all().as_expr().tail(n)]).explode_impl(
2093            all() - by_name(keys.iter().cloned(), false, false),
2094            ExplodeOptions {
2095                empty_as_null: true,
2096                keep_nulls: true,
2097            },
2098            true,
2099        )
2100    }
2101
2102    /// Apply a function over the groups as a new DataFrame.
2103    ///
2104    /// **It is not recommended that you use this as materializing the DataFrame is very
2105    /// expensive.**
2106    pub fn apply(self, f: PlanCallback<DataFrame, DataFrame>, schema: SchemaRef) -> LazyFrame {
2107        if !self.predicates.is_empty() {
2108            panic!("not yet implemented: `apply` cannot be used with `having` predicates");
2109        }
2110
2111        #[cfg(feature = "dynamic_group_by")]
2112        let options = GroupbyOptions {
2113            dynamic: self.dynamic_options,
2114            rolling: self.rolling_options,
2115            slice: None,
2116        };
2117
2118        #[cfg(not(feature = "dynamic_group_by"))]
2119        let options = GroupbyOptions { slice: None };
2120
2121        let lp = DslPlan::GroupBy {
2122            input: Arc::new(self.logical_plan),
2123            keys: self.keys,
2124            predicates: vec![],
2125            aggs: vec![],
2126            apply: Some((f, schema)),
2127            maintain_order: self.maintain_order,
2128            options: Arc::new(options),
2129        };
2130        LazyFrame::from_logical_plan(lp, self.opt_state)
2131    }
2132}
2133
2134#[must_use]
2135pub struct JoinBuilder {
2136    lf: LazyFrame,
2137    how: JoinType,
2138    other: Option<LazyFrame>,
2139    left_on: Vec<Expr>,
2140    right_on: Vec<Expr>,
2141    allow_parallel: bool,
2142    force_parallel: bool,
2143    suffix: Option<PlSmallStr>,
2144    validation: JoinValidation,
2145    nulls_equal: bool,
2146    coalesce: JoinCoalesce,
2147    maintain_order: MaintainOrderJoin,
2148    build_side: Option<JoinBuildSide>,
2149}
2150impl JoinBuilder {
2151    /// Create the `JoinBuilder` with the provided `LazyFrame` as the left table.
2152    pub fn new(lf: LazyFrame) -> Self {
2153        Self {
2154            lf,
2155            other: None,
2156            how: JoinType::Inner,
2157            left_on: vec![],
2158            right_on: vec![],
2159            allow_parallel: true,
2160            force_parallel: false,
2161            suffix: None,
2162            validation: Default::default(),
2163            nulls_equal: false,
2164            coalesce: Default::default(),
2165            maintain_order: Default::default(),
2166            build_side: None,
2167        }
2168    }
2169
2170    /// The right table in the join.
2171    pub fn with(mut self, other: LazyFrame) -> Self {
2172        self.other = Some(other);
2173        self
2174    }
2175
2176    /// Select the join type.
2177    pub fn how(mut self, how: JoinType) -> Self {
2178        self.how = how;
2179        self
2180    }
2181
2182    pub fn validate(mut self, validation: JoinValidation) -> Self {
2183        self.validation = validation;
2184        self
2185    }
2186
2187    /// The expressions you want to join both tables on.
2188    ///
2189    /// The passed expressions must be valid in both `LazyFrame`s in the join.
2190    /// Selectors expand independently against each input schema. The results are
2191    /// paired in order and both sides must produce the same nonzero number of keys.
2192    pub fn on<E: AsRef<[Expr]>>(mut self, on: E) -> Self {
2193        let on = on.as_ref().to_vec();
2194        self.left_on.clone_from(&on);
2195        self.right_on = on;
2196        self
2197    }
2198
2199    /// The expressions you want to join the left table on.
2200    ///
2201    /// The passed expressions and selectors must be valid in the left table.
2202    /// The total expanded key count must be nonzero and match the right side.
2203    pub fn left_on<E: AsRef<[Expr]>>(mut self, on: E) -> Self {
2204        self.left_on = on.as_ref().to_vec();
2205        self
2206    }
2207
2208    /// The expressions you want to join the right table on.
2209    ///
2210    /// The passed expressions and selectors must be valid in the right table.
2211    /// The total expanded key count must be nonzero and match the left side.
2212    pub fn right_on<E: AsRef<[Expr]>>(mut self, on: E) -> Self {
2213        self.right_on = on.as_ref().to_vec();
2214        self
2215    }
2216
2217    /// Allow parallel table evaluation.
2218    pub fn allow_parallel(mut self, allow: bool) -> Self {
2219        self.allow_parallel = allow;
2220        self
2221    }
2222
2223    /// Force parallel table evaluation.
2224    pub fn force_parallel(mut self, force: bool) -> Self {
2225        self.force_parallel = force;
2226        self
2227    }
2228
2229    /// Join on null values. By default null values will never produce matches.
2230    pub fn join_nulls(mut self, nulls_equal: bool) -> Self {
2231        self.nulls_equal = nulls_equal;
2232        self
2233    }
2234
2235    /// Suffix to add duplicate column names in join.
2236    /// Defaults to `"_right"` if this method is never called.
2237    pub fn suffix<S>(mut self, suffix: S) -> Self
2238    where
2239        S: Into<PlSmallStr>,
2240    {
2241        self.suffix = Some(suffix.into());
2242        self
2243    }
2244
2245    /// Whether to coalesce join columns.
2246    pub fn coalesce(mut self, coalesce: JoinCoalesce) -> Self {
2247        self.coalesce = coalesce;
2248        self
2249    }
2250
2251    /// Whether to preserve the row order.
2252    pub fn maintain_order(mut self, maintain_order: MaintainOrderJoin) -> Self {
2253        self.maintain_order = maintain_order;
2254        self
2255    }
2256
2257    /// Whether to prefer a specific build side.
2258    pub fn build_side(mut self, build_side: Option<JoinBuildSide>) -> Self {
2259        self.build_side = build_side;
2260        self
2261    }
2262
2263    /// Finish builder
2264    pub fn finish(self) -> PolarsResult<LazyFrame> {
2265        let opt_state = self.lf.opt_state;
2266        let other = self.other.expect("'with' not set in join builder");
2267
2268        let args = JoinArgs {
2269            how: self.how,
2270            validation: self.validation,
2271            suffix: self.suffix,
2272            slice: None,
2273            nulls_equal: self.nulls_equal,
2274            coalesce: self.coalesce,
2275            maintain_order: self.maintain_order,
2276            build_side: self.build_side,
2277        };
2278
2279        let lp = self
2280            .lf
2281            .get_plan_builder()
2282            .join(
2283                other.logical_plan,
2284                self.left_on,
2285                self.right_on,
2286                JoinOptions {
2287                    allow_parallel: self.allow_parallel,
2288                    force_parallel: self.force_parallel,
2289                    args,
2290                }
2291                .into(),
2292            )?
2293            .build();
2294        Ok(LazyFrame::from_logical_plan(lp, opt_state))
2295    }
2296
2297    // Finish with join predicates
2298    pub fn join_where(self, predicates: Vec<Expr>) -> LazyFrame {
2299        let opt_state = self.lf.opt_state;
2300        let other = self.other.expect("with not set");
2301
2302        // Decompose `And` conjunctions into their component expressions
2303        fn decompose_and(predicate: Expr, expanded_predicates: &mut Vec<Expr>) {
2304            if let Expr::BinaryExpr {
2305                op: Operator::And,
2306                left,
2307                right,
2308            } = predicate
2309            {
2310                decompose_and((*left).clone(), expanded_predicates);
2311                decompose_and((*right).clone(), expanded_predicates);
2312            } else {
2313                expanded_predicates.push(predicate);
2314            }
2315        }
2316        let mut expanded_predicates = Vec::with_capacity(predicates.len() * 2);
2317        for predicate in predicates {
2318            decompose_and(predicate, &mut expanded_predicates);
2319        }
2320        let predicates: Vec<Expr> = expanded_predicates;
2321
2322        // Decompose `is_between` predicates to allow for cleaner expression of range joins
2323        #[cfg(feature = "is_between")]
2324        let predicates: Vec<Expr> = {
2325            let mut expanded_predicates = Vec::with_capacity(predicates.len() * 2);
2326            for predicate in predicates {
2327                if let Expr::Function {
2328                    function: FunctionExpr::Boolean(BooleanFunction::IsBetween { closed }),
2329                    input,
2330                    ..
2331                } = &predicate
2332                {
2333                    if let [expr, lower, upper] = input.as_slice() {
2334                        match closed {
2335                            ClosedInterval::Both => {
2336                                expanded_predicates.push(expr.clone().gt_eq(lower.clone()));
2337                                expanded_predicates.push(expr.clone().lt_eq(upper.clone()));
2338                            },
2339                            ClosedInterval::Right => {
2340                                expanded_predicates.push(expr.clone().gt(lower.clone()));
2341                                expanded_predicates.push(expr.clone().lt_eq(upper.clone()));
2342                            },
2343                            ClosedInterval::Left => {
2344                                expanded_predicates.push(expr.clone().gt_eq(lower.clone()));
2345                                expanded_predicates.push(expr.clone().lt(upper.clone()));
2346                            },
2347                            ClosedInterval::None => {
2348                                expanded_predicates.push(expr.clone().gt(lower.clone()));
2349                                expanded_predicates.push(expr.clone().lt(upper.clone()));
2350                            },
2351                        }
2352                        continue;
2353                    }
2354                }
2355                expanded_predicates.push(predicate);
2356            }
2357            expanded_predicates
2358        };
2359
2360        let args = JoinArgs {
2361            how: self.how,
2362            validation: self.validation,
2363            suffix: self.suffix,
2364            slice: None,
2365            nulls_equal: self.nulls_equal,
2366            coalesce: self.coalesce,
2367            maintain_order: self.maintain_order,
2368            build_side: self.build_side,
2369        };
2370        let options = JoinOptions {
2371            allow_parallel: self.allow_parallel,
2372            force_parallel: self.force_parallel,
2373            args,
2374        };
2375
2376        let lp = DslPlan::Join {
2377            input_left: Arc::new(self.lf.logical_plan),
2378            input_right: Arc::new(other.logical_plan),
2379            condition: JoinCondition::NonEqui { predicates },
2380            options: Arc::from(options),
2381        };
2382
2383        LazyFrame::from_logical_plan(lp, opt_state)
2384    }
2385}
2386
2387pub const BUILD_STREAMING_EXECUTOR: Option<polars_mem_engine::StreamingExecutorBuilder> = {
2388    #[cfg(not(feature = "streaming"))]
2389    {
2390        None
2391    }
2392    #[cfg(feature = "streaming")]
2393    {
2394        Some(polars_stream::build_streaming_query_executor)
2395    }
2396};
2397
2398pub(crate) fn run_in_memory_query(
2399    node: Node,
2400    ir_arena: &mut Arena<IR>,
2401    expr_arena: &mut Arena<AExpr>,
2402    engine: Engine,
2403    observer: Option<Box<dyn QueryObserver>>,
2404) -> PolarsResult<QueryResult> {
2405    let _guard = observer
2406        .as_ref()
2407        .map(|o| o.on_query_planned(to_planned_query(node, ir_arena, expr_arena)));
2408
2409    let result = if let IR::SinkMultiple { inputs } = ir_arena.get(node) {
2410        polars_ensure!(
2411            engine != Engine::Gpu,
2412            InvalidOperation:
2413            "collect_all is not supported for the gpu engine"
2414        );
2415
2416        let physical_plan = create_multiple_physical_plans(
2417            inputs.clone().as_slice(),
2418            ir_arena,
2419            expr_arena,
2420            BUILD_STREAMING_EXECUTOR,
2421        )?;
2422        physical_plan.execute().map(QueryResult::Multiple)
2423    } else {
2424        let mut physical_plan =
2425            create_physical_plan(node, ir_arena, expr_arena, BUILD_STREAMING_EXECUTOR)?;
2426        let mut state = ExecutionState::new();
2427        physical_plan.execute(&mut state).map(QueryResult::Single)
2428    };
2429
2430    result.inspect_err(|err| {
2431        if let Some(o) = observer.as_ref() {
2432            o.on_query_failed(err);
2433        }
2434    })
2435}
2436
2437fn to_planned_query(node: Node, ir_arena: &Arena<IR>, expr_arena: &Arena<AExpr>) -> PlannedQuery {
2438    let ir = ir_plan_to_description(&[node], ir_arena, expr_arena);
2439    PlannedQuery::new(ir)
2440}
2441
2442pub struct CollectBatches {
2443    recv: Receiver<PolarsResult<DataFrame>>,
2444    runner: Option<Box<dyn FnOnce() + Send + 'static>>,
2445}
2446
2447impl CollectBatches {
2448    /// Start running the query, if not already.
2449    pub fn start(&mut self) {
2450        if let Some(runner) = self.runner.take() {
2451            runner()
2452        }
2453    }
2454}
2455
2456impl Iterator for CollectBatches {
2457    type Item = PolarsResult<DataFrame>;
2458
2459    fn next(&mut self) -> Option<Self::Item> {
2460        self.start();
2461        self.recv.recv().ok()
2462    }
2463}