1#[cfg(feature = "python")]
3mod python;
4
5mod cached_arenas;
6mod err;
7#[cfg(not(target_arch = "wasm32"))]
8mod exitable;
9
10use std::num::NonZeroUsize;
11use std::sync::mpsc::{Receiver, sync_channel};
12use std::sync::{Arc, Mutex};
13
14pub use anonymous_scan::*;
15#[cfg(feature = "csv")]
16pub use csv::*;
17#[cfg(not(target_arch = "wasm32"))]
18pub use exitable::*;
19pub use file_list_reader::*;
20#[cfg(feature = "json")]
21pub use ndjson::*;
22#[cfg(feature = "parquet")]
23pub use parquet::*;
24use polars_compute::rolling::QuantileMethod;
25use polars_core::error::feature_gated;
26#[cfg(feature = "pivot")]
27use polars_core::frame::PivotColumnNaming;
28use polars_core::prelude::*;
29use polars_core::query_result::QueryResult;
30#[cfg(feature = "is_between")]
31use polars_defs::expr::ClosedInterval;
32use polars_defs::join::{
33 JoinArgs, JoinBuildSide, JoinCoalesce, JoinType, JoinValidation, MaintainOrderJoin,
34};
35use polars_defs::time::group_by::{DynamicGroupOptions, RollingGroupOptions};
36use polars_io::RowIndex;
37use polars_mem_engine::scan_predicate::functions::apply_scan_predicate_to_scan_ir;
38use polars_mem_engine::{Executor, create_multiple_physical_plans, create_physical_plan};
39use polars_observer::{PlannedQuery, QueryObserver};
40pub use polars_plan::frame::{AllowedOptimizations, OptFlags};
41use polars_plan::plans::ExecutionHooks;
42use polars_plan::prelude::ir_plan_to_description;
43use polars_utils::pl_str::PlSmallStr;
44use polars_utils::slice_enum::Slice;
45
46use crate::frame::cached_arenas::CachedArena;
47use crate::prelude::*;
48
49pub trait IntoLazy {
50 fn lazy(self) -> LazyFrame;
51}
52
53impl IntoLazy for DataFrame {
54 fn lazy(self) -> LazyFrame {
56 let lp = DslBuilder::from_existing_df(self).build();
57 LazyFrame {
58 logical_plan: lp,
59 opt_state: Default::default(),
60 cached_arena: Default::default(),
61 }
62 }
63}
64
65fn hive_join(
67 left: &DataFrame,
68 right: &DataFrame,
69 left_on: &str,
70 right_on: &str,
71 args: JoinArgs,
72) -> PolarsResult<DataFrame> {
73 use polars_ops::frame::DataFrameJoinOps;
74
75 left.join(right, [left_on], [right_on], args, None)
76}
77
78impl IntoLazy for LazyFrame {
79 fn lazy(self) -> LazyFrame {
80 self
81 }
82}
83
84#[derive(Clone, Default)]
89#[must_use]
90pub struct LazyFrame {
91 pub logical_plan: DslPlan,
92 pub(crate) opt_state: OptFlags,
93 pub(crate) cached_arena: Arc<Mutex<Option<CachedArena>>>,
94}
95
96impl From<DslPlan> for LazyFrame {
97 fn from(plan: DslPlan) -> Self {
98 Self {
99 logical_plan: plan,
100 opt_state: OptFlags::default(),
101 cached_arena: Default::default(),
102 }
103 }
104}
105
106impl LazyFrame {
107 pub(crate) fn from_inner(
108 logical_plan: DslPlan,
109 opt_state: OptFlags,
110 cached_arena: Arc<Mutex<Option<CachedArena>>>,
111 ) -> Self {
112 Self {
113 logical_plan,
114 opt_state,
115 cached_arena,
116 }
117 }
118
119 pub(crate) fn get_plan_builder(self) -> DslBuilder {
120 DslBuilder::from(self.logical_plan)
121 }
122
123 fn get_opt_state(&self) -> OptFlags {
124 self.opt_state
125 }
126
127 pub fn from_logical_plan(logical_plan: DslPlan, opt_state: OptFlags) -> Self {
128 LazyFrame {
129 logical_plan,
130 opt_state,
131 cached_arena: Default::default(),
132 }
133 }
134
135 pub fn get_current_optimizations(&self) -> OptFlags {
137 self.opt_state
138 }
139
140 pub fn with_optimizations(mut self, opt_state: OptFlags) -> Self {
142 self.opt_state = opt_state;
143 self
144 }
145
146 pub fn without_optimizations(self) -> Self {
148 self.with_optimizations(OptFlags::from_bits_truncate(0) | OptFlags::TYPE_COERCION)
149 }
150
151 pub fn with_projection_pushdown(mut self, toggle: bool) -> Self {
153 self.opt_state.set(OptFlags::PROJECTION_PUSHDOWN, toggle);
154 self
155 }
156
157 pub fn with_cluster_with_columns(mut self, toggle: bool) -> Self {
159 self.opt_state.set(OptFlags::CLUSTER_WITH_COLUMNS, toggle);
160 self
161 }
162
163 pub fn with_check_order(mut self, toggle: bool) -> Self {
166 self.opt_state.set(OptFlags::CHECK_ORDER_OBSERVE, toggle);
167 self
168 }
169
170 pub fn with_predicate_pushdown(mut self, toggle: bool) -> Self {
172 self.opt_state.set(OptFlags::PREDICATE_PUSHDOWN, toggle);
173 self
174 }
175
176 pub fn with_type_coercion(mut self, toggle: bool) -> Self {
178 self.opt_state.set(OptFlags::TYPE_COERCION, toggle);
179 self
180 }
181
182 pub fn with_type_check(mut self, toggle: bool) -> Self {
184 self.opt_state.set(OptFlags::TYPE_CHECK, toggle);
185 self
186 }
187
188 pub fn with_simplify_expr(mut self, toggle: bool) -> Self {
190 self.opt_state.set(OptFlags::SIMPLIFY_EXPR, toggle);
191 self
192 }
193
194 #[cfg(feature = "cse")]
196 pub fn with_comm_subplan_elim(mut self, toggle: bool) -> Self {
197 self.opt_state.set(OptFlags::COMM_SUBPLAN_ELIM, toggle);
198 self
199 }
200
201 #[cfg(feature = "cse")]
203 pub fn with_comm_subexpr_elim(mut self, toggle: bool) -> Self {
204 self.opt_state.set(OptFlags::COMM_SUBEXPR_ELIM, toggle);
205 self
206 }
207
208 pub fn with_slice_pushdown(mut self, toggle: bool) -> Self {
210 self.opt_state.set(OptFlags::SLICE_PUSHDOWN, toggle);
211 self
212 }
213
214 #[cfg(feature = "streaming")]
215 pub fn with_streaming(mut self, toggle: bool) -> Self {
216 self.opt_state.set(OptFlags::STREAMING, toggle);
217 self
218 }
219
220 pub fn with_gpu(mut self, toggle: bool) -> Self {
221 self.opt_state.set(OptFlags::GPU, toggle);
222 self
223 }
224
225 pub fn with_row_estimate(mut self, toggle: bool) -> Self {
227 self.opt_state.set(OptFlags::ROW_ESTIMATE, toggle);
228 self
229 }
230
231 pub fn _with_eager(mut self, toggle: bool) -> Self {
233 self.opt_state.set(OptFlags::EAGER, toggle);
234 self
235 }
236
237 pub fn describe_plan(&self) -> PolarsResult<String> {
239 Ok(self.clone().to_alp()?.describe())
240 }
241
242 pub fn describe_plan_tree(&self) -> PolarsResult<String> {
244 Ok(self.clone().to_alp()?.describe_tree_format())
245 }
246
247 pub fn describe_optimized_plan(&self) -> PolarsResult<String> {
251 Ok(self.clone().to_alp_optimized()?.describe())
252 }
253
254 pub fn describe_optimized_plan_tree(&self) -> PolarsResult<String> {
258 Ok(self.clone().to_alp_optimized()?.describe_tree_format())
259 }
260
261 pub fn explain(&self, optimized: bool) -> PolarsResult<String> {
266 if optimized {
267 self.describe_optimized_plan()
268 } else {
269 self.describe_plan()
270 }
271 }
272
273 pub fn sort(self, by: impl IntoVec<PlSmallStr>, sort_options: SortMultipleOptions) -> Self {
313 let opt_state = self.get_opt_state();
314 let lp = self
315 .get_plan_builder()
316 .sort(by.into_vec().into_iter().map(col).collect(), sort_options)
317 .build();
318 Self::from_logical_plan(lp, opt_state)
319 }
320
321 pub fn sort_by_exprs<E: AsRef<[Expr]>>(
341 self,
342 by_exprs: E,
343 sort_options: SortMultipleOptions,
344 ) -> Self {
345 let by_exprs = by_exprs.as_ref().to_vec();
346 if by_exprs.is_empty() {
347 self
348 } else {
349 let opt_state = self.get_opt_state();
350 let lp = self.get_plan_builder().sort(by_exprs, sort_options).build();
351 Self::from_logical_plan(lp, opt_state)
352 }
353 }
354
355 pub fn top_k<E: AsRef<[Expr]>>(
356 self,
357 k: IdxSize,
358 by_exprs: E,
359 sort_options: SortMultipleOptions,
360 ) -> Self {
361 self.sort_by_exprs(
363 by_exprs,
364 sort_options.with_order_reversed().with_nulls_last(true),
365 )
366 .slice(Slice::head(k as usize))
367 }
368
369 pub fn bottom_k<E: AsRef<[Expr]>>(
370 self,
371 k: IdxSize,
372 by_exprs: E,
373 sort_options: SortMultipleOptions,
374 ) -> Self {
375 self.sort_by_exprs(by_exprs, sort_options.with_nulls_last(true))
377 .slice(Slice::head(k as usize))
378 }
379
380 pub fn reverse(self) -> Self {
396 self.select(vec![col(PlSmallStr::from_static("*")).reverse()])
397 }
398
399 pub fn rename<I, J, T, S>(self, existing: I, new: J, strict: bool) -> Self
407 where
408 I: IntoIterator<Item = T>,
409 J: IntoIterator<Item = S>,
410 T: AsRef<str>,
411 S: AsRef<str>,
412 {
413 let iter = existing.into_iter();
414 let cap = iter.size_hint().0;
415 let mut existing_vec: Vec<PlSmallStr> = Vec::with_capacity(cap);
416 let mut new_vec: Vec<PlSmallStr> = Vec::with_capacity(cap);
417
418 for (existing, new) in iter.zip(new) {
421 let existing = existing.as_ref();
422 let new = new.as_ref();
423 if new != existing {
424 existing_vec.push(existing.into());
425 new_vec.push(new.into());
426 }
427 }
428
429 self.map_private(DslFunction::Rename {
430 existing: existing_vec.into(),
431 new: new_vec.into(),
432 strict,
433 })
434 }
435
436 pub fn drop(self, columns: Selector) -> Self {
443 let opt_state = self.get_opt_state();
444 let lp = self.get_plan_builder().drop(columns).build();
445 Self::from_logical_plan(lp, opt_state)
446 }
447
448 pub fn shift<E: Into<Expr>>(self, n: E) -> Self {
453 self.select(vec![col(PlSmallStr::from_static("*")).shift(n.into())])
454 }
455
456 pub fn shift_and_fill<E: Into<Expr>, IE: Into<Expr>>(self, n: E, fill_value: IE) -> Self {
461 self.select(vec![
462 col(PlSmallStr::from_static("*")).shift_and_fill(n.into(), fill_value.into()),
463 ])
464 }
465
466 pub fn fill_null<E: Into<Expr>>(self, fill_value: E) -> LazyFrame {
468 let opt_state = self.get_opt_state();
469 let lp = self.get_plan_builder().fill_null(fill_value.into()).build();
470 Self::from_logical_plan(lp, opt_state)
471 }
472
473 pub fn fill_nan<E: Into<Expr>>(self, fill_value: E) -> LazyFrame {
475 let opt_state = self.get_opt_state();
476 let lp = self.get_plan_builder().fill_nan(fill_value.into()).build();
477 Self::from_logical_plan(lp, opt_state)
478 }
479
480 pub fn cache(self) -> Self {
484 let opt_state = self.get_opt_state();
485 let lp = self.get_plan_builder().cache().build();
486 Self::from_logical_plan(lp, opt_state)
487 }
488
489 pub fn cast(self, dtypes: PlHashMap<&str, DataType>, strict: bool) -> Self {
491 let cast_cols: Vec<Expr> = dtypes
492 .into_iter()
493 .map(|(name, dt)| {
494 let name = PlSmallStr::from_str(name);
495
496 if strict {
497 col(name).strict_cast(dt)
498 } else {
499 col(name).cast(dt)
500 }
501 })
502 .collect();
503
504 if cast_cols.is_empty() {
505 self
506 } else {
507 self.with_columns(cast_cols)
508 }
509 }
510
511 pub fn cast_all(self, dtype: impl Into<DataTypeExpr>, strict: bool) -> Self {
513 self.with_columns(vec![if strict {
514 col(PlSmallStr::from_static("*")).strict_cast(dtype)
515 } else {
516 col(PlSmallStr::from_static("*")).cast(dtype)
517 }])
518 }
519
520 pub fn optimize(
521 self,
522 lp_arena: &mut Arena<IR>,
523 expr_arena: &mut Arena<AExpr>,
524 ) -> PolarsResult<Node> {
525 self.optimize_with_scratch(lp_arena, expr_arena, &mut vec![])
526 }
527
528 pub fn to_alp_optimized(mut self) -> PolarsResult<IRPlan> {
529 let (mut lp_arena, mut expr_arena) = self.get_arenas();
530 let node = self.optimize_with_scratch(&mut lp_arena, &mut expr_arena, &mut vec![])?;
531
532 Ok(IRPlan::new(node, lp_arena, expr_arena))
533 }
534
535 pub fn to_alp(mut self) -> PolarsResult<IRPlan> {
536 let (mut lp_arena, mut expr_arena) = self.get_arenas();
537 let node = to_alp(
538 self.logical_plan,
539 &mut expr_arena,
540 &mut lp_arena,
541 &mut self.opt_state,
542 )?;
543 let plan = IRPlan::new(node, lp_arena, expr_arena);
544 Ok(plan)
545 }
546
547 pub(crate) fn optimize_with_scratch(
548 self,
549 ir_arena: &mut Arena<IR>,
550 expr_arena: &mut Arena<AExpr>,
551 scratch: &mut Vec<Node>,
552 ) -> PolarsResult<Node> {
553 let mut opt_flags = self.opt_state;
554 #[allow(clippy::eq_op)]
557 #[cfg(feature = "cse")]
558 if opt_flags.contains(OptFlags::EAGER) {
559 opt_flags &= !(OptFlags::COMM_SUBEXPR_ELIM | OptFlags::COMM_SUBEXPR_ELIM);
560 }
561 let root = to_alp(self.logical_plan, expr_arena, ir_arena, &mut opt_flags)?;
562
563 let lp_top = optimize(
564 root,
565 opt_flags,
566 ir_arena,
567 expr_arena,
568 scratch,
569 ExecutionHooks {
570 apply_scan_predicate_to_scan_ir,
571 hive_join,
572 },
573 )?;
574
575 Ok(lp_top)
576 }
577
578 fn prepare_collect_post_opt<P>(
579 mut self,
580 check_sink: bool,
581 post_opt: P,
582 ) -> PolarsResult<(ExecutionState, Box<dyn Executor>, bool)>
583 where
584 P: FnOnce(Node, &mut Arena<IR>, &mut Arena<AExpr>) -> PolarsResult<()>,
585 {
586 let (mut lp_arena, mut expr_arena) = self.get_arenas();
587
588 let mut scratch = vec![];
589 let lp_top = self.optimize_with_scratch(&mut lp_arena, &mut expr_arena, &mut scratch)?;
590 post_opt(lp_top, &mut lp_arena, &mut expr_arena)?;
591
592 let no_file_sink = if check_sink {
594 !matches!(
595 lp_arena.get(lp_top),
596 IR::Sink {
597 payload: SinkTypeIR::File { .. },
598 ..
599 }
600 )
601 } else {
602 true
603 };
604 let physical_plan = create_physical_plan(
605 lp_top,
606 &mut lp_arena,
607 &mut expr_arena,
608 BUILD_STREAMING_EXECUTOR,
609 )?;
610
611 let state = ExecutionState::new();
612 Ok((state, physical_plan, no_file_sink))
613 }
614
615 pub fn _collect_post_opt<P>(self, post_opt: P) -> PolarsResult<DataFrame>
621 where
622 P: FnOnce(Node, &mut Arena<IR>, &mut Arena<AExpr>) -> PolarsResult<()>,
623 {
624 let (mut state, mut physical_plan, _) = self
625 .with_gpu(true)
626 .prepare_collect_post_opt(false, post_opt)?;
627 physical_plan.execute(&mut state)
628 }
629
630 #[allow(unused_mut)]
631 fn prepare_collect(
632 self,
633 check_sink: bool,
634 ) -> PolarsResult<(ExecutionState, Box<dyn Executor>, bool)> {
635 self.prepare_collect_post_opt(check_sink, |_, _, _| Ok(()))
636 }
637
638 pub fn collect_with_engine(mut self, engine: Engine) -> PolarsResult<QueryResult> {
643 let engine = match engine {
644 Engine::Streaming => Engine::Streaming,
645 _ if std::env::var("POLARS_FORCE_STREAMING").as_deref() == Ok("1") => Engine::Streaming,
646 Engine::Auto => {
647 if self.opt_state.eager() {
648 Engine::InMemory
649 } else {
650 Engine::Streaming
651 }
652 },
653 v => v,
654 };
655
656 if engine != Engine::Streaming
657 && std::env::var("POLARS_AUTO_STREAMING").as_deref() == Ok("1")
658 {
659 feature_gated!("streaming", {
660 if let Some(r) = self.clone()._collect_with_streaming_suppress_todo_panic() {
661 return r;
662 }
663 })
664 }
665 match engine {
666 Engine::Streaming => {
667 feature_gated!("streaming", self = self.with_streaming(true))
668 },
669 Engine::Gpu => self = self.with_gpu(true),
670 _ => (),
671 }
672
673 let observer = self
674 .opt_state
675 .query_monitoring()
676 .then(polars_observer::new_query_observer)
677 .flatten();
678
679 if let Some(o) = observer.as_ref() {
680 o.on_query_started()
681 }
682
683 let mut ir_plan = self.to_alp_optimized().inspect_err(|err| {
684 if let Some(o) = observer.as_ref() {
685 o.on_query_failed(err)
686 }
687 })?;
688 ir_plan.ensure_root_node_is_sink();
689
690 match engine {
691 Engine::Streaming => feature_gated!("streaming", {
692 polars_stream::run_query(
693 ir_plan.lp_top,
694 &mut ir_plan.lp_arena,
695 &mut ir_plan.expr_arena,
696 observer,
697 )
698 }),
699 Engine::InMemory | Engine::Gpu => run_in_memory_query(
700 ir_plan.lp_top,
701 &mut ir_plan.lp_arena,
702 &mut ir_plan.expr_arena,
703 engine,
704 observer,
705 ),
706 Engine::Auto => unreachable!(),
707 }
708 }
709
710 pub fn explain_all(plans: Vec<DslPlan>, opt_state: OptFlags) -> PolarsResult<String> {
711 let sink_multiple = LazyFrame {
712 logical_plan: DslPlan::SinkMultiple { inputs: plans },
713 opt_state,
714 cached_arena: Default::default(),
715 };
716 sink_multiple.explain(true)
717 }
718
719 pub fn collect_all_with_engine(
720 plans: Vec<DslPlan>,
721 engine: Engine,
722 opt_state: OptFlags,
723 ) -> PolarsResult<Vec<DataFrame>> {
724 if plans.is_empty() {
725 return Ok(Vec::new());
726 }
727
728 LazyFrame {
729 logical_plan: DslPlan::SinkMultiple { inputs: plans },
730 opt_state,
731 cached_arena: Default::default(),
732 }
733 .collect_with_engine(engine)
734 .map(|r| r.unwrap_multiple())
735 }
736
737 pub fn collect(self) -> PolarsResult<DataFrame> {
755 self.collect_with_engine(Engine::Auto).map(|r| match r {
756 QueryResult::Single(df) => df,
757 QueryResult::Multiple(_) => DataFrame::empty(),
759 })
760 }
761
762 #[cfg(feature = "async")]
767 pub fn collect_batches(
768 self,
769 engine: Engine,
770 maintain_order: bool,
771 chunk_size: Option<NonZeroUsize>,
772 lazy: bool,
773 ) -> PolarsResult<CollectBatches> {
774 let (send, recv) = sync_channel(1);
775 let runner_send = send.clone();
776 let ldf = self.sink_batches(
777 PlanCallback::new(move |df| {
778 let send_result = send.send(Ok(df));
780 Ok(send_result.is_err())
781 }),
782 maintain_order,
783 chunk_size,
784 )?;
785 let runner = move || {
786 polars_core::runtime::ASYNC.spawn_blocking(move || {
788 if let Err(e) = ldf.collect_with_engine(engine) {
789 runner_send.send(Err(e)).ok();
790 }
791 });
792 };
793
794 let mut collect_batches = CollectBatches {
795 recv,
796 runner: Some(Box::new(runner)),
797 };
798 if !lazy {
799 collect_batches.start();
800 }
801 Ok(collect_batches)
802 }
803
804 pub fn sink_batches(
805 mut self,
806 function: PlanCallback<DataFrame, bool>,
807 maintain_order: bool,
808 chunk_size: Option<NonZeroUsize>,
809 ) -> PolarsResult<Self> {
810 use polars_plan::prelude::sink::CallbackSinkType;
811
812 polars_ensure!(
813 !matches!(self.logical_plan, DslPlan::Sink { .. }),
814 InvalidOperation: "cannot create a sink on top of another sink"
815 );
816
817 self.logical_plan = DslPlan::Sink {
818 input: Arc::new(self.logical_plan),
819 payload: SinkType::Callback(CallbackSinkType {
820 function,
821 maintain_order,
822 chunk_size,
823 }),
824 };
825
826 Ok(self)
827 }
828
829 #[cfg(feature = "streaming")]
831 fn _collect_with_streaming_suppress_todo_panic(
832 mut self,
833 ) -> Option<PolarsResult<polars_core::query_result::QueryResult>> {
834 self.opt_state |= OptFlags::STREAMING;
835 let mut ir_plan = match self.to_alp_optimized() {
836 Ok(v) => v,
837 Err(e) => return Some(Err(e)),
838 };
839
840 ir_plan.ensure_root_node_is_sink();
841
842 let f = || {
843 polars_stream::run_query(
844 ir_plan.lp_top,
845 &mut ir_plan.lp_arena,
846 &mut ir_plan.expr_arena,
847 None,
848 )
849 };
850
851 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
852 Ok(v) => Some(v),
853 Err(e) => {
854 if e.downcast_ref::<&str>()
857 .is_some_and(|s| s.starts_with("not yet implemented"))
858 {
859 if polars_core::config::verbose() {
860 eprintln!(
861 "caught unimplemented error in new streaming engine, falling back to normal engine"
862 );
863 }
864 None
865 } else {
866 std::panic::resume_unwind(e)
867 }
868 },
869 }
870 }
871
872 pub fn sink(
873 mut self,
874 sink_type: SinkDestination,
875 file_format: FileWriteFormat,
876 unified_sink_args: UnifiedSinkArgs,
877 ) -> PolarsResult<Self> {
878 polars_ensure!(
879 !matches!(self.logical_plan, DslPlan::Sink { .. }),
880 InvalidOperation: "cannot create a sink on top of another sink"
881 );
882
883 self.logical_plan = DslPlan::Sink {
884 input: Arc::new(self.logical_plan),
885 payload: match sink_type {
886 SinkDestination::File { target } => SinkType::File(FileSinkOptions {
887 target,
888 file_format,
889 unified_sink_args,
890 }),
891 SinkDestination::Partitioned {
892 base_path,
893 file_path_provider,
894 partition_strategy,
895 max_rows_per_file,
896 approximate_bytes_per_file,
897 } => SinkType::Partitioned(PartitionedSinkOptions {
898 base_path,
899 file_path_provider,
900 partition_strategy,
901 file_format,
902 unified_sink_args,
903 max_rows_per_file,
904 approximate_bytes_per_file,
905 }),
906 },
907 };
908 Ok(self)
909 }
910
911 pub fn filter(self, predicate: Expr) -> Self {
929 let opt_state = self.get_opt_state();
930 let lp = self.get_plan_builder().filter(predicate).build();
931 Self::from_logical_plan(lp, opt_state)
932 }
933
934 pub fn remove(self, predicate: Expr) -> Self {
952 self.filter(predicate.neq_missing(lit(true)))
953 }
954
955 pub fn select<E: AsRef<[Expr]>>(self, exprs: E) -> Self {
981 let exprs = exprs.as_ref().to_vec();
982 self.select_impl(
983 exprs,
984 ProjectionOptions {
985 run_parallel: true,
986 duplicate_check: true,
987 should_broadcast: true,
988 maintain_dataframe_height: false,
989 },
990 )
991 }
992
993 pub fn select_seq<E: AsRef<[Expr]>>(self, exprs: E) -> Self {
994 let exprs = exprs.as_ref().to_vec();
995 self.select_impl(
996 exprs,
997 ProjectionOptions {
998 run_parallel: false,
999 duplicate_check: true,
1000 should_broadcast: true,
1001 maintain_dataframe_height: false,
1002 },
1003 )
1004 }
1005
1006 fn select_impl(self, exprs: Vec<Expr>, options: ProjectionOptions) -> Self {
1007 let opt_state = self.get_opt_state();
1008 let lp = self.get_plan_builder().project(exprs, options).build();
1009 Self::from_logical_plan(lp, opt_state)
1010 }
1011
1012 pub fn group_by<E: AsRef<[IE]>, IE: Into<Expr> + Clone>(self, by: E) -> LazyGroupBy {
1033 let keys = by
1034 .as_ref()
1035 .iter()
1036 .map(|e| e.clone().into())
1037 .collect::<Vec<_>>();
1038 let opt_state = self.get_opt_state();
1039
1040 #[cfg(feature = "dynamic_group_by")]
1041 {
1042 LazyGroupBy {
1043 logical_plan: self.logical_plan,
1044 opt_state,
1045 keys,
1046 predicates: vec![],
1047 maintain_order: false,
1048 dynamic_options: None,
1049 rolling_options: None,
1050 }
1051 }
1052
1053 #[cfg(not(feature = "dynamic_group_by"))]
1054 {
1055 LazyGroupBy {
1056 logical_plan: self.logical_plan,
1057 opt_state,
1058 keys,
1059 predicates: vec![],
1060 maintain_order: false,
1061 }
1062 }
1063 }
1064
1065 #[cfg(feature = "dynamic_group_by")]
1073 pub fn rolling<E: AsRef<[Expr]>>(
1074 mut self,
1075 index_column: Expr,
1076 group_by: E,
1077 mut options: RollingGroupOptions,
1078 ) -> LazyGroupBy {
1079 if let Expr::Column(name) = index_column {
1080 options.index_column = name;
1081 } else {
1082 let output_field = index_column
1083 .to_field(&self.collect_schema().unwrap())
1084 .unwrap();
1085 return self.with_column(index_column).rolling(
1086 Expr::Column(output_field.name().clone()),
1087 group_by,
1088 options,
1089 );
1090 }
1091 let opt_state = self.get_opt_state();
1092 LazyGroupBy {
1093 logical_plan: self.logical_plan,
1094 opt_state,
1095 predicates: vec![],
1096 keys: group_by.as_ref().to_vec(),
1097 maintain_order: true,
1098 dynamic_options: None,
1099 rolling_options: Some(options),
1100 }
1101 }
1102
1103 #[cfg(feature = "dynamic_group_by")]
1119 pub fn group_by_dynamic<E: AsRef<[Expr]>>(
1120 mut self,
1121 index_column: Expr,
1122 group_by: E,
1123 mut options: DynamicGroupOptions,
1124 ) -> LazyGroupBy {
1125 if let Expr::Column(name) = index_column {
1126 options.index_column = name;
1127 } else {
1128 let output_field = index_column
1129 .to_field(&self.collect_schema().unwrap())
1130 .unwrap();
1131 return self.with_column(index_column).group_by_dynamic(
1132 Expr::Column(output_field.name().clone()),
1133 group_by,
1134 options,
1135 );
1136 }
1137 let opt_state = self.get_opt_state();
1138 LazyGroupBy {
1139 logical_plan: self.logical_plan,
1140 opt_state,
1141 predicates: vec![],
1142 keys: group_by.as_ref().to_vec(),
1143 maintain_order: true,
1144 dynamic_options: Some(options),
1145 rolling_options: None,
1146 }
1147 }
1148
1149 pub fn group_by_stable<E: AsRef<[IE]>, IE: Into<Expr> + Clone>(self, by: E) -> LazyGroupBy {
1151 let keys = by
1152 .as_ref()
1153 .iter()
1154 .map(|e| e.clone().into())
1155 .collect::<Vec<_>>();
1156 let opt_state = self.get_opt_state();
1157
1158 #[cfg(feature = "dynamic_group_by")]
1159 {
1160 LazyGroupBy {
1161 logical_plan: self.logical_plan,
1162 opt_state,
1163 keys,
1164 predicates: vec![],
1165 maintain_order: true,
1166 dynamic_options: None,
1167 rolling_options: None,
1168 }
1169 }
1170
1171 #[cfg(not(feature = "dynamic_group_by"))]
1172 {
1173 LazyGroupBy {
1174 logical_plan: self.logical_plan,
1175 opt_state,
1176 keys,
1177 predicates: vec![],
1178 maintain_order: true,
1179 }
1180 }
1181 }
1182
1183 #[cfg(feature = "semi_anti_join")]
1200 pub fn anti_join<E: Into<Expr>>(
1201 self,
1202 other: LazyFrame,
1203 left_on: E,
1204 right_on: E,
1205 ) -> PolarsResult<LazyFrame> {
1206 self.join(
1207 other,
1208 [left_on.into()],
1209 [right_on.into()],
1210 JoinArgs::new(JoinType::Anti),
1211 )
1212 }
1213
1214 #[cfg(feature = "cross_join")]
1216 pub fn cross_join(self, other: LazyFrame, suffix: Option<PlSmallStr>) -> LazyFrame {
1217 self.join(
1218 other,
1219 vec![],
1220 vec![],
1221 JoinArgs::new(JoinType::Cross).with_suffix(suffix),
1222 )
1223 .unwrap()
1224 }
1225
1226 pub fn left_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1243 self.join(
1244 other,
1245 [left_on.into()],
1246 [right_on.into()],
1247 JoinArgs::new(JoinType::Left),
1248 )
1249 .unwrap()
1250 }
1251
1252 pub fn inner_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1269 self.join(
1270 other,
1271 [left_on.into()],
1272 [right_on.into()],
1273 JoinArgs::new(JoinType::Inner),
1274 )
1275 .unwrap()
1276 }
1277
1278 pub fn full_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1295 self.join(
1296 other,
1297 [left_on.into()],
1298 [right_on.into()],
1299 JoinArgs::new(JoinType::Full),
1300 )
1301 .unwrap()
1302 }
1303
1304 #[cfg(feature = "semi_anti_join")]
1321 pub fn semi_join<E: Into<Expr>>(self, other: LazyFrame, left_on: E, right_on: E) -> LazyFrame {
1322 self.join(
1323 other,
1324 [left_on.into()],
1325 [right_on.into()],
1326 JoinArgs::new(JoinType::Semi),
1327 )
1328 .unwrap()
1329 }
1330
1331 pub fn join<E: AsRef<[Expr]>>(
1353 self,
1354 other: LazyFrame,
1355 left_on: E,
1356 right_on: E,
1357 args: JoinArgs,
1358 ) -> PolarsResult<LazyFrame> {
1359 let left_on = left_on.as_ref().to_vec();
1360 let right_on = right_on.as_ref().to_vec();
1361
1362 self._join_impl(other, left_on, right_on, args)
1363 }
1364
1365 fn _join_impl(
1366 self,
1367 other: LazyFrame,
1368 left_on: Vec<Expr>,
1369 right_on: Vec<Expr>,
1370 args: JoinArgs,
1371 ) -> PolarsResult<LazyFrame> {
1372 let JoinArgs {
1373 how,
1374 validation,
1375 suffix,
1376 slice,
1377 nulls_equal,
1378 coalesce,
1379 maintain_order,
1380 build_side,
1381 } = args;
1382
1383 if slice.is_some() {
1384 panic!("impl error: slice is not handled")
1385 }
1386
1387 let mut builder = self
1388 .join_builder()
1389 .with(other)
1390 .left_on(left_on)
1391 .right_on(right_on)
1392 .how(how)
1393 .validate(validation)
1394 .join_nulls(nulls_equal)
1395 .coalesce(coalesce)
1396 .maintain_order(maintain_order)
1397 .build_side(build_side);
1398
1399 if let Some(suffix) = suffix {
1400 builder = builder.suffix(suffix);
1401 }
1402
1403 builder.finish()
1405 }
1406
1407 pub fn join_builder(self) -> JoinBuilder {
1413 JoinBuilder::new(self)
1414 }
1415
1416 pub fn gather(self, idxs: LazyFrame, null_on_oob: bool) -> LazyFrame {
1420 let opt_state = self.get_opt_state();
1421 let lp = self
1422 .get_plan_builder()
1423 .gather(idxs.logical_plan, null_on_oob)
1424 .build();
1425 Self::from_logical_plan(lp, opt_state)
1426 }
1427
1428 pub fn with_column(self, expr: Expr) -> LazyFrame {
1446 let opt_state = self.get_opt_state();
1447 let lp = self
1448 .get_plan_builder()
1449 .with_columns(
1450 vec![expr],
1451 ProjectionOptions {
1452 run_parallel: false,
1453 duplicate_check: true,
1454 should_broadcast: true,
1455 maintain_dataframe_height: false,
1456 },
1457 )
1458 .build();
1459 Self::from_logical_plan(lp, opt_state)
1460 }
1461
1462 pub fn with_columns<E: AsRef<[Expr]>>(self, exprs: E) -> LazyFrame {
1477 let exprs = exprs.as_ref().to_vec();
1478 self.with_columns_impl(
1479 exprs,
1480 ProjectionOptions {
1481 run_parallel: true,
1482 duplicate_check: true,
1483 should_broadcast: true,
1484 maintain_dataframe_height: false,
1485 },
1486 )
1487 }
1488
1489 pub fn with_columns_seq<E: AsRef<[Expr]>>(self, exprs: E) -> LazyFrame {
1491 let exprs = exprs.as_ref().to_vec();
1492 self.with_columns_impl(
1493 exprs,
1494 ProjectionOptions {
1495 run_parallel: false,
1496 duplicate_check: true,
1497 should_broadcast: true,
1498 maintain_dataframe_height: false,
1499 },
1500 )
1501 }
1502
1503 pub fn match_to_schema(
1505 self,
1506 schema: SchemaRef,
1507 per_column: Arc<[MatchToSchemaPerColumn]>,
1508 extra_columns: ExtraColumnsPolicy,
1509 ) -> LazyFrame {
1510 let opt_state = self.get_opt_state();
1511 let lp = self
1512 .get_plan_builder()
1513 .match_to_schema(schema, per_column, extra_columns)
1514 .build();
1515 Self::from_logical_plan(lp, opt_state)
1516 }
1517
1518 pub fn pipe_with_schema(
1519 self,
1520 callback: PlanCallback<(Vec<DslPlan>, Vec<SchemaRef>), DslPlan>,
1521 ) -> Self {
1522 let opt_state = self.get_opt_state();
1523 let lp = self
1524 .get_plan_builder()
1525 .pipe_with_schema(vec![], callback)
1526 .build();
1527 Self::from_logical_plan(lp, opt_state)
1528 }
1529
1530 pub fn pipe_with_schemas(
1531 self,
1532 others: Vec<LazyFrame>,
1533 callback: PlanCallback<(Vec<DslPlan>, Vec<SchemaRef>), DslPlan>,
1534 ) -> Self {
1535 let opt_state = self.get_opt_state();
1536 let lp = self
1537 .get_plan_builder()
1538 .pipe_with_schema(
1539 others.into_iter().map(|lf| lf.logical_plan).collect(),
1540 callback,
1541 )
1542 .build();
1543 Self::from_logical_plan(lp, opt_state)
1544 }
1545
1546 fn with_columns_impl(self, exprs: Vec<Expr>, options: ProjectionOptions) -> LazyFrame {
1547 let opt_state = self.get_opt_state();
1548 let lp = self.get_plan_builder().with_columns(exprs, options).build();
1549 Self::from_logical_plan(lp, opt_state)
1550 }
1551
1552 pub fn max(self) -> Self {
1556 self.map_private(DslFunction::Stats(StatsFunction::Max))
1557 }
1558
1559 pub fn min(self) -> Self {
1563 self.map_private(DslFunction::Stats(StatsFunction::Min))
1564 }
1565
1566 pub fn sum(self) -> Self {
1576 self.map_private(DslFunction::Stats(StatsFunction::Sum))
1577 }
1578
1579 pub fn mean(self) -> Self {
1584 self.map_private(DslFunction::Stats(StatsFunction::Mean))
1585 }
1586
1587 pub fn median(self) -> Self {
1593 self.map_private(DslFunction::Stats(StatsFunction::Median))
1594 }
1595
1596 pub fn quantile(self, quantile: Expr, method: QuantileMethod) -> Self {
1598 self.map_private(DslFunction::Stats(StatsFunction::Quantile {
1599 quantile,
1600 method,
1601 }))
1602 }
1603
1604 pub fn std(self, ddof: u8) -> Self {
1617 self.map_private(DslFunction::Stats(StatsFunction::Std { ddof }))
1618 }
1619
1620 pub fn var(self, ddof: u8) -> Self {
1630 self.map_private(DslFunction::Stats(StatsFunction::Var { ddof }))
1631 }
1632
1633 pub fn explode(self, columns: Selector, options: ExplodeOptions) -> LazyFrame {
1635 self.explode_impl(columns, options, false)
1636 }
1637
1638 fn explode_impl(
1640 self,
1641 columns: Selector,
1642 options: ExplodeOptions,
1643 allow_empty: bool,
1644 ) -> LazyFrame {
1645 let opt_state = self.get_opt_state();
1646 let lp = self
1647 .get_plan_builder()
1648 .explode(columns, options, allow_empty)
1649 .build();
1650 Self::from_logical_plan(lp, opt_state)
1651 }
1652
1653 pub fn null_count(self) -> LazyFrame {
1655 self.select(vec![col(PlSmallStr::from_static("*")).null_count()])
1656 }
1657
1658 pub fn unique_stable(
1663 self,
1664 subset: Option<Selector>,
1665 keep_strategy: UniqueKeepStrategy,
1666 ) -> LazyFrame {
1667 let subset = subset.map(|s| vec![Expr::Selector(s)]);
1668 self.unique_stable_generic(subset, keep_strategy)
1669 }
1670
1671 pub fn unique_stable_generic(
1672 self,
1673 subset: Option<Vec<Expr>>,
1674 keep_strategy: UniqueKeepStrategy,
1675 ) -> LazyFrame {
1676 let opt_state = self.get_opt_state();
1677 let options = DistinctOptionsDSL {
1678 subset,
1679 maintain_order: true,
1680 keep_strategy,
1681 };
1682 let lp = self.get_plan_builder().distinct(options).build();
1683 Self::from_logical_plan(lp, opt_state)
1684 }
1685
1686 pub fn unique(self, subset: Option<Selector>, keep_strategy: UniqueKeepStrategy) -> LazyFrame {
1694 let subset = subset.map(|s| vec![Expr::Selector(s)]);
1695 self.unique_generic(subset, keep_strategy)
1696 }
1697
1698 pub fn unique_generic(
1699 self,
1700 subset: Option<Vec<Expr>>,
1701 keep_strategy: UniqueKeepStrategy,
1702 ) -> LazyFrame {
1703 let opt_state = self.get_opt_state();
1704 let options = DistinctOptionsDSL {
1705 subset,
1706 maintain_order: false,
1707 keep_strategy,
1708 };
1709 let lp = self.get_plan_builder().distinct(options).build();
1710 Self::from_logical_plan(lp, opt_state)
1711 }
1712
1713 pub fn drop_nans(self, subset: Option<Selector>) -> LazyFrame {
1718 let opt_state = self.get_opt_state();
1719 let lp = self.get_plan_builder().drop_nans(subset).build();
1720 Self::from_logical_plan(lp, opt_state)
1721 }
1722
1723 pub fn drop_nulls(self, subset: Option<Selector>) -> LazyFrame {
1728 let opt_state = self.get_opt_state();
1729 let lp = self.get_plan_builder().drop_nulls(subset).build();
1730 Self::from_logical_plan(lp, opt_state)
1731 }
1732
1733 pub fn slice(self, slice: impl Into<Slice>) -> LazyFrame {
1743 let opt_state = self.get_opt_state();
1744 let lp = self.get_plan_builder().slice(slice.into()).build();
1745 Self::from_logical_plan(lp, opt_state)
1746 }
1747
1748 pub fn clear(self) -> LazyFrame {
1750 self.slice(Slice::EMPTY)
1751 }
1752
1753 pub fn first(self) -> LazyFrame {
1755 self.slice(Slice::head(1))
1756 }
1757
1758 pub fn last(self) -> LazyFrame {
1760 self.slice(Slice::tail(1))
1761 }
1762
1763 pub fn tail(self, n: IdxSize) -> LazyFrame {
1765 self.slice(Slice::tail(n as usize))
1766 }
1767
1768 #[cfg(feature = "pivot")]
1769 #[expect(clippy::too_many_arguments)]
1770 pub fn pivot(
1771 self,
1772 on: Selector,
1773 on_columns: Arc<DataFrame>,
1774 index: Selector,
1775 values: Selector,
1776 agg: Expr,
1777 maintain_order: bool,
1778 separator: PlSmallStr,
1779 column_naming: PivotColumnNaming,
1780 ) -> LazyFrame {
1781 let opt_state = self.get_opt_state();
1782 let lp = self
1783 .get_plan_builder()
1784 .pivot(
1785 on,
1786 on_columns,
1787 index,
1788 values,
1789 agg,
1790 maintain_order,
1791 separator,
1792 column_naming,
1793 )
1794 .build();
1795 Self::from_logical_plan(lp, opt_state)
1796 }
1797
1798 #[cfg(feature = "pivot")]
1802 pub fn unpivot(self, args: UnpivotArgsDSL) -> LazyFrame {
1803 let opt_state = self.get_opt_state();
1804 let lp = self.get_plan_builder().unpivot(args).build();
1805 Self::from_logical_plan(lp, opt_state)
1806 }
1807
1808 pub fn limit(self, n: IdxSize) -> LazyFrame {
1810 self.slice(Slice::head(n as usize))
1811 }
1812
1813 pub fn map<F>(
1827 self,
1828 function: F,
1829 optimizations: AllowedOptimizations,
1830 schema: Option<Arc<dyn UdfSchema>>,
1831 name: Option<&'static str>,
1832 ) -> LazyFrame
1833 where
1834 F: 'static + Fn(DataFrame) -> PolarsResult<DataFrame> + Send + Sync,
1835 {
1836 let opt_state = self.get_opt_state();
1837 let lp = self
1838 .get_plan_builder()
1839 .map(
1840 function,
1841 optimizations,
1842 schema,
1843 PlSmallStr::from_static(name.unwrap_or("ANONYMOUS UDF")),
1844 )
1845 .build();
1846 Self::from_logical_plan(lp, opt_state)
1847 }
1848
1849 #[cfg(feature = "python")]
1850 pub fn map_python(
1851 self,
1852 function: polars_utils::python_function::PythonFunction,
1853 optimizations: AllowedOptimizations,
1854 schema: Option<SchemaRef>,
1855 validate_output: bool,
1856 ) -> LazyFrame {
1857 let opt_state = self.get_opt_state();
1858 let lp = self
1859 .get_plan_builder()
1860 .map_python(function, optimizations, schema, validate_output)
1861 .build();
1862 Self::from_logical_plan(lp, opt_state)
1863 }
1864
1865 pub(crate) fn map_private(self, function: DslFunction) -> LazyFrame {
1866 let opt_state = self.get_opt_state();
1867 let lp = self.get_plan_builder().map_private(function).build();
1868 Self::from_logical_plan(lp, opt_state)
1869 }
1870
1871 pub fn with_row_index<S>(self, name: S, offset: Option<IdxSize>) -> LazyFrame
1880 where
1881 S: Into<PlSmallStr>,
1882 {
1883 let name = name.into();
1884
1885 match &self.logical_plan {
1886 v @ DslPlan::Scan {
1887 scan_type,
1888 unified_scan_args,
1889 ..
1890 } if unified_scan_args.row_index.is_none()
1891 && !matches!(
1892 &**scan_type,
1893 FileScanDsl::Anonymous { .. } | FileScanDsl::ExpandedPaths { .. }
1894 ) =>
1895 {
1896 let DslPlan::Scan {
1897 sources,
1898 mut unified_scan_args,
1899 scan_type,
1900 cached_ir: _,
1901 } = v.clone()
1902 else {
1903 unreachable!()
1904 };
1905
1906 unified_scan_args.row_index = Some(RowIndex {
1907 name,
1908 offset: offset.unwrap_or(0),
1909 });
1910
1911 DslPlan::Scan {
1912 sources,
1913 unified_scan_args,
1914 scan_type,
1915 cached_ir: Default::default(),
1916 }
1917 .into()
1918 },
1919 _ => self.map_private(DslFunction::RowIndex { name, offset }),
1920 }
1921 }
1922
1923 pub fn count(self) -> LazyFrame {
1925 self.select(vec![col(PlSmallStr::from_static("*")).count()])
1926 }
1927
1928 #[cfg(feature = "dtype-struct")]
1931 pub fn unnest(self, cols: Selector, separator: Option<PlSmallStr>) -> Self {
1932 self.map_private(DslFunction::Unnest {
1933 columns: cols,
1934 separator,
1935 })
1936 }
1937
1938 #[cfg(feature = "merge_sorted")]
1939 pub fn merge_sorted<I, S>(
1940 self,
1941 other: LazyFrame,
1942 key: I,
1943 maintain_order: bool,
1944 ) -> PolarsResult<LazyFrame>
1945 where
1946 I: IntoIterator<Item = S>,
1947 S: Into<PlSmallStr>,
1948 {
1949 let key: Arc<[PlSmallStr]> = key.into_iter().map(Into::into).collect();
1950
1951 polars_ensure!(
1952 !key.is_empty(),
1953 ComputeError: "merge_sorted requires at least one key column"
1954 );
1955
1956 let lp = DslPlan::MergeSorted {
1957 input_left: Arc::new(self.logical_plan),
1958 input_right: Arc::new(other.logical_plan),
1959 key,
1960 maintain_order,
1961 };
1962 Ok(LazyFrame::from_logical_plan(lp, self.opt_state))
1963 }
1964
1965 pub fn hint(self, hint: HintIR) -> PolarsResult<LazyFrame> {
1966 let lp = DslPlan::MapFunction {
1967 input: Arc::new(self.logical_plan),
1968 function: DslFunction::Hint(hint),
1969 };
1970 Ok(LazyFrame::from_logical_plan(lp, self.opt_state))
1971 }
1972}
1973
1974#[derive(Clone)]
1976pub struct LazyGroupBy {
1977 pub logical_plan: DslPlan,
1978 opt_state: OptFlags,
1979 keys: Vec<Expr>,
1980 predicates: Vec<Expr>,
1981 maintain_order: bool,
1982 #[cfg(feature = "dynamic_group_by")]
1983 dynamic_options: Option<DynamicGroupOptions>,
1984 #[cfg(feature = "dynamic_group_by")]
1985 rolling_options: Option<RollingGroupOptions>,
1986}
1987
1988impl From<LazyGroupBy> for LazyFrame {
1989 fn from(lgb: LazyGroupBy) -> Self {
1990 Self {
1991 logical_plan: lgb.logical_plan,
1992 opt_state: lgb.opt_state,
1993 cached_arena: Default::default(),
1994 }
1995 }
1996}
1997
1998impl LazyGroupBy {
1999 pub fn having(mut self, predicate: Expr) -> Self {
2020 self.predicates.push(predicate);
2021 self
2022 }
2023
2024 pub fn agg<E: AsRef<[Expr]>>(self, aggs: E) -> LazyFrame {
2046 #[cfg(feature = "dynamic_group_by")]
2047 let lp = DslBuilder::from(self.logical_plan)
2048 .group_by(
2049 self.keys,
2050 self.predicates,
2051 aggs,
2052 None,
2053 self.maintain_order,
2054 self.dynamic_options,
2055 self.rolling_options,
2056 )
2057 .build();
2058
2059 #[cfg(not(feature = "dynamic_group_by"))]
2060 let lp = DslBuilder::from(self.logical_plan)
2061 .group_by(self.keys, self.predicates, aggs, None, self.maintain_order)
2062 .build();
2063 LazyFrame::from_logical_plan(lp, self.opt_state)
2064 }
2065
2066 pub fn head(self, n: Option<usize>) -> LazyFrame {
2068 let keys = self
2069 .keys
2070 .iter()
2071 .filter_map(|expr| expr_output_name(expr).ok())
2072 .collect::<Vec<_>>();
2073
2074 self.agg([all().as_expr().head(n)]).explode_impl(
2075 all() - by_name(keys.iter().cloned(), false, false),
2076 ExplodeOptions {
2077 empty_as_null: true,
2078 keep_nulls: true,
2079 },
2080 true,
2081 )
2082 }
2083
2084 pub fn tail(self, n: Option<usize>) -> LazyFrame {
2086 let keys = self
2087 .keys
2088 .iter()
2089 .filter_map(|expr| expr_output_name(expr).ok())
2090 .collect::<Vec<_>>();
2091
2092 self.agg([all().as_expr().tail(n)]).explode_impl(
2093 all() - by_name(keys.iter().cloned(), false, false),
2094 ExplodeOptions {
2095 empty_as_null: true,
2096 keep_nulls: true,
2097 },
2098 true,
2099 )
2100 }
2101
2102 pub fn apply(self, f: PlanCallback<DataFrame, DataFrame>, schema: SchemaRef) -> LazyFrame {
2107 if !self.predicates.is_empty() {
2108 panic!("not yet implemented: `apply` cannot be used with `having` predicates");
2109 }
2110
2111 #[cfg(feature = "dynamic_group_by")]
2112 let options = GroupbyOptions {
2113 dynamic: self.dynamic_options,
2114 rolling: self.rolling_options,
2115 slice: None,
2116 };
2117
2118 #[cfg(not(feature = "dynamic_group_by"))]
2119 let options = GroupbyOptions { slice: None };
2120
2121 let lp = DslPlan::GroupBy {
2122 input: Arc::new(self.logical_plan),
2123 keys: self.keys,
2124 predicates: vec![],
2125 aggs: vec![],
2126 apply: Some((f, schema)),
2127 maintain_order: self.maintain_order,
2128 options: Arc::new(options),
2129 };
2130 LazyFrame::from_logical_plan(lp, self.opt_state)
2131 }
2132}
2133
2134#[must_use]
2135pub struct JoinBuilder {
2136 lf: LazyFrame,
2137 how: JoinType,
2138 other: Option<LazyFrame>,
2139 left_on: Vec<Expr>,
2140 right_on: Vec<Expr>,
2141 allow_parallel: bool,
2142 force_parallel: bool,
2143 suffix: Option<PlSmallStr>,
2144 validation: JoinValidation,
2145 nulls_equal: bool,
2146 coalesce: JoinCoalesce,
2147 maintain_order: MaintainOrderJoin,
2148 build_side: Option<JoinBuildSide>,
2149}
2150impl JoinBuilder {
2151 pub fn new(lf: LazyFrame) -> Self {
2153 Self {
2154 lf,
2155 other: None,
2156 how: JoinType::Inner,
2157 left_on: vec![],
2158 right_on: vec![],
2159 allow_parallel: true,
2160 force_parallel: false,
2161 suffix: None,
2162 validation: Default::default(),
2163 nulls_equal: false,
2164 coalesce: Default::default(),
2165 maintain_order: Default::default(),
2166 build_side: None,
2167 }
2168 }
2169
2170 pub fn with(mut self, other: LazyFrame) -> Self {
2172 self.other = Some(other);
2173 self
2174 }
2175
2176 pub fn how(mut self, how: JoinType) -> Self {
2178 self.how = how;
2179 self
2180 }
2181
2182 pub fn validate(mut self, validation: JoinValidation) -> Self {
2183 self.validation = validation;
2184 self
2185 }
2186
2187 pub fn on<E: AsRef<[Expr]>>(mut self, on: E) -> Self {
2193 let on = on.as_ref().to_vec();
2194 self.left_on.clone_from(&on);
2195 self.right_on = on;
2196 self
2197 }
2198
2199 pub fn left_on<E: AsRef<[Expr]>>(mut self, on: E) -> Self {
2204 self.left_on = on.as_ref().to_vec();
2205 self
2206 }
2207
2208 pub fn right_on<E: AsRef<[Expr]>>(mut self, on: E) -> Self {
2213 self.right_on = on.as_ref().to_vec();
2214 self
2215 }
2216
2217 pub fn allow_parallel(mut self, allow: bool) -> Self {
2219 self.allow_parallel = allow;
2220 self
2221 }
2222
2223 pub fn force_parallel(mut self, force: bool) -> Self {
2225 self.force_parallel = force;
2226 self
2227 }
2228
2229 pub fn join_nulls(mut self, nulls_equal: bool) -> Self {
2231 self.nulls_equal = nulls_equal;
2232 self
2233 }
2234
2235 pub fn suffix<S>(mut self, suffix: S) -> Self
2238 where
2239 S: Into<PlSmallStr>,
2240 {
2241 self.suffix = Some(suffix.into());
2242 self
2243 }
2244
2245 pub fn coalesce(mut self, coalesce: JoinCoalesce) -> Self {
2247 self.coalesce = coalesce;
2248 self
2249 }
2250
2251 pub fn maintain_order(mut self, maintain_order: MaintainOrderJoin) -> Self {
2253 self.maintain_order = maintain_order;
2254 self
2255 }
2256
2257 pub fn build_side(mut self, build_side: Option<JoinBuildSide>) -> Self {
2259 self.build_side = build_side;
2260 self
2261 }
2262
2263 pub fn finish(self) -> PolarsResult<LazyFrame> {
2265 let opt_state = self.lf.opt_state;
2266 let other = self.other.expect("'with' not set in join builder");
2267
2268 let args = JoinArgs {
2269 how: self.how,
2270 validation: self.validation,
2271 suffix: self.suffix,
2272 slice: None,
2273 nulls_equal: self.nulls_equal,
2274 coalesce: self.coalesce,
2275 maintain_order: self.maintain_order,
2276 build_side: self.build_side,
2277 };
2278
2279 let lp = self
2280 .lf
2281 .get_plan_builder()
2282 .join(
2283 other.logical_plan,
2284 self.left_on,
2285 self.right_on,
2286 JoinOptions {
2287 allow_parallel: self.allow_parallel,
2288 force_parallel: self.force_parallel,
2289 args,
2290 }
2291 .into(),
2292 )?
2293 .build();
2294 Ok(LazyFrame::from_logical_plan(lp, opt_state))
2295 }
2296
2297 pub fn join_where(self, predicates: Vec<Expr>) -> LazyFrame {
2299 let opt_state = self.lf.opt_state;
2300 let other = self.other.expect("with not set");
2301
2302 fn decompose_and(predicate: Expr, expanded_predicates: &mut Vec<Expr>) {
2304 if let Expr::BinaryExpr {
2305 op: Operator::And,
2306 left,
2307 right,
2308 } = predicate
2309 {
2310 decompose_and((*left).clone(), expanded_predicates);
2311 decompose_and((*right).clone(), expanded_predicates);
2312 } else {
2313 expanded_predicates.push(predicate);
2314 }
2315 }
2316 let mut expanded_predicates = Vec::with_capacity(predicates.len() * 2);
2317 for predicate in predicates {
2318 decompose_and(predicate, &mut expanded_predicates);
2319 }
2320 let predicates: Vec<Expr> = expanded_predicates;
2321
2322 #[cfg(feature = "is_between")]
2324 let predicates: Vec<Expr> = {
2325 let mut expanded_predicates = Vec::with_capacity(predicates.len() * 2);
2326 for predicate in predicates {
2327 if let Expr::Function {
2328 function: FunctionExpr::Boolean(BooleanFunction::IsBetween { closed }),
2329 input,
2330 ..
2331 } = &predicate
2332 {
2333 if let [expr, lower, upper] = input.as_slice() {
2334 match closed {
2335 ClosedInterval::Both => {
2336 expanded_predicates.push(expr.clone().gt_eq(lower.clone()));
2337 expanded_predicates.push(expr.clone().lt_eq(upper.clone()));
2338 },
2339 ClosedInterval::Right => {
2340 expanded_predicates.push(expr.clone().gt(lower.clone()));
2341 expanded_predicates.push(expr.clone().lt_eq(upper.clone()));
2342 },
2343 ClosedInterval::Left => {
2344 expanded_predicates.push(expr.clone().gt_eq(lower.clone()));
2345 expanded_predicates.push(expr.clone().lt(upper.clone()));
2346 },
2347 ClosedInterval::None => {
2348 expanded_predicates.push(expr.clone().gt(lower.clone()));
2349 expanded_predicates.push(expr.clone().lt(upper.clone()));
2350 },
2351 }
2352 continue;
2353 }
2354 }
2355 expanded_predicates.push(predicate);
2356 }
2357 expanded_predicates
2358 };
2359
2360 let args = JoinArgs {
2361 how: self.how,
2362 validation: self.validation,
2363 suffix: self.suffix,
2364 slice: None,
2365 nulls_equal: self.nulls_equal,
2366 coalesce: self.coalesce,
2367 maintain_order: self.maintain_order,
2368 build_side: self.build_side,
2369 };
2370 let options = JoinOptions {
2371 allow_parallel: self.allow_parallel,
2372 force_parallel: self.force_parallel,
2373 args,
2374 };
2375
2376 let lp = DslPlan::Join {
2377 input_left: Arc::new(self.lf.logical_plan),
2378 input_right: Arc::new(other.logical_plan),
2379 condition: JoinCondition::NonEqui { predicates },
2380 options: Arc::from(options),
2381 };
2382
2383 LazyFrame::from_logical_plan(lp, opt_state)
2384 }
2385}
2386
2387pub const BUILD_STREAMING_EXECUTOR: Option<polars_mem_engine::StreamingExecutorBuilder> = {
2388 #[cfg(not(feature = "streaming"))]
2389 {
2390 None
2391 }
2392 #[cfg(feature = "streaming")]
2393 {
2394 Some(polars_stream::build_streaming_query_executor)
2395 }
2396};
2397
2398pub(crate) fn run_in_memory_query(
2399 node: Node,
2400 ir_arena: &mut Arena<IR>,
2401 expr_arena: &mut Arena<AExpr>,
2402 engine: Engine,
2403 observer: Option<Box<dyn QueryObserver>>,
2404) -> PolarsResult<QueryResult> {
2405 let _guard = observer
2406 .as_ref()
2407 .map(|o| o.on_query_planned(to_planned_query(node, ir_arena, expr_arena)));
2408
2409 let result = if let IR::SinkMultiple { inputs } = ir_arena.get(node) {
2410 polars_ensure!(
2411 engine != Engine::Gpu,
2412 InvalidOperation:
2413 "collect_all is not supported for the gpu engine"
2414 );
2415
2416 let physical_plan = create_multiple_physical_plans(
2417 inputs.clone().as_slice(),
2418 ir_arena,
2419 expr_arena,
2420 BUILD_STREAMING_EXECUTOR,
2421 )?;
2422 physical_plan.execute().map(QueryResult::Multiple)
2423 } else {
2424 let mut physical_plan =
2425 create_physical_plan(node, ir_arena, expr_arena, BUILD_STREAMING_EXECUTOR)?;
2426 let mut state = ExecutionState::new();
2427 physical_plan.execute(&mut state).map(QueryResult::Single)
2428 };
2429
2430 result.inspect_err(|err| {
2431 if let Some(o) = observer.as_ref() {
2432 o.on_query_failed(err);
2433 }
2434 })
2435}
2436
2437fn to_planned_query(node: Node, ir_arena: &Arena<IR>, expr_arena: &Arena<AExpr>) -> PlannedQuery {
2438 let ir = ir_plan_to_description(&[node], ir_arena, expr_arena);
2439 PlannedQuery::new(ir)
2440}
2441
2442pub struct CollectBatches {
2443 recv: Receiver<PolarsResult<DataFrame>>,
2444 runner: Option<Box<dyn FnOnce() + Send + 'static>>,
2445}
2446
2447impl CollectBatches {
2448 pub fn start(&mut self) {
2450 if let Some(runner) = self.runner.take() {
2451 runner()
2452 }
2453 }
2454}
2455
2456impl Iterator for CollectBatches {
2457 type Item = PolarsResult<DataFrame>;
2458
2459 fn next(&mut self) -> Option<Self::Item> {
2460 self.start();
2461 self.recv.recv().ok()
2462 }
2463}