From 0c5a54987b911c57864f9e839d2e5c38bc6c13ef Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 24 Jul 2026 06:05:06 +0800 Subject: [PATCH 01/18] Enable filters for range-partitioned joins --- .../src/distribution_requirements.rs | 4 +- .../physical-plan/src/joins/hash_join/exec.rs | 106 ++++++++- .../src/joins/hash_join/shared_bounds.rs | 210 ++++++++++++++---- datafusion/physical-plan/src/ordering.rs | 99 +++++++++ datafusion/physical-plan/src/topk/mod.rs | 99 +-------- 5 files changed, 371 insertions(+), 147 deletions(-) diff --git a/datafusion/physical-plan/src/distribution_requirements.rs b/datafusion/physical-plan/src/distribution_requirements.rs index 9c7a1336c06a3..80222ce986086 100644 --- a/datafusion/physical-plan/src/distribution_requirements.rs +++ b/datafusion/physical-plan/src/distribution_requirements.rs @@ -19,10 +19,10 @@ use std::sync::Arc; -use datafusion_common::{Result, internal_err}; +use datafusion_common::{Result, ScalarValue, internal_err, validate_range_split_points}; use datafusion_physical_expr::{ Distribution, EquivalenceProperties, Partitioning, PartitioningSatisfaction, - PhysicalExpr, physical_exprs_equal, + PhysicalExpr, RangePartitioning, physical_exprs_equal, }; use crate::execution_plan::{ExecutionPlan, ExecutionPlanProperties, InvariantLevel}; diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index c5a64da1ea4af..24714c3893dad 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -886,9 +886,6 @@ impl HashJoinExec { if self.mode == PartitionMode::Partitioned && !self.has_partitioned_dynamic_filter_routing() { - // TODO: support partition-routed dynamic filters for compatible - // range co-partitioned joins. - // . return false; } @@ -904,6 +901,14 @@ impl HashJoinExec { Partitioning::Hash(_, left_partition_count), Partitioning::Hash(_, right_partition_count), ) => left_partition_count == right_partition_count, + (Partitioning::Range(_), Partitioning::Range(_)) => { + let children = [self.left.as_ref(), self.right.as_ref()]; + matches!( + self.input_distribution_requirements() + .unsatisfied_co_partitioned_children(self.name(), &children), + Ok(unsatisfied) if unsatisfied.is_empty() + ) + } (left_partitioning, right_partitioning) => { left_partitioning.partition_count() == 1 && right_partitioning.partition_count() == 1 @@ -6766,8 +6771,7 @@ mod tests { } #[test] - fn test_partitioned_dynamic_filter_pushdown_rejects_range_partitioning() -> Result<()> - { + fn test_partitioned_dynamic_filter_pushdown_range_partitioning() -> Result<()> { let (left_schema, right_schema, on) = build_schema_and_on()?; let left_partitioning = Partitioning::Range(RangePartitioning::try_new( [PhysicalSortExpr { @@ -6790,7 +6794,7 @@ mod tests { left_partitioning, )?); let right = Arc::new(PartitionedTestExec::try_new( - right_schema, + Arc::clone(&right_schema), right_partitioning, )?); @@ -6801,8 +6805,35 @@ mod tests { .enable_join_dynamic_filter_pushdown = true; let join = HashJoinExec::try_new( - left, + Arc::clone(&left) as Arc, right, + on.clone(), + None, + &JoinType::Inner, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + )?; + + assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); + + let mismatched_right_partitioning = + Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr { + expr: Arc::clone(&on[0].1), + options: Default::default(), + }] + .into(), + vec![SplitPoint::new(vec![ScalarValue::Int32(Some(11))])], + )?); + let mismatched_right = Arc::new(PartitionedTestExec::try_new( + right_schema, + mismatched_right_partitioning, + )?); + let mismatched_join = HashJoinExec::try_new( + left, + mismatched_right, on, None, &JoinType::Inner, @@ -6812,6 +6843,67 @@ mod tests { false, )?; + assert!( + !mismatched_join.allow_join_dynamic_filter_pushdown(session_config.options()) + ); + + Ok(()) + } + + #[test] + fn test_partitioned_dynamic_filter_pushdown_rejects_float_zero_split() -> Result<()> { + let left_schema = Arc::new(Schema::new(vec![Field::new( + "left_key", + DataType::Float64, + false, + )])); + let right_schema = Arc::new(Schema::new(vec![Field::new( + "right_key", + DataType::Float64, + false, + )])); + let left_key = Arc::new(Column::new("left_key", 0)) as PhysicalExprRef; + let right_key = Arc::new(Column::new("right_key", 0)) as PhysicalExprRef; + let split_points = vec![SplitPoint::new(vec![ScalarValue::Float64(Some(0.0))])]; + let left = Arc::new(PartitionedTestExec::try_new( + left_schema, + Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr::new( + Arc::clone(&left_key), + Default::default(), + )] + .into(), + split_points.clone(), + )?), + )?); + let right = Arc::new(PartitionedTestExec::try_new( + right_schema, + Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr::new( + Arc::clone(&right_key), + Default::default(), + )] + .into(), + split_points, + )?), + )?); + let join = HashJoinExec::try_new( + left, + right, + vec![(left_key, right_key)], + None, + &JoinType::Inner, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + )?; + let mut session_config = SessionConfig::default(); + session_config + .options_mut() + .optimizer + .enable_join_dynamic_filter_pushdown = true; + assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options())); Ok(()) diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs index 7146e8dc2ec34..ddb7254c2eb81 100644 --- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs +++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs @@ -30,6 +30,7 @@ use crate::joins::hash_join::inlist_builder::build_struct_fields; use crate::joins::hash_join::partitioned_hash_eval::{ HashExpr, HashTableLookupExpr, SeededRandomState, }; +use crate::ordering::build_lexicographic_filter; use arrow::array::ArrayRef; use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::config::ConfigOptions; @@ -39,7 +40,10 @@ use datafusion_functions::core::r#struct as struct_func; use datafusion_physical_expr::expressions::{ BinaryExpr, CaseExpr, DynamicFilterPhysicalExpr, InListExpr, lit, }; -use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef, ScalarFunctionExpr}; +use datafusion_physical_expr::{ + PhysicalExpr, PhysicalExprRef, PhysicalSortExpr, RangePartitioning, + ScalarFunctionExpr, +}; use parking_lot::Mutex; use tokio::sync::Notify; @@ -255,6 +259,8 @@ pub(crate) struct SharedBuildAccumulator { repartition_random_state: SeededRandomState, /// Schema of the probe (right) side for evaluating filter expressions probe_schema: Arc, + /// Probe-side Range routing metadata for partitioned dynamic filters. + probe_range_partitioning: Option, } /// Strategy for filter pushdown (decided at collection time) @@ -394,6 +400,15 @@ impl SharedBuildAccumulator { ), }; + let probe_range_partitioning = if partition_mode == PartitionMode::Partitioned { + match right_child.output_partitioning() { + crate::Partitioning::Range(range) => Some(range.clone()), + _ => None, + } + } else { + None + }; + Self { inner: Mutex::new(AccumulatorState { data: mode_data, @@ -404,6 +419,7 @@ impl SharedBuildAccumulator { on_right, repartition_random_state, probe_schema: right_child.schema(), + probe_range_partitioning, } } @@ -595,19 +611,8 @@ impl SharedBuildAccumulator { }, FinalizeInput::Partitioned(partitions) => { let num_partitions = partitions.len(); - let routing_hash_expr = Arc::new(HashExpr::new( - self.on_right.clone(), - self.repartition_random_state.clone(), - "hash_repartition".to_string(), - )) as Arc; - - let modulo_expr = Arc::new(BinaryExpr::new( - routing_hash_expr, - Operator::Modulo, - lit(ScalarValue::UInt64(Some(num_partitions as u64))), - )) as Arc; - - let mut real_branches = Vec::new(); + let mut partition_filters = Vec::with_capacity(num_partitions); + let mut real_partition_ids = Vec::new(); let mut empty_partition_ids = Vec::new(); let mut has_canceled_unknown = false; @@ -617,8 +622,10 @@ impl SharedBuildAccumulator { if matches!(partition.pushdown, PushdownStrategy::Empty) => { empty_partition_ids.push(partition_id); + partition_filters.push(lit(false)); } PartitionStatus::Reported(partition) => { + real_partition_ids.push(partition_id); let membership_expr = create_membership_predicate( &self.on_right, partition.pushdown.clone(), @@ -634,13 +641,11 @@ impl SharedBuildAccumulator { bounds_expr, ) .unwrap_or_else(|| lit(true)); - real_branches.push(( - lit(ScalarValue::UInt64(Some(partition_id as u64))), - then_expr, - )); + partition_filters.push(then_expr); } PartitionStatus::CanceledUnknown => { has_canceled_unknown = true; + partition_filters.push(lit(true)); } PartitionStatus::Pending => { return datafusion_common::internal_err!( @@ -650,38 +655,99 @@ impl SharedBuildAccumulator { } } - let filter_expr = if has_canceled_unknown { - let mut when_then_branches = empty_partition_ids - .into_iter() - .map(|partition_id| { - ( - lit(ScalarValue::UInt64(Some(partition_id as u64))), - lit(false), - ) - }) - .collect::>(); - when_then_branches.extend(real_branches); - - if when_then_branches.is_empty() { - lit(true) - } else { - Arc::new(CaseExpr::try_new( - Some(modulo_expr), - when_then_branches, - Some(lit(true)), - )?) as Arc - } - } else if real_branches.is_empty() { + let filter_expr = if has_canceled_unknown + && real_partition_ids.is_empty() + && empty_partition_ids.is_empty() + { + lit(true) + } else if !has_canceled_unknown && real_partition_ids.is_empty() { lit(false) - } else if real_branches.len() == 1 + } else if !has_canceled_unknown + && real_partition_ids.len() == 1 && empty_partition_ids.len() + 1 == num_partitions { - Arc::clone(&real_branches[0].1) + Arc::clone(&partition_filters[real_partition_ids[0]]) + } else if let Some(range_partitioning) = &self.probe_range_partitioning { + // Range partitioning + assert_eq!( + partition_filters.len(), + range_partitioning.partition_count() + ); + assert_eq!(self.on_right.len(), range_partitioning.ordering().len()); + let sort_exprs = self + .on_right + .iter() + .zip(range_partitioning.ordering()) + .map(|(expr, sort_expr)| { + PhysicalSortExpr::new(Arc::clone(expr), sort_expr.options) + }) + .collect::>(); + let else_expr = partition_filters + .pop() + .expect("Range partitioning always has at least one partition"); + let mut when_then_expr = Vec::with_capacity(partition_filters.len()); + // CASE evaluates in order + // + // CASE + // WHEN key } else { + // Hash partitioning + let routing_hash_expr = Arc::new(HashExpr::new( + self.on_right.clone(), + self.repartition_random_state.clone(), + "hash_repartition".to_string(), + )) + as Arc; + let modulo_expr = Arc::new(BinaryExpr::new( + routing_hash_expr, + Operator::Modulo, + lit(ScalarValue::UInt64(Some(num_partitions as u64))), + )) as Arc; + + let mut when_then_branches = if has_canceled_unknown { + empty_partition_ids + .into_iter() + .map(|partition_id| { + ( + lit(ScalarValue::UInt64(Some(partition_id as u64))), + lit(false), + ) + }) + .collect::>() + } else { + vec![] + }; + when_then_branches.extend(real_partition_ids.into_iter().map( + |partition_id| { + ( + lit(ScalarValue::UInt64(Some(partition_id as u64))), + Arc::clone(&partition_filters[partition_id]), + ) + }, + )); + Arc::new(CaseExpr::try_new( Some(modulo_expr), - real_branches, - Some(lit(false)), + when_then_branches, + Some(lit(has_canceled_unknown)), )?) as Arc }; @@ -722,6 +788,7 @@ pub(super) fn make_partitioned_accumulator_for_test( on_right: vec![], repartition_random_state: SeededRandomState::with_seed(1), probe_schema, + probe_range_partitioning: None, } } @@ -742,7 +809,9 @@ pub(super) fn completed_partitions_for_test(acc: &SharedBuildAccumulator) -> usi mod tests { use super::*; - use arrow::array::{ArrayRef, Int32Array}; + use arrow::array::{ArrayRef, BooleanArray, Int32Array}; + use arrow::record_batch::RecordBatch; + use datafusion_common::SplitPoint; use datafusion_physical_expr::expressions::{Column, Literal}; fn test_on_right() -> Vec { @@ -778,6 +847,7 @@ mod tests { on_right, repartition_random_state: SeededRandomState::with_seed(1), probe_schema: test_probe_schema(), + probe_range_partitioning: None, } } @@ -981,6 +1051,56 @@ mod tests { ); } + #[test] + fn partitioned_range_dynamic_filter_routes_with_searched_case() -> Result<()> { + let mut acc = make_partitioned_expr_accumulator_for_test(4); + acc.probe_range_partitioning = Some(RangePartitioning::try_new( + [PhysicalSortExpr::new( + Arc::clone(&acc.on_right[0]), + Default::default(), + )] + .into(), + vec![ + SplitPoint::new(vec![ScalarValue::Int32(Some(10))]), + SplitPoint::new(vec![ScalarValue::Int32(Some(20))]), + SplitPoint::new(vec![ScalarValue::Int32(Some(30))]), + ], + )?); + + acc.build_filter(FinalizeInput::Partitioned(vec![ + reported(PushdownStrategy::Empty, no_bounds()), + PartitionStatus::CanceledUnknown, + reported(in_list(&[20, 29]), no_bounds()), + reported(in_list(&[30]), no_bounds()), + ]))?; + + let expr = current_expr(&acc); + let case = case_expr(&expr); + assert!( + case.expr().is_none(), + "Range routing must use searched CASE" + ); + assert_eq!(case.when_then_expr().len(), 3); + + let batch = RecordBatch::try_new( + test_probe_schema(), + vec![Arc::new(Int32Array::from(vec![ + 9, 10, 19, 20, 21, 29, 30, 31, + ]))], + )?; + let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; + let result = result + .as_any() + .downcast_ref::() + .expect("dynamic filter should evaluate to BooleanArray"); + assert_eq!( + result, + &BooleanArray::from(vec![false, true, true, true, false, true, true, false,]) + ); + + Ok(()) + } + // Regression guard for the build-report lifecycle fix: on `Drop`, a stream // in `BuildReportState::ReportScheduled` still calls `report_canceled_partition` // because it cannot tell whether the coordinator has already observed the diff --git a/datafusion/physical-plan/src/ordering.rs b/datafusion/physical-plan/src/ordering.rs index 8b596b9cb23eb..9c037e44bcdd4 100644 --- a/datafusion/physical-plan/src/ordering.rs +++ b/datafusion/physical-plan/src/ordering.rs @@ -15,6 +15,13 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + +use datafusion_common::{Result, ScalarValue, assert_or_internal_err}; +use datafusion_expr::Operator; +use datafusion_physical_expr::expressions::{BinaryExpr, is_not_null, is_null, lit}; +use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr}; + /// Specifies how the input to an aggregation or window operator is ordered /// relative to their `GROUP BY` or `PARTITION BY` expressions. /// @@ -52,3 +59,95 @@ pub enum InputOrderMode { /// existing ordering. Sorted, } + +/// Build the filter expression with the given thresholds. +/// This is now called outside of any locks to reduce critical section time. +pub(crate) fn build_lexicographic_filter( + sort_exprs: &[PhysicalSortExpr], + thresholds: &[ScalarValue], +) -> Result> { + assert_or_internal_err!(!sort_exprs.is_empty(), "Sort expressions must not be empty"); + assert_or_internal_err!( + sort_exprs.len() == thresholds.len(), + "Sort expressions and thresholds must have the same length" + ); + + // Create filter expressions for each threshold + let mut filters: Vec> = Vec::with_capacity(thresholds.len()); + + let mut prev_sort_expr: Option> = None; + for (sort_expr, value) in sort_exprs.iter().zip(thresholds.iter()) { + // Create the appropriate operator based on sort order + let op = if sort_expr.options.descending { + // For descending sort, we want col > threshold (exclude smaller values) + Operator::Gt + } else { + // For ascending sort, we want col < threshold (exclude larger values) + Operator::Lt + }; + + let value_null = value.is_null(); + + let comparison = Arc::new(BinaryExpr::new( + Arc::clone(&sort_expr.expr), + op, + lit(value.clone()), + )); + + let comparison_with_null = match (sort_expr.options.nulls_first, value_null) { + // For nulls first, transform to (threshold.value is not null) and (threshold.expr is null or comparison) + (true, true) => lit(false), + (true, false) => Arc::new(BinaryExpr::new( + is_null(Arc::clone(&sort_expr.expr))?, + Operator::Or, + comparison, + )), + // For nulls last, transform to (threshold.value is null and threshold.expr is not null) + // or (threshold.value is not null and comparison) + (false, true) => is_not_null(Arc::clone(&sort_expr.expr))?, + (false, false) => comparison, + }; + + let mut eq_expr = Arc::new(BinaryExpr::new( + Arc::clone(&sort_expr.expr), + Operator::Eq, + lit(value.clone()), + )); + + if value_null { + eq_expr = Arc::new(BinaryExpr::new( + is_null(Arc::clone(&sort_expr.expr))?, + Operator::Or, + eq_expr, + )); + } + + // For a query like order by a, b, the filter for column `b` is only applied if + // the condition a = threshold.value (considering null equality) is met. + // Therefore, we add equality predicates for all preceding fields to the filter logic of the current field, + // and include the current field's equality predicate in `prev_sort_expr` for use with subsequent fields. + match prev_sort_expr.take() { + None => { + prev_sort_expr = Some(eq_expr); + filters.push(comparison_with_null); + } + Some(p) => { + filters.push(Arc::new(BinaryExpr::new( + Arc::clone(&p), + Operator::And, + comparison_with_null, + ))); + + prev_sort_expr = + Some(Arc::new(BinaryExpr::new(p, Operator::And, eq_expr))); + } + } + } + + let dynamic_predicate = filters + .into_iter() + .reduce(|a, b| Arc::new(BinaryExpr::new(a, Operator::Or, b))) + .expect("sort expressions are checked non-empty"); + + Ok(dynamic_predicate) +} diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs index 1e3efff36b1d8..6d30462219c39 100644 --- a/datafusion/physical-plan/src/topk/mod.rs +++ b/datafusion/physical-plan/src/topk/mod.rs @@ -25,7 +25,7 @@ use arrow::{ }, row::{OwnedRow, RowConverter, Rows, SortField}, }; -use datafusion_expr::{ColumnarValue, Operator}; +use datafusion_expr::ColumnarValue; use std::mem::size_of; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc}; @@ -34,6 +34,7 @@ use super::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory, RecordOutput, }; +use crate::ordering::build_lexicographic_filter; use crate::spill::get_record_batch_memory_size; use crate::{SendableRecordBatchStream, stream::RecordBatchStreamAdapter}; @@ -48,7 +49,7 @@ use datafusion_execution::{ }; use datafusion_physical_expr::{ PhysicalExpr, - expressions::{BinaryExpr, DynamicFilterPhysicalExpr, is_not_null, is_null, lit}, + expressions::{DynamicFilterPhysicalExpr, lit}, }; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; use parking_lot::RwLock; @@ -563,7 +564,7 @@ impl TopK { let thresholds = boundary.threshold_values(&self.expr)?; // Build the filter expression OUTSIDE any synchronization - let predicate = Self::build_filter_expression(&self.expr, &thresholds)?; + let predicate = build_lexicographic_filter(&self.expr, &thresholds)?; let new_threshold = boundary.threshold(self.encode_topk_common_prefix_row(boundary)?); @@ -583,101 +584,13 @@ impl TopK { filter.shared_threshold = Some(new_threshold); // Update the filter expression - if let Some(pred) = predicate - && !pred.eq(&lit(true)) - { - filter.expr.update(pred)?; + if !predicate.eq(&lit(true)) { + filter.expr.update(predicate)?; } Ok(()) } - /// Build the filter expression with the given thresholds. - /// This is now called outside of any locks to reduce critical section time. - fn build_filter_expression( - sort_exprs: &[PhysicalSortExpr], - thresholds: &[ScalarValue], - ) -> Result>> { - // Create filter expressions for each threshold - let mut filters: Vec> = - Vec::with_capacity(thresholds.len()); - - let mut prev_sort_expr: Option> = None; - for (sort_expr, value) in sort_exprs.iter().zip(thresholds.iter()) { - // Create the appropriate operator based on sort order - let op = if sort_expr.options.descending { - // For descending sort, we want col > threshold (exclude smaller values) - Operator::Gt - } else { - // For ascending sort, we want col < threshold (exclude larger values) - Operator::Lt - }; - - let value_null = value.is_null(); - - let comparison = Arc::new(BinaryExpr::new( - Arc::clone(&sort_expr.expr), - op, - lit(value.clone()), - )); - - let comparison_with_null = match (sort_expr.options.nulls_first, value_null) { - // For nulls first, transform to (threshold.value is not null) and (threshold.expr is null or comparison) - (true, true) => lit(false), - (true, false) => Arc::new(BinaryExpr::new( - is_null(Arc::clone(&sort_expr.expr))?, - Operator::Or, - comparison, - )), - // For nulls last, transform to (threshold.value is null and threshold.expr is not null) - // or (threshold.value is not null and comparison) - (false, true) => is_not_null(Arc::clone(&sort_expr.expr))?, - (false, false) => comparison, - }; - - let mut eq_expr = Arc::new(BinaryExpr::new( - Arc::clone(&sort_expr.expr), - Operator::Eq, - lit(value.clone()), - )); - - if value_null { - eq_expr = Arc::new(BinaryExpr::new( - is_null(Arc::clone(&sort_expr.expr))?, - Operator::Or, - eq_expr, - )); - } - - // For a query like order by a, b, the filter for column `b` is only applied if - // the condition a = threshold.value (considering null equality) is met. - // Therefore, we add equality predicates for all preceding fields to the filter logic of the current field, - // and include the current field's equality predicate in `prev_sort_expr` for use with subsequent fields. - match prev_sort_expr.take() { - None => { - prev_sort_expr = Some(eq_expr); - filters.push(comparison_with_null); - } - Some(p) => { - filters.push(Arc::new(BinaryExpr::new( - Arc::clone(&p), - Operator::And, - comparison_with_null, - ))); - - prev_sort_expr = - Some(Arc::new(BinaryExpr::new(p, Operator::And, eq_expr))); - } - } - } - - let dynamic_predicate = filters - .into_iter() - .reduce(|a, b| Arc::new(BinaryExpr::new(a, Operator::Or, b))); - - Ok(dynamic_predicate) - } - /// If input ordering shares a common sort prefix with the TopK, /// check if the computation can be finished early. /// From a9167cbe6ad187e39d092fd55b4b48ceabfaffa7 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 24 Jul 2026 06:29:05 +0800 Subject: [PATCH 02/18] revert header change --- .../physical-plan/src/distribution_requirements.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/datafusion/physical-plan/src/distribution_requirements.rs b/datafusion/physical-plan/src/distribution_requirements.rs index dde31f94d32b5..6405b1f121ef7 100644 --- a/datafusion/physical-plan/src/distribution_requirements.rs +++ b/datafusion/physical-plan/src/distribution_requirements.rs @@ -17,13 +17,8 @@ //! Input distribution requirements for physical execution plans. -use std::sync::Arc; - -use datafusion_common::{Result, ScalarValue, internal_err, validate_range_split_points}; -use datafusion_physical_expr::{ - Distribution, EquivalenceProperties, Partitioning, PartitioningSatisfaction, - PhysicalExpr, RangePartitioning, physical_exprs_equal, -}; +use datafusion_common::{Result, internal_err}; +use datafusion_physical_expr::{Distribution, Partitioning, PartitioningSatisfaction}; use crate::execution_plan::{ExecutionPlan, ExecutionPlanProperties, InvariantLevel}; From 4206b36c63ded0121e7b958e5ed4b53b7c47f898 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 24 Jul 2026 13:57:40 +0800 Subject: [PATCH 03/18] remove float zero test --- .../physical-plan/src/joins/hash_join/exec.rs | 59 ------------------- 1 file changed, 59 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 6e0ead66ff79b..54fcba3659d0a 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -6843,65 +6843,6 @@ mod tests { Ok(()) } - #[test] - fn test_partitioned_dynamic_filter_pushdown_rejects_float_zero_split() -> Result<()> { - let left_schema = Arc::new(Schema::new(vec![Field::new( - "left_key", - DataType::Float64, - false, - )])); - let right_schema = Arc::new(Schema::new(vec![Field::new( - "right_key", - DataType::Float64, - false, - )])); - let left_key = Arc::new(Column::new("left_key", 0)) as PhysicalExprRef; - let right_key = Arc::new(Column::new("right_key", 0)) as PhysicalExprRef; - let split_points = vec![SplitPoint::new(vec![ScalarValue::Float64(Some(0.0))])]; - let left = Arc::new(PartitionedTestExec::try_new( - left_schema, - Partitioning::Range(RangePartitioning::try_new( - [PhysicalSortExpr::new( - Arc::clone(&left_key), - Default::default(), - )] - .into(), - split_points.clone(), - )?), - )?); - let right = Arc::new(PartitionedTestExec::try_new( - right_schema, - Partitioning::Range(RangePartitioning::try_new( - [PhysicalSortExpr::new( - Arc::clone(&right_key), - Default::default(), - )] - .into(), - split_points, - )?), - )?); - let join = HashJoinExec::try_new( - left, - right, - vec![(left_key, right_key)], - None, - &JoinType::Inner, - None, - PartitionMode::Partitioned, - NullEquality::NullEqualsNothing, - false, - )?; - let mut session_config = SessionConfig::default(); - session_config - .options_mut() - .optimizer - .enable_join_dynamic_filter_pushdown = true; - - assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options())); - - Ok(()) - } - #[test] fn test_with_dynamic_filter_rejects_invalid_columns() -> Result<()> { let (_, _, on) = build_schema_and_on()?; From 1a141f3e59942ef41594219e2191e7b623e34a25 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 25 Jul 2026 03:14:25 +0800 Subject: [PATCH 04/18] address review batch 1 --- .../src/joins/hash_join/shared_bounds.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs index 9f7b4ee1d9fcb..366fd2b777900 100644 --- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs +++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use crate::ExecutionPlan; use crate::ExecutionPlanProperties; +use crate::Partitioning; use crate::joins::Map; use crate::joins::PartitionMode; use crate::joins::hash_join::exec::HASH_JOIN_SEED; @@ -404,14 +405,13 @@ impl SharedBuildAccumulator { ), }; - let probe_range_partitioning = if partition_mode == PartitionMode::Partitioned { - match right_child.output_partitioning() { - crate::Partitioning::Range(range) => Some(range.clone()), + let probe_range_partitioning = + match (partition_mode, right_child.output_partitioning()) { + (PartitionMode::Partitioned, Partitioning::Range(range)) => { + Some(range.clone()) + } _ => None, - } - } else { - None - }; + }; Self { inner: Mutex::new(AccumulatorState { @@ -691,7 +691,6 @@ impl SharedBuildAccumulator { let else_expr = partition_filters .pop() .expect("Range partitioning always has at least one partition"); - let mut when_then_expr = Vec::with_capacity(partition_filters.len()); // CASE evaluates in order // // CASE @@ -700,17 +699,18 @@ impl SharedBuildAccumulator { // ... // ELSE Fn // END - for (split_point, then_expr) in range_partitioning + let when_then_expr = range_partitioning .split_points() .iter() .zip(partition_filters) - { - let when_expr = build_lexicographic_filter( - &sort_exprs, - split_point.values(), - )?; - when_then_expr.push((when_expr, then_expr)); - } + .map(|(split_point, then_expr)| { + let when_expr = build_lexicographic_filter( + &sort_exprs, + split_point.values(), + )?; + Ok((when_expr, then_expr)) + }) + .collect::>>()?; Arc::new(CaseExpr::try_new(None, when_then_expr, Some(else_expr))?) as Arc From d6b843b704c1cae8726ab69f3e555e935dbcd5e8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 31 Jul 2026 12:22:35 +0800 Subject: [PATCH 05/18] Allow dynamic filters for range-partitioned joins --- datafusion/physical-plan/src/joins/hash_join/exec.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 54fcba3659d0a..c7f646dca513f 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -879,6 +879,13 @@ impl HashJoinExec { // https://github.com/apache/datafusion/issues/20195 if config.optimizer.preserve_file_partitions > 0 && self.mode == PartitionMode::Partitioned + && !matches!( + ( + self.left.output_partitioning(), + self.right.output_partitioning() + ), + (Partitioning::Range(_), Partitioning::Range(_)) + ) { return false; } @@ -6810,6 +6817,11 @@ mod tests { )?; assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); + session_config + .options_mut() + .optimizer + .preserve_file_partitions = 1; + assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); let mismatched_right_partitioning = Partitioning::Range(RangePartitioning::try_new( From 95ad3ca135c0a57170dcd6531d4580a67ab1d5d3 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 5 Aug 2026 01:48:33 +0800 Subject: [PATCH 06/18] jay's review --- .../physical_optimizer/filter_pushdown.rs | 231 +++++++++++++++++- .../physical-plan/src/joins/hash_join/exec.rs | 17 ++ .../src/joins/hash_join/shared_bounds.rs | 96 ++++++++ 3 files changed, 343 insertions(+), 1 deletion(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 7593fe351548e..60add07eaff65 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -49,7 +49,8 @@ use datafusion_physical_expr::{ LexOrdering, PhysicalSortExpr, expressions::col, utils::conjunction, }; use datafusion_physical_expr::{ - Partitioning, ScalarFunctionExpr, aggregate::AggregateExprBuilder, + Partitioning, RangePartitioning, ScalarFunctionExpr, SplitPoint, + aggregate::AggregateExprBuilder, }; use datafusion_physical_optimizer::{ PhysicalOptimizerRule, filter_pushdown::FilterPushdown, @@ -1191,6 +1192,234 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { ); } +// Not portable to sqllogictest: this test pins `PartitionMode::Partitioned` +// by hand-wiring matching Range repartitioning on both join sides, which the +// SQL planner does not currently produce. +#[tokio::test] +async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { + use datafusion_common::JoinType; + use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; + + // Keep the fixture, execution, and assertions parallel to + // `test_hashjoin_dynamic_filter_pushdown_partitioned`. The Range layout, + // router, and preserve-file-partitions coverage are the intentional deltas. + // + // Plan under test: + // + // HashJoinExec: mode=Partitioned, on=[(a, a), (b, b)] + // ├── RepartitionExec: Range([a ASC, b ASC], [(aa, bb)]) + // │ └── DataSourceExec: build [(aa, ba), (ab, bb)] + // └── RepartitionExec: Range([a ASC, b ASC], [(aa, bb)]) + // └── DataSourceExec: probe [(aa, ba), ..., (ad, bd)], DynamicFilter + + // Create build side with limited values + let build_batches = vec![ + record_batch!( + ("a", Utf8, ["aa", "ab"]), + ("b", Utf8, ["ba", "bb"]), + ("c", Float64, [1.0, 2.0]) // Extra column not used in join + ) + .unwrap(), + ]; + let build_side_schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Utf8, false), + Field::new("c", DataType::Float64, false), + ])); + let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) + .with_support(true) + .with_batches(build_batches) + .build(); + + // Create probe side with more values + let probe_batches = vec![ + record_batch!( + ("a", Utf8, ["aa", "ab", "ac", "ad"]), + ("b", Utf8, ["ba", "bb", "bc", "bd"]), + ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join + ) + .unwrap(), + ]; + let probe_side_schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Utf8, false), + Field::new("e", DataType::Float64, false), + ])); + let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) + .with_support(true) + .with_batches(probe_batches) + .build(); + + let split_points = vec![SplitPoint::new(vec![ + ScalarValue::Utf8(Some("aa".to_string())), + ScalarValue::Utf8(Some("bb".to_string())), + ])]; + + // Build side: DataSource -> RepartitionExec (Range) + let build_range_ordering = LexOrdering::new(vec![ + PhysicalSortExpr::new( + col("a", &build_side_schema).unwrap(), + SortOptions::default(), + ), + PhysicalSortExpr::new( + col("b", &build_side_schema).unwrap(), + SortOptions::default(), + ), + ]) + .unwrap(); + let build_repartition = Arc::new( + RepartitionExec::try_new( + build_scan, + Partitioning::Range( + RangePartitioning::try_new(build_range_ordering, split_points.clone()) + .unwrap(), + ), + ) + .unwrap(), + ); + + // Probe side: DataSource -> RepartitionExec (Range) + let probe_range_ordering = LexOrdering::new(vec![ + PhysicalSortExpr::new( + col("a", &probe_side_schema).unwrap(), + SortOptions::default(), + ), + PhysicalSortExpr::new( + col("b", &probe_side_schema).unwrap(), + SortOptions::default(), + ), + ]) + .unwrap(); + let probe_repartition = Arc::new( + RepartitionExec::try_new( + Arc::clone(&probe_scan), + Partitioning::Range( + RangePartitioning::try_new(probe_range_ordering, split_points).unwrap(), + ), + ) + .unwrap(), + ); + + // Create HashJoinExec with partitioned inputs + let on = vec![ + ( + col("a", &build_side_schema).unwrap(), + col("a", &probe_side_schema).unwrap(), + ), + ( + col("b", &build_side_schema).unwrap(), + col("b", &probe_side_schema).unwrap(), + ), + ]; + let hash_join = Arc::new( + HashJoinExec::try_new( + build_repartition, + probe_repartition, + on, + None, + &JoinType::Inner, + None, + PartitionMode::Partitioned, + datafusion_common::NullEquality::NullEqualsNothing, + false, + ) + .unwrap(), + ); + + // Top-level CoalescePartitionsExec + let cp = Arc::new(CoalescePartitionsExec::new(hash_join)) as Arc; + // Add a sort for deterministic output + let plan = Arc::new(SortExec::new( + LexOrdering::new(vec![PhysicalSortExpr::new( + col("a", &probe_side_schema).unwrap(), + SortOptions::new(true, false), // descending, nulls_first + )]) + .unwrap(), + cp, + )) as Arc; + + // expect the predicate to be pushed down into the probe side DataSource + insta::assert_snapshot!( + OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true), + @r" + OptimizationTest: + input: + - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false] + - CoalescePartitionsExec + - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)] + - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true + - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true + output: + Ok: + - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false] + - CoalescePartitionsExec + - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)] + - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true + - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] + " + ); + + // Actually apply the optimization to the plan and execute to see the filter in action + let mut config = ConfigOptions::default(); + config.execution.parquet.pushdown_filters = true; + config.optimizer.enable_dynamic_filter_pushdown = true; + config.optimizer.preserve_file_partitions = 1; + let plan = FilterPushdown::new_post_optimization() + .optimize(plan, &config) + .unwrap(); + + let config = SessionConfig::from(config).with_batch_size(10); + let session_ctx = SessionContext::new_with_config(config); + session_ctx.register_object_store( + ObjectStoreUrl::parse("test://").unwrap().as_ref(), + Arc::new(InMemory::new()), + ); + let state = session_ctx.state(); + let task_ctx = state.task_ctx(); + let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx)) + .await + .unwrap(); + + // Now check what our filter looks like + insta::assert_snapshot!( + format!("{}", format_plan_for_test(&plan)), + @r" + - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false] + - CoalescePartitionsExec + - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)] + - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true + - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE WHEN a@0 IS NULL OR a@0 < aa OR a@0 = aa AND (b@1 IS NULL OR b@1 < bb) THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) ELSE a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) END ] + " + ); + + let result = format!("{}", pretty_format_batches(&batches).unwrap()); + + let probe_scan_metrics = probe_scan.metrics().unwrap(); + + // The probe side had 4 rows, but after applying the dynamic filter only 2 rows should remain. + // The number of output rows from the probe side scan should stay consistent across executions. + // Issue: https://github.com/apache/datafusion/issues/17451 + assert_eq!(probe_scan_metrics.output_rows().unwrap(), 2); + + insta::assert_snapshot!( + result, + @r" + +----+----+-----+----+----+-----+ + | a | b | c | a | b | e | + +----+----+-----+----+----+-----+ + | ab | bb | 2.0 | ab | bb | 2.0 | + | aa | ba | 1.0 | aa | ba | 1.0 | + +----+----+-----+----+----+-----+ + ", + ); +} + // Not portable to sqllogictest: this test specifically pins a // `RepartitionExec(Hash, 12)` between `HashJoinExec(CollectLeft)` and the // probe-side scan to verify the dynamic filter link survives that boundary diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index eea4009412f07..a3ec0fa18606c 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -7093,11 +7093,28 @@ mod tests { )?; assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); + + let hash_join = join + .builder() + .with_new_children(vec![ + Arc::new(PartitionedTestExec::try_new( + join.left().schema(), + Partitioning::Hash(vec![Arc::clone(&on[0].0)], 2), + )?), + Arc::new(PartitionedTestExec::try_new( + join.right().schema(), + Partitioning::Hash(vec![Arc::clone(&on[0].1)], 2), + )?), + ])? + .build()?; + assert!(hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); + session_config .options_mut() .optimizer .preserve_file_partitions = 1; assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); + assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); let mismatched_right_partitioning = Partitioning::Range(RangePartitioning::try_new( diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs index 366fd2b777900..da6cff8a3125a 100644 --- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs +++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs @@ -845,6 +845,7 @@ mod tests { use super::*; use arrow::array::{ArrayRef, BooleanArray, Int32Array}; + use arrow::compute::SortOptions; use arrow::record_batch::RecordBatch; use datafusion_common::SplitPoint; use datafusion_physical_expr::expressions::{Column, Literal}; @@ -1137,6 +1138,101 @@ mod tests { Ok(()) } + #[test] + fn partitioned_range_dynamic_filter_routes_compound_nullable_keys() -> Result<()> { + let probe_schema = Arc::new(Schema::new(vec![ + Field::new("probe_key", DataType::Int32, true), + Field::new("probe_tie", DataType::Int32, true), + ])); + let on_right: Vec = vec![ + Arc::new(Column::new("probe_key", 0)), + Arc::new(Column::new("probe_tie", 1)), + ]; + let mut acc = make_accumulator_for_test( + AccumulatedBuildData::Partitioned { + partitions: vec![PartitionStatus::Pending; 4], + completed_partitions: 0, + }, + on_right, + ); + acc.probe_schema = Arc::clone(&probe_schema); + acc.probe_range_partitioning = Some(RangePartitioning::try_new( + [ + PhysicalSortExpr::new( + Arc::clone(&acc.on_right[0]), + SortOptions::new(false, true), + ), + PhysicalSortExpr::new( + Arc::clone(&acc.on_right[1]), + SortOptions::new(false, false), + ), + ] + .into(), + vec![ + SplitPoint::new(vec![ + ScalarValue::Int32(None), + ScalarValue::Int32(Some(10)), + ]), + SplitPoint::new(vec![ScalarValue::Int32(None), ScalarValue::Int32(None)]), + SplitPoint::new(vec![ + ScalarValue::Int32(Some(10)), + ScalarValue::Int32(None), + ]), + ], + )?); + + acc.build_filter(FinalizeInput::Partitioned(vec![ + reported(PushdownStrategy::Empty, no_bounds()), + PartitionStatus::CanceledUnknown, + reported(PushdownStrategy::Empty, no_bounds()), + PartitionStatus::CanceledUnknown, + ]))?; + + let expr = current_expr(&acc); + let case = case_expr(&expr); + assert!(case.expr().is_none()); + assert_eq!(case.when_then_expr().len(), 3); + + let batch = RecordBatch::try_new( + probe_schema, + vec![ + Arc::new(Int32Array::from(vec![ + None, + None, + None, + None, + Some(9), + Some(10), + Some(10), + Some(11), + ])), + Arc::new(Int32Array::from(vec![ + Some(9), + Some(10), + Some(11), + None, + None, + Some(9), + None, + None, + ])), + ], + )?; + let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; + let result = result + .as_any() + .downcast_ref::() + .expect("dynamic filter should evaluate to BooleanArray"); + assert_eq!( + result, + &BooleanArray::from( + vec![false, true, true, false, false, false, true, true,] + ) + ); + + Ok(()) + } + // Regression guard for the build-report lifecycle fix: on `Drop`, a stream // in `BuildReportState::ReportScheduled` still calls `report_canceled_partition` // because it cannot tell whether the coordinator has already observed the From dd7fefaaab55ac1c9452e8dfa074537a6ef72530 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 5 Aug 2026 02:13:57 +0800 Subject: [PATCH 07/18] add diagram --- .../physical_optimizer/filter_pushdown.rs | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 60add07eaff65..11122e251bcca 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -1200,17 +1200,43 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { use datafusion_common::JoinType; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Keep the fixture, execution, and assertions parallel to - // `test_hashjoin_dynamic_filter_pushdown_partitioned`. The Range layout, - // router, and preserve-file-partitions coverage are the intentional deltas. + // Rough sketch of the Range-partitioned MRE we're trying to recreate. The + // test hand-wires identical Range repartitioning because SQL planning does + // not currently derive the split points: // - // Plan under test: + // EXPLAIN + // SELECT * + // FROM build + // JOIN probe + // ON build.a = probe.a AND build.b = probe.b; // - // HashJoinExec: mode=Partitioned, on=[(a, a), (b, b)] - // ├── RepartitionExec: Range([a ASC, b ASC], [(aa, bb)]) - // │ └── DataSourceExec: build [(aa, ba), (ab, bb)] - // └── RepartitionExec: Range([a ASC, b ASC], [(aa, bb)]) - // └── DataSourceExec: probe [(aa, ba), ..., (ad, bd)], DynamicFilter + // +---------------+------------------------------------------------------------+ + // | plan_type | plan | + // +---------------+------------------------------------------------------------+ + // | physical_plan | ┌───────────────────────────┐ | + // | | │ HashJoinExec │ | + // | | │ -------------------- ├──────────────┐ | + // | | │ on: (a = a), (b = b) │ │ | + // | | └─────────────┬─────────────┘ │ | + // | | ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ | + // | | │ RepartitionExec ││ RepartitionExec │ | + // | | │ -------------------- ││ -------------------- │ | + // | | │ partition_count(in->out): ││ partition_count(in->out): │ | + // | | │ 1 -> 2 ││ 1 -> 2 │ | + // | | │ ││ │ | + // | | │ partitioning_scheme: ││ partitioning_scheme: │ | + // | | │ Range([a ASC, b ASC], 2) ││ Range([a ASC, b ASC], 2) │ | + // | | │ split: (aa, bb) ││ split: (aa, bb) │ | + // | | └─────────────┬─────────────┘└─────────────┬─────────────┘ | + // | | ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ | + // | | │ DataSourceExec (build) ││ DataSourceExec (probe) │ | + // | | │ -------------------- ││ -------------------- │ | + // | | │ rows: (aa,ba), (ab,bb) ││ rows: (aa,ba) ... (ad,bd) │ | + // | | │ ││ predicate: DynamicFilter │ | + // | | │ ││ range CASE -> filter_0/1 │ | + // | | └───────────────────────────┘└───────────────────────────┘ | + // | | | + // +---------------+------------------------------------------------------------+ // Create build side with limited values let build_batches = vec![ From 10ee4c7db4045fca9511bc23fe5c2e712856c167 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 5 Aug 2026 14:36:47 +0800 Subject: [PATCH 08/18] jay's 2nd review --- datafusion/core/tests/physical_optimizer/filter_pushdown.rs | 6 +----- datafusion/physical-plan/src/joins/hash_join/exec.rs | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 11122e251bcca..abd6dcbf064c1 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -1192,17 +1192,13 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { ); } -// Not portable to sqllogictest: this test pins `PartitionMode::Partitioned` -// by hand-wiring matching Range repartitioning on both join sides, which the -// SQL planner does not currently produce. #[tokio::test] async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { use datafusion_common::JoinType; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; // Rough sketch of the Range-partitioned MRE we're trying to recreate. The - // test hand-wires identical Range repartitioning because SQL planning does - // not currently derive the split points: + // test hand-wires identical Range repartitioning: // // EXPLAIN // SELECT * diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index a3ec0fa18606c..ce336df0f3b19 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -7047,7 +7047,7 @@ mod tests { } #[test] - fn test_partitioned_dynamic_filter_pushdown_range_partitioning() -> Result<()> { + fn test_partitioned_dynamic_filter_pushdown_eligibility() -> Result<()> { let (left_schema, right_schema, on) = build_schema_and_on()?; let left_partitioning = Partitioning::Range(RangePartitioning::try_new( [PhysicalSortExpr { From 3a9d25337ac1cebf1ff388cebc2f9a24b1b72f02 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Aug 2026 05:44:44 +0800 Subject: [PATCH 09/18] dedup filter pushdown test --- .../physical_optimizer/filter_pushdown.rs | 348 +++++------------- 1 file changed, 85 insertions(+), 263 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index a4a88334e2c39..81eb74cbf1a8b 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -18,7 +18,7 @@ use std::sync::{Arc, LazyLock}; use arrow::{ - array::record_batch, + array::{RecordBatch, record_batch}, datatypes::{DataType, Field, Schema, SchemaRef}, util::pretty::pretty_format_batches, }; @@ -939,6 +939,67 @@ async fn test_topk_filter_passes_through_coalesce_partitions() { ); } +fn hashjoin_pushdown_scans() -> ( + SchemaRef, + Arc, + SchemaRef, + Arc, +) { + let build_side_schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Utf8, false), + Field::new("c", DataType::Float64, false), + ])); + let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) + .with_support(true) + .with_batches(vec![ + record_batch!( + ("a", Utf8, ["aa", "ab"]), + ("b", Utf8, ["ba", "bb"]), + ("c", Float64, [1.0, 2.0]) + ) + .unwrap(), + ]) + .build(); + + let probe_side_schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Utf8, false), + Field::new("e", DataType::Float64, false), + ])); + let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) + .with_support(true) + .with_batches(vec![ + record_batch!( + ("a", Utf8, ["aa", "ab", "ac", "ad"]), + ("b", Utf8, ["ba", "bb", "bc", "bd"]), + ("e", Float64, [1.0, 2.0, 3.0, 4.0]) + ) + .unwrap(), + ]) + .build(); + + (build_side_schema, build_scan, probe_side_schema, probe_scan) +} + +async fn optimize_and_collect_pushdown_plan( + plan: Arc, + config: ConfigOptions, +) -> (Arc, Vec) { + let plan = FilterPushdown::new_post_optimization() + .optimize(plan, &config) + .unwrap(); + let session_ctx = + SessionContext::new_with_config(SessionConfig::from(config).with_batch_size(10)); + session_ctx.register_object_store( + ObjectStoreUrl::parse("test://").unwrap().as_ref(), + Arc::new(InMemory::new()), + ); + let task_ctx = session_ctx.state().task_ctx(); + let batches = collect(Arc::clone(&plan), task_ctx).await.unwrap(); + (plan, batches) +} + // Not portable to sqllogictest: this test pins `PartitionMode::Partitioned` // by hand-wiring `RepartitionExec(Hash, 12)` on both join sides. A SQL // INNER JOIN over small parquet inputs plans as `CollectLeft`, so the @@ -988,43 +1049,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { // | | | // +---------------+------------------------------------------------------------+ - // Create build side with limited values - let build_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab"]), - ("b", Utf8, ["ba", "bb"]), - ("c", Float64, [1.0, 2.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let build_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("c", DataType::Float64, false), - ])); - let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) - .with_support(true) - .with_batches(build_batches) - .build(); - - // Create probe side with more values - let probe_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab", "ac", "ad"]), - ("b", Utf8, ["ba", "bb", "bc", "bd"]), - ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let probe_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("e", DataType::Float64, false), - ])); - let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) - .with_support(true) - .with_batches(probe_batches) - .build(); + let (build_side_schema, build_scan, probe_side_schema, probe_scan) = + hashjoin_pushdown_scans(); // Create RepartitionExec nodes for both sides with hash partitioning on join keys let partition_count = 12; @@ -1122,20 +1148,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { let mut config = ConfigOptions::default(); config.execution.parquet.pushdown_filters = true; config.optimizer.enable_dynamic_filter_pushdown = true; - let plan = FilterPushdown::new_post_optimization() - .optimize(plan, &config) - .unwrap(); - let config = SessionConfig::new().with_batch_size(10); - let session_ctx = SessionContext::new_with_config(config); - session_ctx.register_object_store( - ObjectStoreUrl::parse("test://").unwrap().as_ref(), - Arc::new(InMemory::new()), - ); - let state = session_ctx.state(); - let task_ctx = state.task_ctx(); - let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx)) - .await - .unwrap(); + let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; // Now check what our filter looks like #[cfg(not(feature = "force_hash_collisions"))] @@ -1234,43 +1247,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { // | | | // +---------------+------------------------------------------------------------+ - // Create build side with limited values - let build_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab"]), - ("b", Utf8, ["ba", "bb"]), - ("c", Float64, [1.0, 2.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let build_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("c", DataType::Float64, false), - ])); - let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) - .with_support(true) - .with_batches(build_batches) - .build(); - - // Create probe side with more values - let probe_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab", "ac", "ad"]), - ("b", Utf8, ["ba", "bb", "bc", "bd"]), - ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let probe_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("e", DataType::Float64, false), - ])); - let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) - .with_support(true) - .with_batches(probe_batches) - .build(); + let (build_side_schema, build_scan, probe_side_schema, probe_scan) = + hashjoin_pushdown_scans(); let split_points = vec![SplitPoint::new(vec![ ScalarValue::Utf8(Some("aa".to_string())), @@ -1390,21 +1368,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { config.execution.parquet.pushdown_filters = true; config.optimizer.enable_dynamic_filter_pushdown = true; config.optimizer.preserve_file_partitions = 1; - let plan = FilterPushdown::new_post_optimization() - .optimize(plan, &config) - .unwrap(); - - let config = SessionConfig::from(config).with_batch_size(10); - let session_ctx = SessionContext::new_with_config(config); - session_ctx.register_object_store( - ObjectStoreUrl::parse("test://").unwrap().as_ref(), - Arc::new(InMemory::new()), - ); - let state = session_ctx.state(); - let task_ctx = state.task_ctx(); - let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx)) - .await - .unwrap(); + let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; // Now check what our filter looks like insta::assert_snapshot!( @@ -1453,42 +1417,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() { use datafusion_common::JoinType; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let build_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab"]), - ("b", Utf8, ["ba", "bb"]), - ("c", Float64, [1.0, 2.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let build_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("c", DataType::Float64, false), - ])); - let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) - .with_support(true) - .with_batches(build_batches) - .build(); - - // Create probe side with more values - let probe_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab", "ac", "ad"]), - ("b", Utf8, ["ba", "bb", "bc", "bd"]), - ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let probe_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("e", DataType::Float64, false), - ])); - let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) - .with_support(true) - .with_batches(probe_batches) - .build(); + let (build_side_schema, build_scan, probe_side_schema, probe_scan) = + hashjoin_pushdown_scans(); // Create RepartitionExec nodes for both sides with hash partitioning on join keys let partition_count = 12; @@ -1570,20 +1500,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() { let mut config = ConfigOptions::default(); config.execution.parquet.pushdown_filters = true; config.optimizer.enable_dynamic_filter_pushdown = true; - let plan = FilterPushdown::new_post_optimization() - .optimize(plan, &config) - .unwrap(); - let config = SessionConfig::new().with_batch_size(10); - let session_ctx = SessionContext::new_with_config(config); - session_ctx.register_object_store( - ObjectStoreUrl::parse("test://").unwrap().as_ref(), - Arc::new(InMemory::new()), - ); - let state = session_ctx.state(); - let task_ctx = state.task_ctx(); - let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx)) - .await - .unwrap(); + let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; // Now check what our filter looks like insta::assert_snapshot!( @@ -2857,43 +2774,8 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() { use datafusion_common::JoinType; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Create build side with limited values - let build_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab"]), - ("b", Utf8, ["ba", "bb"]), - ("c", Float64, [1.0, 2.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let build_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("c", DataType::Float64, false), - ])); - let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) - .with_support(true) - .with_batches(build_batches) - .build(); - - // Create probe side with more values - let probe_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab", "ac", "ad"]), - ("b", Utf8, ["ba", "bb", "bc", "bd"]), - ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let probe_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("e", DataType::Float64, false), - ])); - let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) - .with_support(true) - .with_batches(probe_batches) - .build(); + let (build_side_schema, build_scan, probe_side_schema, probe_scan) = + hashjoin_pushdown_scans(); // Create RepartitionExec nodes for both sides with hash partitioning on join keys let partition_count = 12; @@ -2963,24 +2845,11 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() { )) as Arc; // Apply the optimization with config setting that forces HashTable strategy - let session_config = SessionConfig::default() - .with_batch_size(10) - .set_usize("datafusion.optimizer.hash_join_inlist_pushdown_max_size", 1) - .set_bool("datafusion.execution.parquet.pushdown_filters", true) - .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", true); - let plan = FilterPushdown::new_post_optimization() - .optimize(plan, session_config.options()) - .unwrap(); - let session_ctx = SessionContext::new_with_config(session_config); - session_ctx.register_object_store( - ObjectStoreUrl::parse("test://").unwrap().as_ref(), - Arc::new(InMemory::new()), - ); - let state = session_ctx.state(); - let task_ctx = state.task_ctx(); - let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx)) - .await - .unwrap(); + let mut config = ConfigOptions::default(); + config.optimizer.hash_join_inlist_pushdown_max_size = 1; + config.execution.parquet.pushdown_filters = true; + config.optimizer.enable_dynamic_filter_pushdown = true; + let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; // Verify that hash_lookup is used instead of IN (SET) let plan_str = format_plan_for_test(&plan).to_string(); @@ -3023,42 +2892,8 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() { use datafusion_common::JoinType; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let build_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab"]), - ("b", Utf8, ["ba", "bb"]), - ("c", Float64, [1.0, 2.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let build_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("c", DataType::Float64, false), - ])); - let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema)) - .with_support(true) - .with_batches(build_batches) - .build(); - - // Create probe side with more values - let probe_batches = vec![ - record_batch!( - ("a", Utf8, ["aa", "ab", "ac", "ad"]), - ("b", Utf8, ["ba", "bb", "bc", "bd"]), - ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join - ) - .unwrap(), - ]; - let probe_side_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Utf8, false), - Field::new("e", DataType::Float64, false), - ])); - let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema)) - .with_support(true) - .with_batches(probe_batches) - .build(); + let (build_side_schema, build_scan, probe_side_schema, probe_scan) = + hashjoin_pushdown_scans(); // Create RepartitionExec nodes for both sides with hash partitioning on join keys let partition_count = 12; @@ -3114,24 +2949,11 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() { )) as Arc; // Apply the optimization with config setting that forces HashTable strategy - let session_config = SessionConfig::default() - .with_batch_size(10) - .set_usize("datafusion.optimizer.hash_join_inlist_pushdown_max_size", 1) - .set_bool("datafusion.execution.parquet.pushdown_filters", true) - .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", true); - let plan = FilterPushdown::new_post_optimization() - .optimize(plan, session_config.options()) - .unwrap(); - let session_ctx = SessionContext::new_with_config(session_config); - session_ctx.register_object_store( - ObjectStoreUrl::parse("test://").unwrap().as_ref(), - Arc::new(InMemory::new()), - ); - let state = session_ctx.state(); - let task_ctx = state.task_ctx(); - let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx)) - .await - .unwrap(); + let mut config = ConfigOptions::default(); + config.optimizer.hash_join_inlist_pushdown_max_size = 1; + config.execution.parquet.pushdown_filters = true; + config.optimizer.enable_dynamic_filter_pushdown = true; + let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; // Verify that hash_lookup is used instead of IN (SET) let plan_str = format_plan_for_test(&plan).to_string(); From 6dae145e1fea942d1f0eb3f6f6bdb240603ba3a9 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Aug 2026 05:46:28 +0800 Subject: [PATCH 10/18] review for exec.rs --- .../physical-plan/src/joins/hash_join/exec.rs | 115 ++++++++++-------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 62a345c80806e..fa304ca6c4e04 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -883,20 +883,23 @@ impl HashJoinExec { return false; } - // `preserve_file_partitions` can report Hash partitioning for Hive-style - // file groups, but those partitions are not actually hash-distributed. - // Partitioned dynamic filters rely on hash routing, so disable them in - // this mode to avoid incorrect results. Follow-up work: enable dynamic - // filtering for preserve_file_partitioned scans (issue #20195). + // `preserve_file_partitions` can report Hive-style file groups as Hash + // partitioned even though their partition indexes do not follow the + // hash router used by partitioned dynamic filters. Reject Hash inputs + // because the metadata cannot distinguish those scans from a real hash + // repartition. Compatible Range inputs remain safe because matching + // ordering and split points align each build filter with its probe + // partition. Other unsupported layouts are rejected. + // Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195). // https://github.com/apache/datafusion/issues/20195 if config.optimizer.preserve_file_partitions > 0 && self.mode == PartitionMode::Partitioned - && !matches!( + && matches!( ( self.left.output_partitioning(), self.right.output_partitioning() ), - (Partitioning::Range(_), Partitioning::Range(_)) + (Partitioning::Hash(_, _), Partitioning::Hash(_, _)) ) { return false; @@ -7040,8 +7043,10 @@ mod tests { Ok(()) } - #[test] - fn test_partitioned_dynamic_filter_pushdown_eligibility() -> Result<()> { + fn range_partitioned_dynamic_filter_test_join( + left_split: i32, + right_split: i32, + ) -> Result<(HashJoinExec, JoinOn)> { let (left_schema, right_schema, on) = build_schema_and_on()?; let left_partitioning = Partitioning::Range(RangePartitioning::try_new( [PhysicalSortExpr { @@ -7049,7 +7054,7 @@ mod tests { options: Default::default(), }] .into(), - vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])], + vec![SplitPoint::new(vec![ScalarValue::Int32(Some(left_split))])], )?); let right_partitioning = Partitioning::Range(RangePartitioning::try_new( [PhysicalSortExpr { @@ -7057,25 +7062,19 @@ mod tests { options: Default::default(), }] .into(), - vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])], + vec![SplitPoint::new(vec![ScalarValue::Int32(Some(right_split))])], )?); let left = Arc::new(PartitionedTestExec::try_new( left_schema, left_partitioning, )?); let right = Arc::new(PartitionedTestExec::try_new( - Arc::clone(&right_schema), + right_schema, right_partitioning, )?); - let mut session_config = SessionConfig::default(); - session_config - .options_mut() - .optimizer - .enable_join_dynamic_filter_pushdown = true; - let join = HashJoinExec::try_new( - Arc::clone(&left) as Arc, + left, right, on.clone(), None, @@ -7085,11 +7084,14 @@ mod tests { NullEquality::NullEqualsNothing, false, )?; + Ok((join, on)) + } - assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); - - let hash_join = join - .builder() + fn with_hash_partitioned_children( + join: &HashJoinExec, + on: &JoinOn, + ) -> Result { + join.builder() .with_new_children(vec![ Arc::new(PartitionedTestExec::try_new( join.left().schema(), @@ -7100,45 +7102,56 @@ mod tests { Partitioning::Hash(vec![Arc::clone(&on[0].1)], 2), )?), ])? - .build()?; + .build() + } + + #[test] + fn test_partitioned_dynamic_filter_pushdown_allows_supported_partitioning() + -> Result<()> { + let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?; + let hash_join = with_hash_partitioned_children(&range_join, &on)?; + let mut session_config = SessionConfig::default(); + session_config + .options_mut() + .optimizer + .enable_join_dynamic_filter_pushdown = true; + + assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options())); assert!(hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); session_config .options_mut() .optimizer .preserve_file_partitions = 1; - assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); - assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); + assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options())); - let mismatched_right_partitioning = - Partitioning::Range(RangePartitioning::try_new( - [PhysicalSortExpr { - expr: Arc::clone(&on[0].1), - options: Default::default(), - }] - .into(), - vec![SplitPoint::new(vec![ScalarValue::Int32(Some(11))])], - )?); - let mismatched_right = Arc::new(PartitionedTestExec::try_new( - right_schema, - mismatched_right_partitioning, - )?); - let mismatched_join = HashJoinExec::try_new( - left, - mismatched_right, - on, - None, - &JoinType::Inner, - None, - PartitionMode::Partitioned, - NullEquality::NullEqualsNothing, - false, - )?; + Ok(()) + } + + #[test] + fn test_partitioned_dynamic_filter_pushdown_rejects_unsupported_partitioning() + -> Result<()> { + let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?; + let hash_join = with_hash_partitioned_children(&range_join, &on)?; + let (mismatched_range_join, _) = + range_partitioned_dynamic_filter_test_join(10, 11)?; + let mut session_config = SessionConfig::default(); + session_config + .options_mut() + .optimizer + .enable_join_dynamic_filter_pushdown = true; assert!( - !mismatched_join.allow_join_dynamic_filter_pushdown(session_config.options()) + !mismatched_range_join + .allow_join_dynamic_filter_pushdown(session_config.options()) ); + session_config + .options_mut() + .optimizer + .preserve_file_partitions = 1; + assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); + Ok(()) } From 7f848311960a757bffb2405a26691428ef659215 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Aug 2026 06:46:49 +0800 Subject: [PATCH 11/18] replace csv with pq and add slt --- .../src/test_context/range_partitioning.rs | 140 ++++----- .../test_files/range_partitioning.slt | 270 ++++++++++++------ 2 files changed, 255 insertions(+), 155 deletions(-) diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs index 4141e000145a8..3cde3939f0b7c 100644 --- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs +++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::fs::{create_dir_all, remove_dir_all, write}; +use std::fs::{File, create_dir_all, remove_dir_all}; use std::path::Path; use std::sync::Arc; @@ -25,11 +25,12 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion::catalog::streaming::StreamingTable; use datafusion::common::{ScalarValue, SplitPoint}; -use datafusion::datasource::file_format::csv::CsvFormat; +use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, }; use datafusion::logical_expr::{Partitioning, RangePartitioning, col}; +use datafusion::parquet::arrow::ArrowWriter; use datafusion::physical_expr::{ Partitioning as PhysicalPartitioning, PhysicalSortExpr, RangePartitioning as PhysicalRangePartitioning, expressions::col as physical_col, @@ -45,6 +46,30 @@ use datafusion::prelude::SessionContext; /// Registers a simple range-partitioned listing table for testing before /// declaring such tables is supported via SQL. pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { + const RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [ + &[(1, 1, 10), (5, 2, 50)], + &[(10, 1, 100), (15, 2, 150)], + &[(20, 1, 200), (25, 2, 250)], + &[(30, 1, 300), (35, 2, 350)], + ]; + const SHIFTED_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [ + &[(1, 1, 10), (5, 2, 50), (10, 1, 100)], + &[(15, 2, 150)], + &[(20, 1, 200), (25, 2, 250)], + &[(30, 1, 300), (35, 2, 350)], + ]; + const NARROW_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 3] = [ + &[(1, 1, 10), (5, 2, 50)], + &[(10, 1, 100), (15, 2, 150)], + &[(20, 1, 200), (25, 2, 250), (30, 1, 300), (35, 2, 350)], + ]; + const SPARSE_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [ + &[(5, 2, 50), (8, 3, 80)], + &[(10, 1, 100)], + &[(20, 1, 200)], + &[(30, 1, 300), (40, 4, 400)], + ]; + let schema = Arc::new(Schema::new(vec![ Field::new("range_key", DataType::Int32, false), Field::new("non_range_key", DataType::Int32, false), @@ -65,18 +90,13 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { let range_table_dir = Path::new(env!("CARGO_MANIFEST_DIR")) .join("test_files/scratch_range_partitioning/range_partitioned"); - register_csv_listing_table( + register_parquet_listing_table( ctx, "range_partitioned", &range_table_dir, Arc::clone(&schema), - [ - "1,1,10\n5,2,50\n", - "10,1,100\n15,2,150\n", - "20,1,200\n25,2,250\n", - "30,1,300\n35,2,350\n", - ], - Some(output_partitioning), + RANGE_PARTITIONS, + output_partitioning, ); register_unbounded_range_stream_table( @@ -84,24 +104,14 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { "unbounded_range_like", Arc::clone(&schema), [10, 20, 30], - [ - vec![(1, 1, 10), (5, 2, 50)], - vec![(10, 1, 100), (15, 2, 150)], - vec![(20, 1, 200), (25, 2, 250)], - vec![(30, 1, 300), (35, 2, 350)], - ], + RANGE_PARTITIONS.map(|rows| rows.to_vec()), ); register_unbounded_range_stream_table( ctx, "unbounded_range_like_shifted", Arc::clone(&schema), [15, 20, 30], - [ - vec![(1, 1, 10), (5, 2, 50), (10, 1, 100)], - vec![(15, 2, 150)], - vec![(20, 1, 200), (25, 2, 250)], - vec![(30, 1, 300), (35, 2, 350)], - ], + SHIFTED_RANGE_PARTITIONS.map(|rows| rows.to_vec()), ); let shifted_output_partitioning = Partitioning::Range( @@ -116,19 +126,14 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { .expect("range partitioning should be valid"), ); - register_csv_listing_table( + register_parquet_listing_table( ctx, "range_partitioned_shifted", Path::new(env!("CARGO_MANIFEST_DIR")) .join("test_files/scratch_range_partitioning/range_partitioned_shifted"), Arc::clone(&schema), - [ - "1,1,10\n5,2,50\n10,1,100\n", - "15,2,150\n", - "20,1,200\n25,2,250\n", - "30,1,300\n35,2,350\n", - ], - Some(shifted_output_partitioning), + SHIFTED_RANGE_PARTITIONS, + shifted_output_partitioning, ); // Same rows as `range_partitioned` but split into only three range @@ -145,18 +150,14 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { .expect("range partitioning should be valid"), ); - register_csv_listing_table( + register_parquet_listing_table( ctx, "range_partitioned_narrow", Path::new(env!("CARGO_MANIFEST_DIR")) .join("test_files/scratch_range_partitioning/range_partitioned_narrow"), Arc::clone(&schema), - [ - "1,1,10\n5,2,50\n", - "10,1,100\n15,2,150\n", - "20,1,200\n25,2,250\n30,1,300\n35,2,350\n", - ], - Some(narrow_output_partitioning), + NARROW_RANGE_PARTITIONS, + narrow_output_partitioning, ); let sparse_output_partitioning = Partitioning::Range( @@ -171,29 +172,24 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { .expect("range partitioning should be valid"), ); - register_csv_listing_table( + register_parquet_listing_table( ctx, "range_partitioned_sparse", Path::new(env!("CARGO_MANIFEST_DIR")) .join("test_files/scratch_range_partitioning/range_partitioned_sparse"), - schema, - [ - "5,2,50\n8,3,80\n", - "10,1,100\n", - "20,1,200\n", - "30,1,300\n40,4,400\n", - ], - Some(sparse_output_partitioning), + Arc::clone(&schema), + SPARSE_RANGE_PARTITIONS, + sparse_output_partitioning, ); } -fn register_csv_listing_table( +fn register_parquet_listing_table( ctx: &SessionContext, name: &str, table_dir: impl AsRef, - schema: Arc, - partitions: impl IntoIterator, - output_partitioning: Option, + schema: SchemaRef, + partitions: impl IntoIterator, + output_partitioning: Partitioning, ) { let table_dir = table_dir.as_ref(); if table_dir.exists() { @@ -201,8 +197,17 @@ fn register_csv_listing_table( } create_dir_all(table_dir).expect("test table dir should be created"); for (idx, rows) in partitions.into_iter().enumerate() { - write(table_dir.join(format!("part-{idx}.csv")), rows) - .expect("test table csv partition should be written"); + let batch = range_batch(Arc::clone(&schema), rows); + let file = File::create(table_dir.join(format!("part-{idx}.parquet"))) + .expect("test table parquet partition should be created"); + let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), None) + .expect("test table parquet writer should be created"); + writer + .write(&batch) + .expect("test table parquet partition should be written"); + writer + .close() + .expect("test table parquet writer should close"); } let table_path = format!( @@ -213,9 +218,8 @@ fn register_csv_listing_table( ); let table_url = ListingTableUrl::parse(&table_path).expect("test table url should parse"); - let options = - ListingOptions::new(Arc::new(CsvFormat::default().with_has_header(false))) - .with_output_partitioning(output_partitioning); + let options = ListingOptions::new(Arc::new(ParquetFormat::default())) + .with_output_partitioning(Some(output_partitioning)); let config = ListingTableConfig::new(table_url) .with_listing_options(options) .with_schema(schema); @@ -269,20 +273,22 @@ fn range_stream_partition( schema: SchemaRef, rows: &[(i32, i32, i32)], ) -> Arc { - let range_key: Vec = rows.iter().map(|(range_key, _, _)| *range_key).collect(); - let non_range_key: Vec = rows - .iter() - .map(|(_, non_range_key, _)| *non_range_key) - .collect(); - let value: Vec = rows.iter().map(|(_, _, value)| *value).collect(); - let batch = RecordBatch::try_new( + Arc::new(TestPartitionStream::new_with_batches(vec![range_batch( + schema, rows, + )])) +} + +fn range_batch(schema: SchemaRef, rows: &[(i32, i32, i32)]) -> RecordBatch { + RecordBatch::try_new( schema, vec![ - Arc::new(Int32Array::from(range_key)) as ArrayRef, - Arc::new(Int32Array::from(non_range_key)) as ArrayRef, - Arc::new(Int32Array::from(value)) as ArrayRef, + Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.0))) + as ArrayRef, + Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.1))) + as ArrayRef, + Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.2))) + as ArrayRef, ], ) - .expect("range stream batch should be valid"); - Arc::new(TestPartitionStream::new_with_batches(vec![batch])) + .expect("range batch should be valid") } diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index 1e0a1582eac65..6f209af412142 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -16,7 +16,7 @@ # under the License. # The sqllogictest harness registers range_partitioned(range_key, non_range_key, value) -# as a CSV ListingTable with four declared range-partitioned file groups: +# as a Parquet ListingTable with four declared range-partitioned file groups: # # partition 0: range_key in [..., 10), rows (1, 1, 10), (5, 2, 50) # partition 1: range_key in [10, 20), rows (10, 1, 100), (15, 2, 150) @@ -26,6 +26,21 @@ statement ok set datafusion.explain.physical_plan_only = true; +statement ok +set datafusion.execution.collect_statistics = false; + +statement ok +set datafusion.optimizer.enable_join_dynamic_filter_pushdown = false; + +statement ok +set datafusion.optimizer.enable_topk_dynamic_filter_pushdown = false; + +statement ok +set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = false; + +statement ok +set datafusion.execution.parquet.pushdown_filters = false; + ########## # TEST 1: Aggregate on Range Partition Column # With subset threshold met and preserve-file disabled, Range([range_key]) @@ -43,7 +58,7 @@ EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key; ---- physical_plan 01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key ORDER BY range_key; @@ -77,7 +92,7 @@ physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[non_range_key@0 as non_range_key], aggr=[sum(range_partitioned.value)] 02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[non_range_key@0 as non_range_key], aggr=[sum(range_partitioned.value)] -04)------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet query II SELECT non_range_key, SUM(value) FROM range_partitioned GROUP BY non_range_key ORDER BY non_range_key; @@ -103,7 +118,7 @@ EXPLAIN SELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP ---- physical_plan 01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)] -02)--DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP BY range_key, non_range_key ORDER BY range_key, non_range_key; @@ -139,7 +154,7 @@ EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key; ---- physical_plan 01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet ########## @@ -165,7 +180,7 @@ physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)] 02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)] -04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet ########## @@ -191,7 +206,7 @@ physical_plan 02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=5 03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] 04)------RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet statement ok set datafusion.execution.target_partitions = 4; @@ -220,7 +235,7 @@ EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key; ---- physical_plan 01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet statement ok set datafusion.execution.target_partitions = 4; @@ -252,7 +267,7 @@ physical_plan 02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=5 03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] 04)------RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet statement ok set datafusion.execution.target_partitions = 4; @@ -284,8 +299,8 @@ JOIN range_partitioned r ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -318,9 +333,9 @@ JOIN range_partitioned_shifted r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -351,9 +366,9 @@ JOIN range_partitioned r ON l.non_range_key = r.non_range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(non_range_key@0, non_range_key@0)], projection=[non_range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet query III SELECT l.non_range_key, l.value, r.value @@ -408,9 +423,9 @@ ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 03)--FilterExec: value@1 <= 150 -04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -436,9 +451,9 @@ ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(range_key@0, range_key@0)] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 03)--FilterExec: value@1 <= 150, projection=[range_key@0] -04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT l.range_key, l.value @@ -460,9 +475,9 @@ ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(range_key@0, range_key@0)] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 03)--FilterExec: value@1 <= 150, projection=[range_key@0] -04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT l.range_key, l.value @@ -493,10 +508,10 @@ ON l.range_key = r.range_key AND l.non_range_key = r.non_range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5] 02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 05)----FilterExec: value@2 <= 150 -06)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +06)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet # Range([range_key]) does not satisfy a join keyed on non_range_key. query TT @@ -507,9 +522,9 @@ LEFT JOIN range_partitioned r ON l.non_range_key = r.non_range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(non_range_key@1, non_range_key@0)], projection=[range_key@0, non_range_key@1, value@2, value@4] 02)--RepartitionExec: partitioning=Hash([non_range_key@1], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet ########## # TEST 14: Left-Side Range Hash Joins With Incompatible Range Layouts @@ -526,9 +541,9 @@ LEFT JOIN range_partitioned_shifted r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -555,9 +570,9 @@ LEFT JOIN range_partitioned_narrow r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=3 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=parquet ########## # TEST 15: LeftMark Subqueries Over Range Hash Joins @@ -574,9 +589,9 @@ WHERE l.non_range_key = 2 OR l.range_key IN ( physical_plan 01)FilterExec: non_range_key@1 = 2 OR mark@3, projection=[range_key@0, value@2] 02)--HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(range_key@0, range_key@0)] -03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)----FilterExec: value@1 <= 150, projection=[range_key@0] -05)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT l.range_key, l.value @@ -625,9 +640,9 @@ JOIN range_partitioned r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -660,8 +675,8 @@ JOIN range_partitioned r ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -699,9 +714,9 @@ JOIN range_partitioned s ON r.range_key = s.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@2, range_key@0)], projection=[range_key@0, value@1, value@3, value@5] 02)--HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)] -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -05)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +05)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query IIII SELECT l.range_key, l.value, r.value, s.value @@ -744,10 +759,10 @@ physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, l_sum@1, r_sum@3] 02)--ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value)@1 as l_sum] 03)----AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] -04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 05)--ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value)@1 as r_sum] 06)----AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)] -07)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +07)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III WITH @@ -789,8 +804,8 @@ GROUP BY l.range_key; physical_plan 01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(l.value + r.value)] 02)--HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT l.range_key, SUM(l.value + r.value) @@ -823,8 +838,8 @@ RIGHT JOIN range_partitioned r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0)], projection=[value@1, range_key@2, value@3] 02)--FilterExec: value@1 <= 150 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[] +04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.value, r.range_key, r.value @@ -855,8 +870,8 @@ RIGHT SEMI JOIN range_partitioned r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(range_key@0, range_key@0)] 02)--FilterExec: value@1 <= 150, projection=[range_key@0] -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[] +04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT r.range_key, r.value @@ -883,8 +898,8 @@ RIGHT ANTI JOIN range_partitioned r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=RightAnti, on=[(range_key@0, range_key@0)] 02)--FilterExec: value@1 <= 150, projection=[range_key@0] -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[] +04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT r.range_key, r.value @@ -913,9 +928,9 @@ physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0)], projection=[value@1, range_key@2, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 03)----FilterExec: value@1 <= 150 -04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[] 05)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -06)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +06)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query III SELECT l.value, r.range_key, r.value @@ -949,9 +964,9 @@ RIGHT JOIN range_partitioned r ON l.range_key = r.range_key AND l.non_range_key physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5] 02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query IIII SELECT l.range_key, l.non_range_key, l.value, r.value @@ -987,9 +1002,9 @@ RIGHT JOIN range_partitioned_narrow r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0)], projection=[value@1, range_key@2, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=3 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=parquet query III SELECT l.value, r.range_key, r.value @@ -1022,9 +1037,9 @@ physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(non_range_key@0, non_range_key@1)], projection=[value@1, range_key@2, value@4] 02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 03)----FilterExec: range_key@0 < 10, projection=[non_range_key@1, value@2] -04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 < 10, pruning_predicate=range_key_null_count@1 != row_count@2 AND range_key_min@0 < 10, required_guarantees=[] 05)--RepartitionExec: partitioning=Hash([non_range_key@1], 4), input_partitions=4 -06)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +06)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.value, r.range_key, r.value @@ -1056,9 +1071,9 @@ WHERE r.non_range_key = 2 OR r.range_key IN ( physical_plan 01)FilterExec: non_range_key@1 = 2 OR mark@3, projection=[range_key@0, value@2] 02)--HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(range_key@0, range_key@0)] -03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)----FilterExec: value@1 <= 150, projection=[range_key@0] -05)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet # Matched rows have mark=true and are returned; unmatched rows have # mark=false and are only returned when non_range_key = 2. @@ -1111,9 +1126,9 @@ physical_plan 01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value] 02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)] 03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true] -04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, sort_order_for_reorder=[range_key@0 ASC] 05)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true] -06)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +06)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, sort_order_for_reorder=[range_key@0 ASC] query III SELECT l.range_key, l.value, r.value @@ -1146,10 +1161,10 @@ physical_plan 02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)] 03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true] 04)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 06)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true] 07)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -08)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +08)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -1249,8 +1264,8 @@ FULL JOIN range_partitioned r ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -1281,9 +1296,9 @@ FULL JOIN range_partitioned_shifted r ON l.range_key = r.range_key; physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet 04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query III SELECT l.range_key, l.value, r.value @@ -1316,8 +1331,8 @@ FULL JOIN range_partitioned_sparse r ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)], projection=[range_key@0, range_key@2, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query IIII SELECT l.range_key, r.range_key, l.value, r.value @@ -1358,8 +1373,8 @@ SELECT range_key, value FROM range_partitioned; ---- physical_plan 01)InterleaveExec -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, value FROM range_partitioned @@ -1407,7 +1422,7 @@ physical_plan 01)ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] 02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortExec: expr=[range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true] -04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value) FROM range_partitioned ORDER BY range_key; @@ -1436,7 +1451,7 @@ physical_plan 01)ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING] 02)--WindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] 03)----SortExec: expr=[range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true] -04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM range_partitioned ORDER BY range_key; @@ -1465,7 +1480,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortExec: expr=[non_range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true] 04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet query III SELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER BY value) FROM range_partitioned ORDER BY non_range_key, value; @@ -1495,7 +1510,7 @@ physical_plan 02)--WindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] 03)----SortExec: expr=[non_range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true] 04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet query III SELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER BY value ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM range_partitioned ORDER BY non_range_key, value; @@ -1527,7 +1542,7 @@ physical_plan 01)ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] 02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortExec: expr=[range_key@0 ASC NULLS LAST, non_range_key@1 ASC NULLS LAST, value@2 ASC NULLS LAST], preserve_partitioning=[true] -04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) OVER (PARTITION BY range_key, non_range_key ORDER BY value) FROM range_partitioned ORDER BY range_key; @@ -1560,7 +1575,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortExec: expr=[range_key@0 ASC NULLS LAST, non_range_key@1 ASC NULLS LAST, value@2 ASC NULLS LAST], preserve_partitioning=[true] 04)------RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) OVER (PARTITION BY range_key, non_range_key ORDER BY value) FROM range_partitioned ORDER BY range_key; @@ -1595,7 +1610,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortPreservingMergeExec: [value@1 ASC NULLS LAST] 04)------SortExec: expr=[value@1 ASC NULLS LAST], preserve_partitioning=[true] -05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) OVER (ORDER BY value) FROM range_partitioned ORDER BY range_key; @@ -1635,7 +1650,7 @@ physical_plan 01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rn] 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0], order=[value@1 DESC] -04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query III SELECT * FROM ( @@ -1677,7 +1692,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[non_range_key@0], order=[value@1 DESC] 04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet query III SELECT * FROM ( @@ -1712,7 +1727,7 @@ physical_plan 01)ProjectionExec: expr=[range_key@0 as range_key, non_range_key@1 as non_range_key, value@2 as value, row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn] 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0, non_range_key@1], order=[value@2 DESC] -04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query IIII SELECT * FROM ( @@ -1758,7 +1773,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0, non_range_key@1], order=[value@2 DESC] 04)------RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 -05)--------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)--------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet statement ok reset datafusion.optimizer.subset_repartition_threshold; @@ -1788,9 +1803,9 @@ SELECT range_key, value FROM range_partitioned_shifted; ---- physical_plan 01)UnionExec -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +04)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query II SELECT range_key, value FROM range_partitioned @@ -1841,8 +1856,8 @@ SELECT range_key, value FROM range_partitioned_shifted; ---- physical_plan 01)UnionExec -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquet query II SELECT range_key, value FROM range_partitioned @@ -1884,8 +1899,8 @@ EXPLAIN SELECT range_key, SUM(value) FROM ( physical_plan 01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(value)] 02)--InterleaveExec -03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false -04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet query II SELECT range_key, SUM(value) FROM ( @@ -1903,5 +1918,84 @@ SELECT range_key, SUM(value) FROM ( 30 600 35 700 +########## +# TEST 51: Hash Join Dynamic Filter Pushdown on Compatible Range Inputs +# The Parquet-backed probe accepts the partition-routed dynamic filter. Matching +# Range split points keep build filter i aligned with probe partition i. The +# build has rows only in partitions 0 and 2, so the runtime filter must route +# with a four-way CASE rather than collapse to a single filter. +########## + +statement ok +set datafusion.execution.target_partitions = 4; + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +statement ok +set datafusion.optimizer.enable_join_dynamic_filter_pushdown = true; + +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +query TT +EXPLAIN SELECT b.range_key, b.value, p.value +FROM ( + SELECT range_key, value + FROM range_partitioned + WHERE range_key IN (5, 20) +) b +JOIN range_partitioned p ON b.range_key = p.range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20, pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] +03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + +query TT +EXPLAIN ANALYZE SELECT b.range_key, b.value, p.value +FROM ( + SELECT range_key, value + FROM range_partitioned + WHERE range_key IN (5, 20) +) b +JOIN range_partitioned p ON b.range_key = p.range_key; +---- +Plan with Metrics +01)HashJoinExec: mode=Partitionedmetrics=[output_rows=2,] +02)--DataSourceExec: file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20metrics=[output_rows=2,] +03)--DataSourceExec: file_type=parquet, predicate=DynamicFilter [ CASE WHEN range_key@0 IS NULL OR range_key@0 < 10 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN range_key@0 IS NULL OR range_key@0 < 20 THEN false WHEN range_key@0 IS NULL OR range_key@0 < 30 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]metrics=[output_rows=2,] + +query III +SELECT b.range_key, b.value, p.value +FROM ( + SELECT range_key, value + FROM range_partitioned + WHERE range_key IN (5, 20) +) b +JOIN range_partitioned p ON b.range_key = p.range_key +ORDER BY b.range_key; +---- +5 50 50 +20 200 200 + +statement ok +reset datafusion.optimizer.preserve_file_partitions; + +statement ok +reset datafusion.execution.collect_statistics; + +statement ok +reset datafusion.optimizer.enable_join_dynamic_filter_pushdown; + +statement ok +reset datafusion.optimizer.enable_topk_dynamic_filter_pushdown; + +statement ok +reset datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown; + +statement ok +reset datafusion.execution.parquet.pushdown_filters; + statement ok reset datafusion.explain.physical_plan_only; From 4990a577654f25cb1ed226c03a80208b4a33bc7b Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Aug 2026 17:37:23 +0800 Subject: [PATCH 12/18] cargo fmt --- datafusion/physical-plan/src/joins/hash_join/exec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 20b093ffae657..c5a7ad11d3a54 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -889,7 +889,7 @@ impl HashJoinExec { // because the metadata cannot distinguish those scans from a real hash // repartition. Compatible Range inputs remain safe because matching // ordering and split points align each build filter with its probe - // partition. Other unsupported layouts are rejected. + // partition. Other unsupported layouts are rejected. // Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195). // https://github.com/apache/datafusion/issues/20195 if config.optimizer.preserve_file_partitions > 0 From ee07f7b92d4dfcbe0cb8dcb0d459e3116b8d8da8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 10 Aug 2026 17:38:37 +0800 Subject: [PATCH 13/18] remove duplicate imports --- .../physical_optimizer/filter_pushdown.rs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 163c1fcaac061..9fb4effa818d0 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -186,9 +186,6 @@ fn test_pushdown_into_scan_with_config_options() { // distinction this test exercises is not reachable via SQL. #[tokio::test] async fn test_static_filter_pushdown_through_hash_join() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Create build side with limited values let build_batches = vec![ record_batch!( @@ -1011,9 +1008,6 @@ async fn optimize_and_collect_pushdown_plan( // per-partition CASE filter this test exercises is not reachable via SQL. #[tokio::test] async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Rough sketch of the MRE we're trying to recreate: // COPY (select i as k from generate_series(1, 10000000) as t(i)) // TO 'test_files/scratch/push_down_filter/t1.parquet' @@ -1212,9 +1206,6 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { #[tokio::test] async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Rough sketch of the Range-partitioned MRE we're trying to recreate. The // test hand-wires identical Range repartitioning: // @@ -1419,9 +1410,6 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { // (push_down_filter_parquet.slt::test_hashjoin_dynamic_filter_pushdown). #[tokio::test] async fn test_hashjoin_dynamic_filter_pushdown_collect_left() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let (build_side_schema, build_scan, probe_side_schema, probe_scan) = hashjoin_pushdown_scans(); @@ -1544,9 +1532,6 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() { #[test] fn test_hashjoin_parent_filter_pushdown_same_column_names() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let build_side_schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Utf8, false), Field::new("build_val", DataType::Utf8, false), @@ -1613,9 +1598,6 @@ fn test_hashjoin_parent_filter_pushdown_same_column_names() { #[test] fn test_hashjoin_parent_filter_pushdown_mark_join() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let left_schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Utf8, false), Field::new("val", DataType::Utf8, false), @@ -1683,9 +1665,6 @@ fn test_hashjoin_parent_filter_pushdown_mark_join() { /// only rely on the output side to preserve their semantics. #[test] fn test_hashjoin_parent_filter_pushdown_semi_anti_join() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let left_schema = Arc::new(Schema::new(vec![ Field::new("k", DataType::Utf8, false), Field::new("v", DataType::Utf8, false), @@ -2641,9 +2620,6 @@ fn test_pushdown_with_computed_grouping_key() { // on a hand-wired plan, which does trigger the `false` path. #[tokio::test] async fn test_hashjoin_dynamic_filter_all_partitions_empty() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Test scenario where all build-side partitions are empty // This validates the code path that sets the filter to `false` when no rows can match @@ -2776,9 +2752,6 @@ async fn test_hashjoin_dynamic_filter_all_partitions_empty() { // PartitionMode::Partitioned, which SQL never picks for small parquet inputs. #[tokio::test] async fn test_hashjoin_hash_table_pushdown_partitioned() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let (build_side_schema, build_scan, probe_side_schema, probe_scan) = hashjoin_pushdown_scans(); @@ -2894,9 +2867,6 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() { // IN (SET) invariant is captured in the slt port. #[tokio::test] async fn test_hashjoin_hash_table_pushdown_collect_left() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - let (build_side_schema, build_scan, probe_side_schema, probe_scan) = hashjoin_pushdown_scans(); @@ -3004,9 +2974,6 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() { )] #[tokio::test] async fn test_hashjoin_dynamic_filter_pushdown_is_used() { - use datafusion_common::JoinType; - use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; - // Test both cases: probe side with and without filter pushdown support for (probe_supports_pushdown, expected_is_used) in [(false, false), (true, true)] { let build_side_schema = Arc::new(Schema::new(vec![ @@ -3589,7 +3556,6 @@ fn test_filter_pushdown_through_sort_with_projection() { #[test] fn post_phase_is_idempotent_on_hash_join() { use crate::physical_optimizer::test_utils::{hash_join_exec, parquet_exec, schema}; - use datafusion_common::JoinType; use datafusion_physical_expr::expressions::Column; use datafusion_physical_optimizer::filter_pushdown::FilterPushdown; use datafusion_physical_plan::get_plan_string; From 8903354dec7671888e4ad0dff483aef920525bac Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 10 Aug 2026 22:57:41 +0800 Subject: [PATCH 14/18] CASE WHEN over range_split[...] -> range_partition PhysicalExpr --- .../physical_optimizer/filter_pushdown.rs | 2 +- .../src/joins/hash_join/shared_bounds.rs | 117 +++++--- .../physical-plan/src/repartition/mod.rs | 276 ++++++++++++++++-- .../proto-models/proto/datafusion.proto | 6 + .../proto/src/physical_plan/from_proto.rs | 2 + .../tests/cases/roundtrip_physical_plan.rs | 54 +++- .../test_files/range_partitioning.slt | 2 +- 7 files changed, 401 insertions(+), 58 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 9fb4effa818d0..a9280e0e7a4cc 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -1376,7 +1376,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1 - - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE WHEN a@0 IS NULL OR a@0 < aa OR a@0 = aa AND (b@1 IS NULL OR b@1 < bb) THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) ELSE a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) END ] + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) ELSE a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) END ] " ); diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs index da6cff8a3125a..e3406dff2ea41 100644 --- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs +++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs @@ -31,7 +31,7 @@ use crate::joins::hash_join::inlist_builder::build_struct_fields; use crate::joins::hash_join::partitioned_hash_eval::{ HashExpr, HashTableLookupExpr, SeededRandomState, }; -use crate::ordering::build_lexicographic_filter; +use crate::repartition::RangeExpr; use arrow::array::ArrayRef; use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::config::ConfigOptions; @@ -42,8 +42,7 @@ use datafusion_physical_expr::expressions::{ BinaryExpr, CaseExpr, DynamicFilterPhysicalExpr, InListExpr, IsNullExpr, lit, }; use datafusion_physical_expr::{ - PhysicalExpr, PhysicalExprRef, PhysicalSortExpr, RangePartitioning, - ScalarFunctionExpr, + PhysicalExpr, PhysicalExprRef, RangePartitioning, ScalarFunctionExpr, }; use parking_lot::Mutex; @@ -679,41 +678,37 @@ impl SharedBuildAccumulator { partition_filters.len(), range_partitioning.partition_count() ); - assert_eq!(self.on_right.len(), range_partitioning.ordering().len()); - let sort_exprs = self - .on_right - .iter() - .zip(range_partitioning.ordering()) - .map(|(expr, sort_expr)| { - PhysicalSortExpr::new(Arc::clone(expr), sort_expr.options) - }) - .collect::>(); + let routing_range_expr = Arc::new(RangeExpr::try_new( + self.on_right.clone(), + range_partitioning, + )?) + as Arc; let else_expr = partition_filters .pop() .expect("Range partitioning always has at least one partition"); - // CASE evaluates in order - // - // CASE - // WHEN key >>()?; + .collect(); - Arc::new(CaseExpr::try_new(None, when_then_expr, Some(else_expr))?) - as Arc + Arc::new(CaseExpr::try_new( + Some(routing_range_expr), + when_then_expr, + Some(else_expr), + )?) as Arc } else { // Hash partitioning let routing_hash_expr = Arc::new(HashExpr::new( @@ -844,11 +839,14 @@ pub(super) fn completed_partitions_for_test(acc: &SharedBuildAccumulator) -> usi mod tests { use super::*; - use arrow::array::{ArrayRef, BooleanArray, Int32Array}; + use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array}; use arrow::compute::SortOptions; use arrow::record_batch::RecordBatch; use datafusion_common::SplitPoint; - use datafusion_physical_expr::expressions::{Column, Literal}; + use datafusion_physical_expr::{ + PhysicalSortExpr, + expressions::{Column, Literal}, + }; fn test_on_right() -> Vec { vec![Arc::new(Column::new("probe_key", 0))] @@ -1089,7 +1087,7 @@ mod tests { } #[test] - fn partitioned_range_dynamic_filter_routes_with_searched_case() -> Result<()> { + fn partitioned_range_dynamic_filter_routes_with_range_expr() -> Result<()> { let mut acc = make_partitioned_expr_accumulator_for_test(4); acc.probe_range_partitioning = Some(RangePartitioning::try_new( [PhysicalSortExpr::new( @@ -1114,8 +1112,10 @@ mod tests { let expr = current_expr(&acc); let case = case_expr(&expr); assert!( - case.expr().is_none(), - "Range routing must use searched CASE" + case.expr() + .and_then(|expr| expr.downcast_ref::()) + .is_some(), + "Range routing must use RangeExpr" ); assert_eq!(case.when_then_expr().len(), 3); @@ -1190,7 +1190,7 @@ mod tests { let expr = current_expr(&acc); let case = case_expr(&expr); - assert!(case.expr().is_none()); + assert!(case.expr().is_some()); assert_eq!(case.when_then_expr().len(), 3); let batch = RecordBatch::try_new( @@ -1233,6 +1233,51 @@ mod tests { Ok(()) } + #[test] + fn partitioned_range_dynamic_filter_preserves_signed_zero_routing() -> Result<()> { + let probe_schema = Arc::new(Schema::new(vec![Field::new( + "probe_key", + DataType::Float64, + false, + )])); + let on_right: Vec = vec![Arc::new(Column::new("probe_key", 0))]; + let mut acc = make_accumulator_for_test( + AccumulatedBuildData::Partitioned { + partitions: vec![PartitionStatus::Pending; 2], + completed_partitions: 0, + }, + on_right, + ); + acc.probe_schema = Arc::clone(&probe_schema); + acc.probe_range_partitioning = Some(RangePartitioning::try_new( + [PhysicalSortExpr::new( + Arc::clone(&acc.on_right[0]), + SortOptions::default(), + )] + .into(), + vec![SplitPoint::new(vec![ScalarValue::Float64(Some(0.0))])], + )?); + + acc.build_filter(FinalizeInput::Partitioned(vec![ + PartitionStatus::CanceledUnknown, + reported(PushdownStrategy::Empty, no_bounds()), + ]))?; + + let expr = current_expr(&acc); + let batch = RecordBatch::try_new( + probe_schema, + vec![Arc::new(Float64Array::from(vec![-0.0, 0.0]))], + )?; + let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; + let result = result + .as_any() + .downcast_ref::() + .expect("dynamic filter should evaluate to BooleanArray"); + assert_eq!(result, &BooleanArray::from(vec![true, false])); + + Ok(()) + } + // Regression guard for the build-report lifecycle fix: on `Drop`, a stream // in `BuildReportState::ReportScheduled` still calls `report_canceled_partition` // because it cannot tell whether the coordinator has already observed the diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 873f35fd6aed9..525075b57a8c0 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -20,7 +20,7 @@ //! maintaining the order of the input rows in the output. use std::cmp::Ordering; -use std::fmt::{Debug, Formatter}; +use std::fmt::{Debug, Display, Formatter}; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; @@ -47,9 +47,9 @@ use crate::{ check_if_same_properties, }; -use arrow::array::{Array, PrimitiveArray, RecordBatch, RecordBatchOptions}; +use arrow::array::{Array, PrimitiveArray, RecordBatch, RecordBatchOptions, UInt64Array}; use arrow::compute::take_arrays; -use arrow::datatypes::{SchemaRef, UInt32Type}; +use arrow::datatypes::{DataType, Schema, SchemaRef, UInt32Type}; use arrow_schema::SortOptions; use datafusion_common::config::ConfigOptions; use datafusion_common::stats::Precision; @@ -57,13 +57,20 @@ use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, transpos use datafusion_common::{ ColumnStatistics, DataFusionError, HashMap, ScalarValue, SplitPoint, assert_or_internal_err, internal_datafusion_err, internal_err, + validate_range_split_points, }; use datafusion_common::{Result, not_impl_err}; use datafusion_common_runtime::SpawnedTask; use datafusion_execution::TaskContext; use datafusion_execution::memory_pool::MemoryConsumer; +use datafusion_expr::ColumnarValue; use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr, RangePartitioning}; +use datafusion_physical_expr_common::physical_expr::PhysicalExprRef; use datafusion_physical_expr_common::sort_expr::LexOrdering; +#[cfg(feature = "proto")] +use datafusion_physical_expr_common::sort_expr::{ + sort_exprs_try_from_proto, sort_exprs_try_to_proto, +}; use crate::filter_pushdown::{ ChildPushdownResult, FilterDescription, FilterPushdownPhase, @@ -637,6 +644,218 @@ enum BatchPartitionerState { /// executions and runs. pub const REPARTITION_RANDOM_STATE: SeededRandomState = SeededRandomState::with_seed(0); +/// Physical expression that returns the Range partition for each input row. +/// +/// This uses the same routing function as [`BatchPartitioner`], so dynamic +/// filtering and repartitioning agree for every [`ScalarValue`] comparison. +#[derive(Debug, Hash, PartialEq, Eq)] +pub struct RangeExpr { + on_columns: Vec, + split_points: Vec, + sort_options: Vec, +} + +impl RangeExpr { + /// Creates a Range expression for `on_columns` using the supplied routing + /// metadata. + pub fn try_new( + on_columns: Vec, + range_partitioning: &RangePartitioning, + ) -> Result { + let sort_options = range_partitioning + .ordering() + .iter() + .map(|expr| expr.options) + .collect(); + Self::try_new_parts( + on_columns, + range_partitioning.split_points().to_vec(), + sort_options, + ) + } + + fn try_new_parts( + on_columns: Vec, + split_points: Vec, + sort_options: Vec, + ) -> Result { + assert_or_internal_err!(!on_columns.is_empty(), "RangeExpr requires a key"); + assert_or_internal_err!( + on_columns.len() == sort_options.len(), + "RangeExpr key count must match sort options" + ); + validate_range_split_points(&split_points, &sort_options)?; + Ok(Self { + on_columns, + split_points, + sort_options, + }) + } + + /// Returns the Range split points used for routing. + pub fn split_points(&self) -> &[SplitPoint] { + &self.split_points + } + + /// Returns the per-key sort options used for routing. + pub fn sort_options(&self) -> &[SortOptions] { + &self.sort_options + } +} + +impl Display for RangeExpr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "range_partition") + } +} + +impl PhysicalExpr for RangeExpr { + fn children(&self) -> Vec<&PhysicalExprRef> { + self.on_columns.iter().collect() + } + + fn with_new_children( + self: Arc, + children: Vec, + ) -> Result { + assert_or_internal_err!( + children.len() == self.on_columns.len(), + "RangeExpr expected {} children, got {}", + self.on_columns.len(), + children.len() + ); + Ok(Arc::new(Self::try_new_parts( + children, + self.split_points.clone(), + self.sort_options.clone(), + )?)) + } + + fn data_type(&self, _input_schema: &Schema) -> Result { + Ok(DataType::UInt64) + } + + fn nullable(&self, _input_schema: &Schema) -> Result { + Ok(false) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let arrays = evaluate_expressions_to_arrays(self.on_columns.iter(), batch)?; + let mut row_key_buffer = Vec::with_capacity(arrays.len()); + let mut partition_ids = Vec::with_capacity(batch.num_rows()); + for row_idx in 0..batch.num_rows() { + extract_row_at_idx_to_buf(&arrays, row_idx, &mut row_key_buffer)?; + partition_ids.push(range_partition_id( + &row_key_buffer, + &self.split_points, + &self.sort_options, + )? as u64); + } + Ok(ColumnarValue::Array(Arc::new(UInt64Array::from( + partition_ids, + )))) + } + + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "range_partition") + } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>, + ) -> Result> { + // Encode the raw ordered children: rebuilding a `LexOrdering` would + // deduplicate equivalent children after dynamic-filter remapping. + use datafusion_proto_models::protobuf; + + let sort_exprs = self + .on_columns + .iter() + .zip(&self.sort_options) + .map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options)) + .collect::>(); + let sort_expr = sort_exprs_try_to_proto(&sort_exprs, ctx)?; + let split_point = self + .split_points + .iter() + .map(|split_point| { + let value = split_point + .values() + .iter() + .map(|value| value.try_into().map_err(Into::into)) + .collect::>>()?; + Ok(protobuf::PhysicalRangeSplitPoint { value }) + }) + .collect::>>()?; + Ok(Some(protobuf::PhysicalExprNode { + expr_id: None, + expr_type: Some(protobuf::physical_expr_node::ExprType::RangeExpr( + protobuf::PhysicalRangeExprNode { + sort_expr, + split_point, + }, + )), + })) + } +} + +#[cfg(feature = "proto")] +impl RangeExpr { + /// Reconstructs a [`RangeExpr`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalExprNode, + ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>, + ) -> Result { + // Decode the raw ordered children for the same reason as `try_to_proto`. + use datafusion_proto_models::protobuf; + + let range_expr = match &node.expr_type { + Some(protobuf::physical_expr_node::ExprType::RangeExpr(expr)) => expr, + _ => return internal_err!("PhysicalExprNode is not a RangeExpr"), + }; + let sort_exprs = sort_exprs_try_from_proto(&range_expr.sort_expr, ctx)?; + let (on_columns, sort_options) = sort_exprs + .into_iter() + .map(|sort_expr| (sort_expr.expr, sort_expr.options)) + .unzip(); + let split_points = range_expr + .split_point + .iter() + .map(|split_point| { + let values = split_point + .value + .iter() + .map(|value| ScalarValue::try_from(value).map_err(Into::into)) + .collect::>>()?; + Ok(SplitPoint::new(values)) + }) + .collect::>>()?; + Ok(Arc::new(Self::try_new_parts( + on_columns, + split_points, + sort_options, + )?)) + } +} + +fn range_partition_id( + row_key: &[ScalarValue], + split_points: &[SplitPoint], + sort_options: &[SortOptions], +) -> Result { + let mut low = 0; + let mut high = split_points.len(); + while low < high { + let mid = low + (high - low) / 2; + match compare_rows(row_key, split_points[mid].values(), sort_options)? { + Ordering::Less => high = mid, + Ordering::Equal | Ordering::Greater => low = mid + 1, + } + } + Ok(low) +} + /// Computes `value % divisor` without division in the hot loop when `divisor` /// is fixed for many values. /// @@ -971,22 +1190,9 @@ impl BatchPartitioner { // Note that `extract_row_at_idx_to_buf` clears the `row_key_buffer` on each invocation, creating a new row key for comparison for each row extract_row_at_idx_to_buf(arrays, row_idx, row_key_buffer)?; - let mut low = 0; - let mut high = split_points.len(); - while low < high { - let mid = low + (high - low) / 2; - let comparison = compare_rows( - row_key_buffer, - split_points[mid].values(), - sort_options, - )?; - match comparison { - Ordering::Less => high = mid, - Ordering::Equal | Ordering::Greater => low = mid + 1, - } - } - - indices[low].push(row_idx as u32) + let partition = + range_partition_id(row_key_buffer, split_points, sort_options)?; + indices[partition].push(row_idx as u32) } Ok(()) @@ -2248,6 +2454,38 @@ mod tests { use datafusion_physical_expr::{PhysicalSortExpr, RangePartitioning, SplitPoint}; use insta::assert_snapshot; + #[test] + fn range_expr_preserves_duplicate_remapped_children() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::UInt32, false), + Field::new("b", DataType::UInt32, false), + ])); + let range_partitioning = RangePartitioning::try_new( + [ + PhysicalSortExpr::new(col("a", &schema)?, SortOptions::default()), + PhysicalSortExpr::new(col("b", &schema)?, SortOptions::default()), + ] + .into(), + vec![SplitPoint::new(vec![ + ScalarValue::UInt32(Some(10)), + ScalarValue::UInt32(Some(20)), + ])], + )?; + let expr = Arc::new(RangeExpr::try_new( + vec![col("a", &schema)?, col("b", &schema)?], + &range_partitioning, + )?); + let remapped = col("a", &schema)?; + let rewritten = + expr.with_new_children(vec![Arc::clone(&remapped), Arc::clone(&remapped)])?; + + let children = rewritten.children(); + assert_eq!(children.len(), 2); + assert_eq!(children[0], children[1]); + + Ok(()) + } + #[test] fn strength_reduced_u64_remainder_matches_modulo() { let divisors = [ diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 5a8bede195826..c30aed38fe5c8 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -1050,6 +1050,7 @@ message PhysicalExprNode { PhysicalHigherOrderUdfNode higher_order_udf = 24; PhysicalLambdaExprNode lambda = 25; PhysicalLambdaVariableExprNode lambda_variable = 26; + PhysicalRangeExprNode range_expr = 27; } } @@ -1201,6 +1202,11 @@ message PhysicalHashExprNode { string description = 6; } +message PhysicalRangeExprNode { + repeated PhysicalSortExprNode sort_expr = 1; + repeated PhysicalRangeSplitPoint split_point = 2; +} + message FilterExecNode { PhysicalPlanNode input = 1; PhysicalExprNode expr = 2; diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs index 9a3a845d7ce57..df09f22297666 100644 --- a/datafusion/proto/src/physical_plan/from_proto.rs +++ b/datafusion/proto/src/physical_plan/from_proto.rs @@ -46,6 +46,7 @@ use datafusion_physical_plan::expressions::{ }; use datafusion_physical_plan::joins::HashExpr; use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx; +use datafusion_physical_plan::repartition::RangeExpr; use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field}; use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr}; use datafusion_proto_common::common::proto_error; @@ -354,6 +355,7 @@ pub fn parse_physical_expr_with_converter( } ExprType::LikeExpr(_) => LikeExpr::try_from_proto(proto, &decode_ctx)?, ExprType::HashExpr(_) => HashExpr::try_from_proto(proto, &decode_ctx)?, + ExprType::RangeExpr(_) => RangeExpr::try_from_proto(proto, &decode_ctx)?, ExprType::ScalarSubquery(_) => { let results = ctx.scalar_subquery_results().ok_or_else(|| { proto_error( diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 9efbd90f152c8..5555830fe54eb 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -83,7 +83,7 @@ use datafusion::physical_plan::metrics::MetricCategory; use datafusion::physical_plan::placeholder_row::PlaceholderRowExec; use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx; -use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::repartition::{RangeExpr, RepartitionExec}; use datafusion::physical_plan::scalar_subquery::{ ScalarSubqueryExec, ScalarSubqueryLink, }; @@ -3705,6 +3705,58 @@ fn roundtrip_hash_expr() -> Result<()> { roundtrip_test(filter) } +#[test] +fn roundtrip_range_expr() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Float64, false), + Field::new("b", DataType::Float64, false), + ])); + let options = [SortOptions::new(true, true), SortOptions::new(false, false)]; + let range_partitioning = RangePartitioning::try_new( + [ + PhysicalSortExpr::new(col("a", &schema)?, options[0]), + PhysicalSortExpr::new(col("b", &schema)?, options[1]), + ] + .into(), + vec![SplitPoint::new(vec![ + ScalarValue::Float64(Some(0.0)), + ScalarValue::Float64(Some(1.0)), + ])], + )?; + let range_expr: Arc = Arc::new(RangeExpr::try_new( + // Expression remapping may produce duplicate children. Preserve both + // so their sort options stay aligned with the split-point values. + vec![col("a", &schema)?, col("a", &schema)?], + &range_partitioning, + )?); + let filter_expr = binary(range_expr, Operator::Eq, lit(0u64), &schema)?; + let plan = Arc::new(FilterExec::try_new( + filter_expr, + Arc::new(EmptyExec::new(Arc::clone(&schema))), + )?); + + let ctx = SessionContext::new(); + let result = roundtrip_test_and_return( + plan, + &ctx, + &DefaultPhysicalExtensionCodec {}, + &DefaultPhysicalProtoConverter {}, + )?; + let filter = result.downcast_ref::().unwrap(); + let binary = filter.predicate().downcast_ref::().unwrap(); + let range_expr = binary.left().downcast_ref::().unwrap(); + assert_eq!(range_expr.split_points(), range_partitioning.split_points()); + assert_eq!(range_expr.sort_options(), &options); + let children = range_expr.children(); + assert_eq!(children.len(), 2); + for child in children { + let column = child.downcast_ref::().unwrap(); + assert_eq!((column.name(), column.index()), ("a", 0)); + } + + Ok(()) +} + #[test] fn custom_proto_converter_intercepts() -> Result<()> { #[derive(Default)] diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index b86ecbfcb5a30..326856a352f36 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -1882,7 +1882,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; Plan with Metrics 01)HashJoinExec: mode=Partitionedmetrics=[output_rows=2,] 02)--DataSourceExec: file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20metrics=[output_rows=2,] -03)--DataSourceExec: file_type=parquet, predicate=DynamicFilter [ CASE WHEN range_key@0 IS NULL OR range_key@0 < 10 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN range_key@0 IS NULL OR range_key@0 < 20 THEN false WHEN range_key@0 IS NULL OR range_key@0 < 30 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]metrics=[output_rows=2,] +03)--DataSourceExec: file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 1 THEN false WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]metrics=[output_rows=2,] query III SELECT b.range_key, b.value, p.value From eaf7f863186913be7d0dc5ca53b22b3fe7d4ea09 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 10 Aug 2026 23:05:53 +0800 Subject: [PATCH 15/18] codegen --- .../proto-models/src/generated/pbjson.rs | 124 ++++++++++++++++++ .../proto-models/src/generated/prost.rs | 11 +- 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index e04124a61c969..a92311d22dbf5 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -18454,6 +18454,9 @@ impl serde::Serialize for PhysicalExprNode { physical_expr_node::ExprType::LambdaVariable(v) => { struct_ser.serialize_field("lambdaVariable", v)?; } + physical_expr_node::ExprType::RangeExpr(v) => { + struct_ser.serialize_field("rangeExpr", v)?; + } } } struct_ser.end() @@ -18509,6 +18512,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { "lambda", "lambda_variable", "lambdaVariable", + "range_expr", + "rangeExpr", ]; #[allow(clippy::enum_variant_names)] @@ -18538,6 +18543,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { HigherOrderUdf, Lambda, LambdaVariable, + RangeExpr, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -18584,6 +18590,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { "higherOrderUdf" | "higher_order_udf" => Ok(GeneratedField::HigherOrderUdf), "lambda" => Ok(GeneratedField::Lambda), "lambdaVariable" | "lambda_variable" => Ok(GeneratedField::LambdaVariable), + "rangeExpr" | "range_expr" => Ok(GeneratedField::RangeExpr), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -18781,6 +18788,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { return Err(serde::de::Error::duplicate_field("lambdaVariable")); } expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::LambdaVariable) +; + } + GeneratedField::RangeExpr => { + if expr_type__.is_some() { + return Err(serde::de::Error::duplicate_field("rangeExpr")); + } + expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::RangeExpr) ; } } @@ -20841,6 +20855,116 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode { deserializer.deserialize_struct("datafusion.PhysicalPlanNode", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for PhysicalRangeExprNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.sort_expr.is_empty() { + len += 1; + } + if !self.split_point.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalRangeExprNode", len)?; + if !self.sort_expr.is_empty() { + struct_ser.serialize_field("sortExpr", &self.sort_expr)?; + } + if !self.split_point.is_empty() { + struct_ser.serialize_field("splitPoint", &self.split_point)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for PhysicalRangeExprNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "sort_expr", + "sortExpr", + "split_point", + "splitPoint", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + SortExpr, + SplitPoint, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr), + "splitPoint" | "split_point" => Ok(GeneratedField::SplitPoint), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = PhysicalRangeExprNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.PhysicalRangeExprNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut sort_expr__ = None; + let mut split_point__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::SortExpr => { + if sort_expr__.is_some() { + return Err(serde::de::Error::duplicate_field("sortExpr")); + } + sort_expr__ = Some(map_.next_value()?); + } + GeneratedField::SplitPoint => { + if split_point__.is_some() { + return Err(serde::de::Error::duplicate_field("splitPoint")); + } + split_point__ = Some(map_.next_value()?); + } + } + } + Ok(PhysicalRangeExprNode { + sort_expr: sort_expr__.unwrap_or_default(), + split_point: split_point__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("datafusion.PhysicalRangeExprNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for PhysicalRangePartitioning { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 51e1a6fa92713..bbbc3b5d7c716 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1557,7 +1557,7 @@ pub struct PhysicalExprNode { pub expr_id: ::core::option::Option, #[prost( oneof = "physical_expr_node::ExprType", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27" )] pub expr_type: ::core::option::Option, } @@ -1620,6 +1620,8 @@ pub mod physical_expr_node { Lambda(::prost::alloc::boxed::Box), #[prost(message, tag = "26")] LambdaVariable(super::PhysicalLambdaVariableExprNode), + #[prost(message, tag = "27")] + RangeExpr(super::PhysicalRangeExprNode), } } #[derive(Clone, PartialEq, ::prost::Message)] @@ -1859,6 +1861,13 @@ pub struct PhysicalHashExprNode { pub description: ::prost::alloc::string::String, } #[derive(Clone, PartialEq, ::prost::Message)] +pub struct PhysicalRangeExprNode { + #[prost(message, repeated, tag = "1")] + pub sort_expr: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "2")] + pub split_point: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] pub struct FilterExecNode { #[prost(message, optional, boxed, tag = "1")] pub input: ::core::option::Option<::prost::alloc::boxed::Box>, From 01210bd23fdc5462fa77bbe64fcae846192bd490 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 11 Aug 2026 14:12:03 +0800 Subject: [PATCH 16/18] revert topk filter builder --- datafusion/physical-plan/src/ordering.rs | 99 ------------------------ datafusion/physical-plan/src/topk/mod.rs | 99 ++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 105 deletions(-) diff --git a/datafusion/physical-plan/src/ordering.rs b/datafusion/physical-plan/src/ordering.rs index 9c037e44bcdd4..8b596b9cb23eb 100644 --- a/datafusion/physical-plan/src/ordering.rs +++ b/datafusion/physical-plan/src/ordering.rs @@ -15,13 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; - -use datafusion_common::{Result, ScalarValue, assert_or_internal_err}; -use datafusion_expr::Operator; -use datafusion_physical_expr::expressions::{BinaryExpr, is_not_null, is_null, lit}; -use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr}; - /// Specifies how the input to an aggregation or window operator is ordered /// relative to their `GROUP BY` or `PARTITION BY` expressions. /// @@ -59,95 +52,3 @@ pub enum InputOrderMode { /// existing ordering. Sorted, } - -/// Build the filter expression with the given thresholds. -/// This is now called outside of any locks to reduce critical section time. -pub(crate) fn build_lexicographic_filter( - sort_exprs: &[PhysicalSortExpr], - thresholds: &[ScalarValue], -) -> Result> { - assert_or_internal_err!(!sort_exprs.is_empty(), "Sort expressions must not be empty"); - assert_or_internal_err!( - sort_exprs.len() == thresholds.len(), - "Sort expressions and thresholds must have the same length" - ); - - // Create filter expressions for each threshold - let mut filters: Vec> = Vec::with_capacity(thresholds.len()); - - let mut prev_sort_expr: Option> = None; - for (sort_expr, value) in sort_exprs.iter().zip(thresholds.iter()) { - // Create the appropriate operator based on sort order - let op = if sort_expr.options.descending { - // For descending sort, we want col > threshold (exclude smaller values) - Operator::Gt - } else { - // For ascending sort, we want col < threshold (exclude larger values) - Operator::Lt - }; - - let value_null = value.is_null(); - - let comparison = Arc::new(BinaryExpr::new( - Arc::clone(&sort_expr.expr), - op, - lit(value.clone()), - )); - - let comparison_with_null = match (sort_expr.options.nulls_first, value_null) { - // For nulls first, transform to (threshold.value is not null) and (threshold.expr is null or comparison) - (true, true) => lit(false), - (true, false) => Arc::new(BinaryExpr::new( - is_null(Arc::clone(&sort_expr.expr))?, - Operator::Or, - comparison, - )), - // For nulls last, transform to (threshold.value is null and threshold.expr is not null) - // or (threshold.value is not null and comparison) - (false, true) => is_not_null(Arc::clone(&sort_expr.expr))?, - (false, false) => comparison, - }; - - let mut eq_expr = Arc::new(BinaryExpr::new( - Arc::clone(&sort_expr.expr), - Operator::Eq, - lit(value.clone()), - )); - - if value_null { - eq_expr = Arc::new(BinaryExpr::new( - is_null(Arc::clone(&sort_expr.expr))?, - Operator::Or, - eq_expr, - )); - } - - // For a query like order by a, b, the filter for column `b` is only applied if - // the condition a = threshold.value (considering null equality) is met. - // Therefore, we add equality predicates for all preceding fields to the filter logic of the current field, - // and include the current field's equality predicate in `prev_sort_expr` for use with subsequent fields. - match prev_sort_expr.take() { - None => { - prev_sort_expr = Some(eq_expr); - filters.push(comparison_with_null); - } - Some(p) => { - filters.push(Arc::new(BinaryExpr::new( - Arc::clone(&p), - Operator::And, - comparison_with_null, - ))); - - prev_sort_expr = - Some(Arc::new(BinaryExpr::new(p, Operator::And, eq_expr))); - } - } - } - - let dynamic_predicate = filters - .into_iter() - .reduce(|a, b| Arc::new(BinaryExpr::new(a, Operator::Or, b))) - .expect("sort expressions are checked non-empty"); - - Ok(dynamic_predicate) -} diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs index 6d30462219c39..1e3efff36b1d8 100644 --- a/datafusion/physical-plan/src/topk/mod.rs +++ b/datafusion/physical-plan/src/topk/mod.rs @@ -25,7 +25,7 @@ use arrow::{ }, row::{OwnedRow, RowConverter, Rows, SortField}, }; -use datafusion_expr::ColumnarValue; +use datafusion_expr::{ColumnarValue, Operator}; use std::mem::size_of; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc}; @@ -34,7 +34,6 @@ use super::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory, RecordOutput, }; -use crate::ordering::build_lexicographic_filter; use crate::spill::get_record_batch_memory_size; use crate::{SendableRecordBatchStream, stream::RecordBatchStreamAdapter}; @@ -49,7 +48,7 @@ use datafusion_execution::{ }; use datafusion_physical_expr::{ PhysicalExpr, - expressions::{DynamicFilterPhysicalExpr, lit}, + expressions::{BinaryExpr, DynamicFilterPhysicalExpr, is_not_null, is_null, lit}, }; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; use parking_lot::RwLock; @@ -564,7 +563,7 @@ impl TopK { let thresholds = boundary.threshold_values(&self.expr)?; // Build the filter expression OUTSIDE any synchronization - let predicate = build_lexicographic_filter(&self.expr, &thresholds)?; + let predicate = Self::build_filter_expression(&self.expr, &thresholds)?; let new_threshold = boundary.threshold(self.encode_topk_common_prefix_row(boundary)?); @@ -584,13 +583,101 @@ impl TopK { filter.shared_threshold = Some(new_threshold); // Update the filter expression - if !predicate.eq(&lit(true)) { - filter.expr.update(predicate)?; + if let Some(pred) = predicate + && !pred.eq(&lit(true)) + { + filter.expr.update(pred)?; } Ok(()) } + /// Build the filter expression with the given thresholds. + /// This is now called outside of any locks to reduce critical section time. + fn build_filter_expression( + sort_exprs: &[PhysicalSortExpr], + thresholds: &[ScalarValue], + ) -> Result>> { + // Create filter expressions for each threshold + let mut filters: Vec> = + Vec::with_capacity(thresholds.len()); + + let mut prev_sort_expr: Option> = None; + for (sort_expr, value) in sort_exprs.iter().zip(thresholds.iter()) { + // Create the appropriate operator based on sort order + let op = if sort_expr.options.descending { + // For descending sort, we want col > threshold (exclude smaller values) + Operator::Gt + } else { + // For ascending sort, we want col < threshold (exclude larger values) + Operator::Lt + }; + + let value_null = value.is_null(); + + let comparison = Arc::new(BinaryExpr::new( + Arc::clone(&sort_expr.expr), + op, + lit(value.clone()), + )); + + let comparison_with_null = match (sort_expr.options.nulls_first, value_null) { + // For nulls first, transform to (threshold.value is not null) and (threshold.expr is null or comparison) + (true, true) => lit(false), + (true, false) => Arc::new(BinaryExpr::new( + is_null(Arc::clone(&sort_expr.expr))?, + Operator::Or, + comparison, + )), + // For nulls last, transform to (threshold.value is null and threshold.expr is not null) + // or (threshold.value is not null and comparison) + (false, true) => is_not_null(Arc::clone(&sort_expr.expr))?, + (false, false) => comparison, + }; + + let mut eq_expr = Arc::new(BinaryExpr::new( + Arc::clone(&sort_expr.expr), + Operator::Eq, + lit(value.clone()), + )); + + if value_null { + eq_expr = Arc::new(BinaryExpr::new( + is_null(Arc::clone(&sort_expr.expr))?, + Operator::Or, + eq_expr, + )); + } + + // For a query like order by a, b, the filter for column `b` is only applied if + // the condition a = threshold.value (considering null equality) is met. + // Therefore, we add equality predicates for all preceding fields to the filter logic of the current field, + // and include the current field's equality predicate in `prev_sort_expr` for use with subsequent fields. + match prev_sort_expr.take() { + None => { + prev_sort_expr = Some(eq_expr); + filters.push(comparison_with_null); + } + Some(p) => { + filters.push(Arc::new(BinaryExpr::new( + Arc::clone(&p), + Operator::And, + comparison_with_null, + ))); + + prev_sort_expr = + Some(Arc::new(BinaryExpr::new(p, Operator::And, eq_expr))); + } + } + } + + let dynamic_predicate = filters + .into_iter() + .reduce(|a, b| Arc::new(BinaryExpr::new(a, Operator::Or, b))); + + Ok(dynamic_predicate) + } + /// If input ordering shares a common sort prefix with the TopK, /// check if the computation can be finished early. /// From 00ef9f04b81ed1762391d4b9d44eefaf8a2841be Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 11 Aug 2026 14:15:38 +0800 Subject: [PATCH 17/18] assert_or_internal_err, top import and on_columns accessor --- .../src/joins/hash_join/shared_bounds.rs | 5 +++- .../physical-plan/src/repartition/mod.rs | 23 +++++++++---------- .../tests/cases/roundtrip_physical_plan.rs | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs index 768fc22f045ce..94ec4565a4cef 100644 --- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs +++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs @@ -37,6 +37,7 @@ use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::config::ConfigOptions; use datafusion_common::{ DataFusionError, NullEquality, Result, ScalarValue, SharedResult, + assert_or_internal_err, }; use datafusion_expr::Operator; use datafusion_functions::core::r#struct as struct_func; @@ -706,7 +707,9 @@ impl SharedBuildAccumulator { Arc::clone(&partition_filters[real_partition_ids[0]]) } else if let Some(range_partitioning) = &self.probe_range_partitioning { // Range partitioning - assert_eq!( + assert_or_internal_err!( + partition_filters.len() == range_partitioning.partition_count(), + "Dynamic filter partition count {} does not match Range partition count {}", partition_filters.len(), range_partitioning.partition_count() ); diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 3eb056e3b4bbe..944f08117d75a 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -72,6 +72,8 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering; use datafusion_physical_expr_common::sort_expr::{ sort_exprs_try_from_proto, sort_exprs_try_to_proto, }; +#[cfg(feature = "proto")] +use datafusion_proto_models::protobuf; use crate::filter_pushdown::{ ChildPushdownResult, FilterDescription, FilterPushdownPhase, @@ -693,6 +695,11 @@ impl RangeExpr { }) } + /// Get the columns used to compute Range partition IDs. + pub fn on_columns(&self) -> &[PhysicalExprRef] { + &self.on_columns + } + /// Returns the Range split points used for routing. pub fn split_points(&self) -> &[SplitPoint] { &self.split_points @@ -765,11 +772,9 @@ impl PhysicalExpr for RangeExpr { fn try_to_proto( &self, ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>, - ) -> Result> { + ) -> Result> { // Encode the raw ordered children: rebuilding a `LexOrdering` would // deduplicate equivalent children after dynamic-filter remapping. - use datafusion_proto_models::protobuf; - let sort_exprs = self .on_columns .iter() @@ -805,12 +810,10 @@ impl PhysicalExpr for RangeExpr { impl RangeExpr { /// Reconstructs a [`RangeExpr`] from its protobuf representation. pub fn try_from_proto( - node: &datafusion_proto_models::protobuf::PhysicalExprNode, + node: &protobuf::PhysicalExprNode, ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>, ) -> Result { // Decode the raw ordered children for the same reason as `try_to_proto`. - use datafusion_proto_models::protobuf; - let range_expr = match &node.expr_type { Some(protobuf::physical_expr_node::ExprType::RangeExpr(expr)) => expr, _ => return internal_err!("PhysicalExprNode is not a RangeExpr"), @@ -1929,9 +1932,7 @@ impl ExecutionPlan for RepartitionExec { fn try_to_proto( &self, ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { - use datafusion_proto_models::protobuf; - + ) -> Result> { let input = ctx.encode_child(self.input())?; let partitioning = self.partitioning().try_to_proto(&ctx.expr_ctx())?; @@ -1954,11 +1955,9 @@ impl ExecutionPlan for RepartitionExec { impl RepartitionExec { /// Reconstruct a [`RepartitionExec`] from its protobuf representation. pub fn try_from_proto( - node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + node: &protobuf::PhysicalPlanNode, ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, ) -> Result> { - use datafusion_proto_models::protobuf; - let repart = crate::expect_plan_variant!( node, protobuf::physical_plan_node::PhysicalPlanType::Repartition, diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 72189e005e702..ae8cb9483aaf8 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -4085,7 +4085,7 @@ fn roundtrip_range_expr() -> Result<()> { let range_expr = binary.left().downcast_ref::().unwrap(); assert_eq!(range_expr.split_points(), range_partitioning.split_points()); assert_eq!(range_expr.sort_options(), &options); - let children = range_expr.children(); + let children = range_expr.on_columns(); assert_eq!(children.len(), 2); for child in children { let column = child.downcast_ref::().unwrap(); From f9f1f0d09f665aac86410e9348272c5498610f4f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 11 Aug 2026 14:18:15 +0800 Subject: [PATCH 18/18] check the range and sort properties --- .../physical-plan/src/repartition/mod.rs | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 944f08117d75a..db5c1cff3461b 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -2474,16 +2474,18 @@ mod tests { Field::new("a", DataType::UInt32, false), Field::new("b", DataType::UInt32, false), ])); + let sort_options = [SortOptions::new(false, false), SortOptions::new(true, true)]; + let split_points = vec![SplitPoint::new(vec![ + ScalarValue::UInt32(Some(10)), + ScalarValue::UInt32(Some(20)), + ])]; let range_partitioning = RangePartitioning::try_new( [ - PhysicalSortExpr::new(col("a", &schema)?, SortOptions::default()), - PhysicalSortExpr::new(col("b", &schema)?, SortOptions::default()), + PhysicalSortExpr::new(col("a", &schema)?, sort_options[0]), + PhysicalSortExpr::new(col("b", &schema)?, sort_options[1]), ] .into(), - vec![SplitPoint::new(vec![ - ScalarValue::UInt32(Some(10)), - ScalarValue::UInt32(Some(20)), - ])], + split_points.clone(), )?; let expr = Arc::new(RangeExpr::try_new( vec![col("a", &schema)?, col("b", &schema)?], @@ -2493,9 +2495,16 @@ mod tests { let rewritten = expr.with_new_children(vec![Arc::clone(&remapped), Arc::clone(&remapped)])?; - let children = rewritten.children(); - assert_eq!(children.len(), 2); - assert_eq!(children[0], children[1]); + let rewritten = rewritten + .downcast_ref::() + .expect("rewritten expression should remain a RangeExpr"); + assert_eq!(rewritten.on_columns().len(), 2); + assert!(Arc::ptr_eq( + &rewritten.on_columns()[0], + &rewritten.on_columns()[1] + )); + assert_eq!(rewritten.sort_options(), sort_options); + assert_eq!(rewritten.split_points(), split_points); Ok(()) }