diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rs index 3be331a1ee1ba..12f56ff3c7276 100644 --- a/datafusion/physical-plan/src/buffer.rs +++ b/datafusion/physical-plan/src/buffer.rs @@ -305,13 +305,24 @@ impl ExecutionPlan for BufferExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `BufferExec` is a compile error here until it is either serialized or + // explicitly documented as not needing to be. + let Self { + input, + // Derived from the input's properties at construction time. + properties: _, + capacity, + // Runtime metrics, not part of the plan shape. + metrics: _, + } = self; + let input = ctx.encode_child(input)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Buffer(Box::new( protobuf::BufferExecNode { input: Some(Box::new(input)), - capacity: self.capacity() as u64, + capacity: *capacity as u64, }, )), ), @@ -336,9 +347,11 @@ impl BufferExec { protobuf::physical_plan_node::PhysicalPlanType::Buffer, "BufferExec", ); - let input = - ctx.decode_required_child(buffer.input.as_deref(), "BufferExec", "input")?; - Ok(Arc::new(BufferExec::new(input, buffer.capacity as usize))) + // Destructure exhaustively so that a new field on `BufferExecNode` is a + // compile error here rather than a silently dropped field. + let protobuf::BufferExecNode { input, capacity } = &**buffer; + let input = ctx.decode_required_child(input.as_deref(), "BufferExec", "input")?; + Ok(Arc::new(BufferExec::new(input, *capacity as usize))) } } diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs index c5b91767777f2..21f07a7427600 100644 --- a/datafusion/physical-plan/src/coalesce_batches.rs +++ b/datafusion/physical-plan/src/coalesce_batches.rs @@ -297,14 +297,26 @@ impl ExecutionPlan for CoalesceBatchesExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `CoalesceBatchesExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + target_batch_size, + fetch, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Derived plan properties, recomputed on decode. + cache: _, + } = self; + let input = ctx.encode_child(input)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::CoalesceBatches( Box::new(protobuf::CoalesceBatchesExecNode { input: Some(Box::new(input)), - target_batch_size: self.target_batch_size() as u32, - fetch: self.fetch().map(|n| n as u32), + target_batch_size: *target_batch_size as u32, + fetch: fetch.map(|n| n as u32), }), ), ), @@ -335,14 +347,19 @@ impl CoalesceBatchesExec { protobuf::physical_plan_node::PhysicalPlanType::CoalesceBatches, "CoalesceBatchesExec", ); - let input = ctx.decode_required_child( - coalesce_batches.input.as_deref(), - "CoalesceBatchesExec", - "input", - )?; + // Destructure exhaustively so that a new field on + // `CoalesceBatchesExecNode` is a compile error here rather than a + // silently dropped field. + let protobuf::CoalesceBatchesExecNode { + input, + target_batch_size, + fetch, + } = &**coalesce_batches; + let input = + ctx.decode_required_child(input.as_deref(), "CoalesceBatchesExec", "input")?; Ok(Arc::new( - CoalesceBatchesExec::new(input, coalesce_batches.target_batch_size as usize) - .with_fetch(coalesce_batches.fetch.map(|f| f as usize)), + CoalesceBatchesExec::new(input, *target_batch_size as usize) + .with_fetch(fetch.map(|f| f as usize)), )) } } diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index f9694e0d16817..37476e83fd0b1 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -353,13 +353,24 @@ impl ExecutionPlan for CoalescePartitionsExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `CoalescePartitionsExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Derived plan properties, recomputed on decode. + cache: _, + fetch, + } = self; + let input = ctx.encode_child(input)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Merge(Box::new( protobuf::CoalescePartitionsExecNode { input: Some(Box::new(input)), - fetch: self.fetch().map(|f| f as u32), + fetch: fetch.map(|f| f as u32), }, )), ), @@ -386,14 +397,17 @@ impl CoalescePartitionsExec { protobuf::physical_plan_node::PhysicalPlanType::Merge, "CoalescePartitionsExec", ); + // Destructure exhaustively so that a new field on + // `CoalescePartitionsExecNode` is a compile error here rather than a + // silently dropped field. + let protobuf::CoalescePartitionsExecNode { input, fetch } = &**merge; let input = ctx.decode_required_child( - merge.input.as_deref(), + input.as_deref(), "CoalescePartitionsExec", "input", )?; Ok(Arc::new( - CoalescePartitionsExec::new(input) - .with_fetch(merge.fetch.map(|f| f as usize)), + CoalescePartitionsExec::new(input).with_fetch(fetch.map(|f| f as usize)), )) } } diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs index a5b57f546bbfa..a749582a2ca6e 100644 --- a/datafusion/physical-plan/src/coop.rs +++ b/datafusion/physical-plan/src/coop.rs @@ -376,7 +376,15 @@ impl ExecutionPlan for CooperativeExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `CooperativeExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + // Derived from the input's properties at construction time. + properties: _, + } = self; + let input = ctx.encode_child(input)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Cooperative(Box::new( @@ -406,11 +414,12 @@ impl CooperativeExec { protobuf::physical_plan_node::PhysicalPlanType::Cooperative, "CooperativeExec", ); - let input = ctx.decode_required_child( - cooperative.input.as_deref(), - "CooperativeExec", - "input", - )?; + // Destructure exhaustively so that a new field on + // `CooperativeExecNode` is a compile error here rather than a silently + // dropped field. + let protobuf::CooperativeExecNode { input } = &**cooperative; + let input = + ctx.decode_required_child(input.as_deref(), "CooperativeExec", "input")?; Ok(Arc::new(CooperativeExec::new(input))) } } diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs index 3bd38bf238dc1..40cbfaaa2eb53 100644 --- a/datafusion/physical-plan/src/empty.rs +++ b/datafusion/physical-plan/src/empty.rs @@ -192,16 +192,22 @@ impl ExecutionPlan for EmptyExec { _ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let schema = self.schema().as_ref().try_into()?; + // Destructure exhaustively (no `..`) so that adding a field to + // `EmptyExec` is a compile error here until it is either serialized or + // explicitly documented as not needing to be. + let Self { + schema, + partitions, + // Derived from `schema` and `partitions`, recomputed on decode. + cache: _, + } = self; + let schema = schema.as_ref().try_into()?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Empty( protobuf::EmptyExecNode { schema: Some(schema), - partitions: self - .properties() - .output_partitioning() - .partition_count() as u32, + partitions: *partitions as u32, }, ), ), @@ -222,7 +228,10 @@ impl EmptyExec { protobuf::physical_plan_node::PhysicalPlanType::Empty, "EmptyExec", ); - let schema = empty.schema.as_ref().ok_or_else(|| { + // Destructure exhaustively so that a new field on `EmptyExecNode` is a + // compile error here rather than a silently dropped field. + let protobuf::EmptyExecNode { schema, partitions } = empty; + let schema = schema.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "EmptyExec is missing required field 'schema'" ) @@ -230,7 +239,7 @@ impl EmptyExec { let schema = Arc::new(arrow::datatypes::Schema::try_from(schema)?); // A zero (absent) partition count comes from a plan encoded before the // field existed, which always meant a single partition. - let partitions = empty.partitions.max(1) as usize; + let partitions = (*partitions).max(1) as usize; Ok(Arc::new(EmptyExec::new(schema).with_partitions(partitions))) } } diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rs index a270a003eba17..7313396a418fb 100644 --- a/datafusion/physical-plan/src/explain.rs +++ b/datafusion/physical-plan/src/explain.rs @@ -193,17 +193,26 @@ impl ExecutionPlan for ExplainExec { ) -> Result> { use datafusion_proto_models::protobuf; + // Destructure exhaustively (no `..`) so that adding a field to + // `ExplainExec` is a compile error here until it is either serialized + // or explicitly documented as not needing to be. + let Self { + schema, + stringified_plans, + verbose, + // Derived from `schema`, recomputed on decode. + cache: _, + } = self; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Explain( protobuf::ExplainExecNode { - schema: Some(self.schema().as_ref().try_into()?), - stringified_plans: self - .stringified_plans() + schema: Some(schema.as_ref().try_into()?), + stringified_plans: stringified_plans .iter() .map(stringified_plan_to_proto) .collect(), - verbose: self.verbose(), + verbose: *verbose, }, ), ), @@ -225,19 +234,25 @@ impl ExplainExec { protobuf::physical_plan_node::PhysicalPlanType::Explain, "ExplainExec", ); - let schema = explain.schema.as_ref().ok_or_else(|| { + // Destructure exhaustively so that a new field on `ExplainExecNode` is + // a compile error here rather than a silently dropped field. + let protobuf::ExplainExecNode { + schema, + stringified_plans, + verbose, + } = explain; + let schema = schema.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "ExplainExec is missing required field 'schema'" ) })?; Ok(Arc::new(ExplainExec::new( Arc::new(arrow::datatypes::Schema::try_from(schema)?), - explain - .stringified_plans + stringified_plans .iter() .map(stringified_plan_from_proto) .collect(), - explain.verbose, + *verbose, ))) } } diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 50c8246b37ce5..a57690518243d 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -824,15 +824,30 @@ impl ExecutionPlan for FilterExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; - let expr = ctx.encode_expr(self.predicate())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `FilterExec` is a compile error here until it is either serialized or + // explicitly documented as not needing to be. + let Self { + predicate, + input, + // Runtime metrics, not part of the plan shape. + metrics: _, + default_selectivity, + // Derived plan properties, recomputed on decode. + cache: _, + projection, + batch_size, + fetch, + } = self; + let input_node = ctx.encode_child(input)?; + let expr = ctx.encode_expr(predicate)?; // Preserve the exact wire format: `None` (full projection) is serialized // as the identity projection `[0, 1, ..., num_fields - 1]` so that it is // distinguishable from an explicit projection on decode. - let projection = if let Some(v) = self.projection() { + let projection = if let Some(v) = projection { v.iter().map(|x| *x as u32).collect() } else { - (0..self.input().schema().fields().len()) + (0..input.schema().fields().len()) .map(|i| i as u32) .collect() }; @@ -840,12 +855,12 @@ impl ExecutionPlan for FilterExec { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Filter(Box::new( protobuf::FilterExecNode { - input: Some(Box::new(input)), + input: Some(Box::new(input_node)), expr: Some(expr), - default_filter_selectivity: self.default_selectivity() as u32, + default_filter_selectivity: *default_selectivity as u32, projection, - batch_size: self.batch_size() as u32, - fetch: self.fetch().map(|f| f as u32), + batch_size: *batch_size as u32, + fetch: fetch.map(|f| f as u32), }, )), ), @@ -867,28 +882,37 @@ impl FilterExec { ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let filter = crate::expect_plan_variant!( + let filter_node = crate::expect_plan_variant!( node, protobuf::physical_plan_node::PhysicalPlanType::Filter, "FilterExec", ); - let input = - ctx.decode_required_child(filter.input.as_deref(), "FilterExec", "input")?; + // Destructure exhaustively so that a new field on `FilterExecNode` is a + // compile error here rather than a silently dropped field. + let protobuf::FilterExecNode { + input, + expr, + default_filter_selectivity, + projection, + batch_size, + fetch, + } = &**filter_node; + let input = ctx.decode_required_child(input.as_deref(), "FilterExec", "input")?; let predicate = ctx.decode_required_expr( - filter.expr.as_ref(), + expr.as_ref(), input.schema().as_ref(), "FilterExec", "expr", )?; - let filter_selectivity = filter.default_filter_selectivity.try_into(); + let filter_selectivity = (*default_filter_selectivity).try_into(); // `None` is encoded as the full identity projection. Reconstruct it only // when all input columns are present in order, leaving an empty list as // `Some(vec![])`. let num_fields = input.schema().fields().len(); - let mut is_full_projection = filter.projection.len() == num_fields; - let mut projection_vec: Vec = Vec::with_capacity(filter.projection.len()); - for (i, idx) in filter.projection.iter().enumerate() { + let mut is_full_projection = projection.len() == num_fields; + let mut projection_vec: Vec = Vec::with_capacity(projection.len()); + for (i, idx) in projection.iter().enumerate() { let idx = *idx as usize; is_full_projection &= idx == i; projection_vec.push(idx); @@ -900,8 +924,8 @@ impl FilterExec { }; let filter = FilterExecBuilder::new(predicate, input) .apply_projection(projection)? - .with_batch_size(filter.batch_size as usize) - .with_fetch(filter.fetch.map(|f| f as usize)) + .with_batch_size(*batch_size as usize) + .with_fetch(fetch.map(|f| f as usize)) .build()?; match filter_selectivity { Ok(filter_selectivity) => Ok(Arc::new( diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index ddce680fc18ad..f8d0cfa97809f 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -251,15 +251,31 @@ impl ExecutionPlan for GlobalLimitExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `GlobalLimitExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + skip, + fetch, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Not serialized: `GlobalLimitExecNode` has no field for it. It is + // set by the `enforce_sorting` optimizer rule, so a decoded plan + // starts with `None` as `GlobalLimitExec::new` leaves it. + required_ordering: _, + // Derived plan properties, recomputed on decode. + cache: _, + } = self; + let input = ctx.encode_child(input)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(Box::new( protobuf::GlobalLimitExecNode { input: Some(Box::new(input)), - skip: self.skip() as u32, - fetch: match self.fetch() { - Some(n) => n as i64, + skip: *skip as u32, + fetch: match fetch { + Some(n) => *n as i64, _ => -1, // no limit }, }, @@ -281,21 +297,18 @@ impl GlobalLimitExec { protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit, "GlobalLimitExec", ); - let input = ctx.decode_required_child( - limit.input.as_deref(), - "GlobalLimitExec", - "input", - )?; - let fetch = if limit.fetch >= 0 { - Some(limit.fetch as usize) + // Destructure exhaustively so that a new field on + // `GlobalLimitExecNode` is a compile error here rather than a silently + // dropped field. + let protobuf::GlobalLimitExecNode { input, skip, fetch } = &**limit; + let input = + ctx.decode_required_child(input.as_deref(), "GlobalLimitExec", "input")?; + let fetch = if *fetch >= 0 { + Some(*fetch as usize) } else { None }; - Ok(Arc::new(GlobalLimitExec::new( - input, - limit.skip as usize, - fetch, - ))) + Ok(Arc::new(GlobalLimitExec::new(input, *skip as usize, fetch))) } } @@ -479,13 +492,28 @@ impl ExecutionPlan for LocalLimitExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `LocalLimitExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + fetch, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Not serialized: `LocalLimitExecNode` has no field for it. It is + // set by the `enforce_sorting` optimizer rule, so a decoded plan + // starts with `None` as `LocalLimitExec::new` leaves it. + required_ordering: _, + // Derived plan properties, recomputed on decode. + cache: _, + } = self; + let input = ctx.encode_child(input)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(Box::new( protobuf::LocalLimitExecNode { input: Some(Box::new(input)), - fetch: self.fetch() as u32, + fetch: *fetch as u32, }, )), ), @@ -505,9 +533,12 @@ impl LocalLimitExec { protobuf::physical_plan_node::PhysicalPlanType::LocalLimit, "LocalLimitExec", ); + // Destructure exhaustively so that a new field on `LocalLimitExecNode` + // is a compile error here rather than a silently dropped field. + let protobuf::LocalLimitExecNode { input, fetch } = &**limit; let input = - ctx.decode_required_child(limit.input.as_deref(), "LocalLimitExec", "input")?; - Ok(Arc::new(LocalLimitExec::new(input, limit.fetch as usize))) + ctx.decode_required_child(input.as_deref(), "LocalLimitExec", "input")?; + Ok(Arc::new(LocalLimitExec::new(input, *fetch as usize))) } } diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs index 5d71058269f49..e7abd3e3df72b 100644 --- a/datafusion/physical-plan/src/placeholder_row.rs +++ b/datafusion/physical-plan/src/placeholder_row.rs @@ -193,16 +193,22 @@ impl ExecutionPlan for PlaceholderRowExec { _ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let schema = self.schema().as_ref().try_into()?; + // Destructure exhaustively (no `..`) so that adding a field to + // `PlaceholderRowExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + schema, + partitions, + // Derived from `schema` and `partitions`, recomputed on decode. + cache: _, + } = self; + let schema = schema.as_ref().try_into()?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow( protobuf::PlaceholderRowExecNode { schema: Some(schema), - partitions: self - .properties() - .output_partitioning() - .partition_count() as u32, + partitions: *partitions as u32, }, ), ), @@ -223,7 +229,11 @@ impl PlaceholderRowExec { protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow, "PlaceholderRowExec", ); - let schema = placeholder.schema.as_ref().ok_or_else(|| { + // Destructure exhaustively so that a new field on + // `PlaceholderRowExecNode` is a compile error here rather than a + // silently dropped field. + let protobuf::PlaceholderRowExecNode { schema, partitions } = placeholder; + let schema = schema.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "PlaceholderRowExec is missing required field 'schema'" ) @@ -231,7 +241,7 @@ impl PlaceholderRowExec { let schema = Arc::new(Schema::try_from(schema)?); // A zero (absent) partition count comes from a plan encoded before the // field existed, which always meant a single partition. - let partitions = placeholder.partitions.max(1) as usize; + let partitions = (*partitions).max(1) as usize; Ok(Arc::new( PlaceholderRowExec::new(schema).with_partitions(partitions), )) diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index fac837b09f099..64a4b6a295875 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -535,9 +535,23 @@ impl ExecutionPlan for ProjectionExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; - let expr = ctx.encode_expressions(self.expr().iter().map(|p| &p.expr))?; - let expr_name = self.expr().iter().map(|p| p.alias.clone()).collect(); + // Destructure exhaustively (no `..`) so that adding a field to + // `ProjectionExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + // The projector is rebuilt from the projection expressions and the + // input schema on decode; the expressions themselves are serialized. + projector, + input, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Derived plan properties, recomputed on decode. + cache: _, + } = self; + let projection_exprs = projector.projection().as_ref(); + let input = ctx.encode_child(input)?; + let expr = ctx.encode_expressions(projection_exprs.iter().map(|p| &p.expr))?; + let expr_name = projection_exprs.iter().map(|p| p.alias.clone()).collect(); Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Projection(Box::new( @@ -574,16 +588,19 @@ impl ProjectionExec { protobuf::physical_plan_node::PhysicalPlanType::Projection, "ProjectionExec", ); - let input = ctx.decode_required_child( - projection.input.as_deref(), - "ProjectionExec", - "input", - )?; + // Destructure exhaustively so that a new field on `ProjectionExecNode` + // is a compile error here rather than a silently dropped field. + let protobuf::ProjectionExecNode { + input, + expr, + expr_name, + } = &**projection; + let input = + ctx.decode_required_child(input.as_deref(), "ProjectionExec", "input")?; let input_schema = input.schema(); - let exprs = projection - .expr + let exprs = expr .iter() - .zip(projection.expr_name.iter()) + .zip(expr_name.iter()) .map(|(expr, name)| { Ok(ProjectionExpr { expr: ctx.decode_expr(expr, input_schema.as_ref())?, diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 873f35fd6aed9..fd577ff41b90a 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -1711,9 +1711,25 @@ impl ExecutionPlan for RepartitionExec { ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `RepartitionExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + // Execution-time channel state, created on `execute()`. + state: _, + // Runtime metrics, not part of the plan shape. + metrics: _, + preserve_order, + // Derived plan properties. The output partitioning lives here (it is + // the plan's own `partitioning`) and *is* serialized below; the rest + // is recomputed on decode. + cache, + } = self; - let partitioning = self.partitioning().try_to_proto(&ctx.expr_ctx())?; + let input = ctx.encode_child(input)?; + + let partitioning = cache.partitioning.try_to_proto(&ctx.expr_ctx())?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( @@ -1721,7 +1737,7 @@ impl ExecutionPlan for RepartitionExec { protobuf::RepartitionExecNode { input: Some(Box::new(input)), partitioning: Some(partitioning), - preserve_order: self.preserve_order(), + preserve_order: *preserve_order, }, )), ), @@ -1743,15 +1759,19 @@ impl RepartitionExec { protobuf::physical_plan_node::PhysicalPlanType::Repartition, "RepartitionExec", ); - let input = ctx.decode_required_child( - repart.input.as_deref(), - "RepartitionExec", - "input", - )?; + // Destructure exhaustively so that a new field on + // `RepartitionExecNode` is a compile error here rather than a silently + // dropped field. + let protobuf::RepartitionExecNode { + input, + partitioning, + preserve_order, + } = &**repart; + let input = + ctx.decode_required_child(input.as_deref(), "RepartitionExec", "input")?; let input_schema = input.schema(); - let partitioning = repart - .partitioning + let partitioning = partitioning .as_ref() .map(|partitioning| { Partitioning::try_from_proto( @@ -1768,7 +1788,7 @@ impl RepartitionExec { })?; let mut repart_exec = RepartitionExec::try_new(input, partitioning)?; - if repart.preserve_order { + if *preserve_order { repart_exec = repart_exec.with_preserve_order(); } Ok(Arc::new(repart_exec)) diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rs index 73acb2ab13480..d4c993751c285 100644 --- a/datafusion/physical-plan/src/scalar_subquery.rs +++ b/datafusion/physical-plan/src/scalar_subquery.rs @@ -262,10 +262,29 @@ impl ExecutionPlan for ScalarSubqueryExec { ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `ScalarSubqueryExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + subqueries, + // Execution-time one-shot future, created on `execute()`. + subquery_future: _, + // Runtime results container, rebuilt (empty) on decode and shared + // with the input's `ScalarSubqueryExpr` nodes. + results: _, + // Copied from the input's properties, recomputed on decode. + cache: _, + } = self; + let input = ctx.encode_child(input)?; // Subquery indices are positional and recovered during decoding. - let subqueries = - ctx.encode_children(self.subqueries().iter().map(|subquery| &subquery.plan))?; + let subqueries = ctx.encode_children(subqueries.iter().map( + |ScalarSubqueryLink { + plan, + // Positional: recovered from the element's position on decode. + index: _, + }| plan, + ))?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery(Box::new( @@ -293,8 +312,12 @@ impl ScalarSubqueryExec { protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery, "ScalarSubqueryExec", ); - let results = ScalarSubqueryResults::new(scalar_subquery.subqueries.len()); - let input_node = scalar_subquery.input.as_deref().ok_or_else(|| { + // Destructure exhaustively so that a new field on + // `ScalarSubqueryExecNode` is a compile error here rather than a + // silently dropped field. + let protobuf::ScalarSubqueryExecNode { input, subqueries } = &**scalar_subquery; + let results = ScalarSubqueryResults::new(subqueries.len()); + let input_node = input.as_deref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "ScalarSubqueryExec is missing required field 'input'" ) @@ -302,8 +325,7 @@ impl ScalarSubqueryExec { // The input's ScalarSubqueryExpr nodes must share this results container. let input = ctx.decode_child_with_scalar_subquery_results(input_node, results.clone())?; - let subqueries = scalar_subquery - .subqueries + let subqueries = subqueries .iter() .enumerate() .map(|(index, plan)| { diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index b1b6a84fd9c71..ecc59d28c8784 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1568,9 +1568,24 @@ impl ExecutionPlan for SortExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; - let expr = self - .expr() + // Destructure exhaustively (no `..`) so that adding a field to + // `SortExec` is a compile error here until it is either serialized or + // explicitly documented as not needing to be. + let Self { + input, + expr, + // Runtime metrics, not part of the plan shape. + metrics_set: _, + preserve_partitioning, + fetch, + // Derived from `input` and `expr` at construction time. + common_sort_prefix: _, + // Derived plan properties, recomputed on decode. + cache: _, + filter, + } = self; + let input = ctx.encode_child(input)?; + let expr = expr .iter() .map(|sort_expr| { let sort_node = Box::new(protobuf::PhysicalSortExprNode { @@ -1586,11 +1601,12 @@ impl ExecutionPlan for SortExec { }) }) .collect::>>()?; - let dynamic_filter = self - .dynamic_expressions_produced() - .into_iter() - .next() - .map(|expr| ctx.encode_expr(&expr)) + let dynamic_filter = filter + .as_ref() + .map(|filter| { + let df_expr: Arc = filter.read().expr(); + ctx.encode_expr(&df_expr) + }) .transpose()?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( @@ -1598,11 +1614,11 @@ impl ExecutionPlan for SortExec { protobuf::SortExecNode { input: Some(Box::new(input)), expr, - fetch: match self.fetch() { - Some(n) => n as i64, + fetch: match fetch { + Some(n) => *n as i64, None => -1, }, - preserve_partitioning: self.preserve_partitioning(), + preserve_partitioning: *preserve_partitioning, dynamic_filter, }, )), @@ -1624,11 +1640,18 @@ impl SortExec { protobuf::physical_plan_node::PhysicalPlanType::Sort, "SortExec", ); - let input = - ctx.decode_required_child(sort.input.as_deref(), "SortExec", "input")?; + // Destructure exhaustively so that a new field on `SortExecNode` is a + // compile error here rather than a silently dropped field. + let protobuf::SortExecNode { + input, + expr, + fetch, + preserve_partitioning, + dynamic_filter, + } = &**sort; + let input = ctx.decode_required_child(input.as_deref(), "SortExec", "input")?; let input_schema = input.schema(); - let exprs = sort - .expr + let exprs = expr .iter() .map(|expr| { let Some(ExprType::Sort(sort_expr)) = expr.expr_type.as_ref() else { @@ -1653,12 +1676,12 @@ impl SortExec { let Some(ordering) = LexOrdering::new(exprs) else { return datafusion_common::internal_err!("SortExec requires an ordering"); }; - let fetch = (sort.fetch >= 0).then_some(sort.fetch as usize); + let fetch = (*fetch >= 0).then_some(*fetch as usize); let new_sort = SortExec::new(ordering, input) .with_fetch(fetch) - .with_preserve_partitioning(sort.preserve_partitioning); + .with_preserve_partitioning(*preserve_partitioning); - let new_sort = if let Some(df_proto) = &sort.dynamic_filter { + let new_sort = if let Some(df_proto) = dynamic_filter { let df_expr = ctx.decode_expr(df_proto, new_sort.input().schema().as_ref())?; let df = (df_expr as Arc) diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index 2add3e1eb82f0..118a69fabd01c 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -443,9 +443,24 @@ impl ExecutionPlan for SortPreservingMergeExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let input = ctx.encode_child(self.input())?; - let expr = self - .expr() + // Destructure exhaustively (no `..`) so that adding a field to + // `SortPreservingMergeExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + input, + expr, + // Runtime metrics, not part of the plan shape. + metrics: _, + fetch, + // Derived plan properties, recomputed on decode. + cache: _, + // Not serialized: `SortPreservingMergeExecNode` has no field for it, + // so decoding always yields the `true` default from + // `SortPreservingMergeExec::new`. + enable_round_robin_repartition: _, + } = self; + let input = ctx.encode_child(input)?; + let expr = expr .iter() .map(|e| { Ok(protobuf::PhysicalExprNode { @@ -466,7 +481,7 @@ impl ExecutionPlan for SortPreservingMergeExec { Box::new(protobuf::SortPreservingMergeExecNode { input: Some(Box::new(input)), expr, - fetch: self.fetch().map(|f| f as i64).unwrap_or(-1), + fetch: fetch.map(|f| f as i64).unwrap_or(-1), }), ), ), @@ -488,14 +503,17 @@ impl SortPreservingMergeExec { protobuf::physical_plan_node::PhysicalPlanType::SortPreservingMerge, "SortPreservingMergeExec", ); + // Destructure exhaustively so that a new field on + // `SortPreservingMergeExecNode` is a compile error here rather than a + // silently dropped field. + let protobuf::SortPreservingMergeExecNode { input, expr, fetch } = &**spm; let input = ctx.decode_required_child( - spm.input.as_deref(), + input.as_deref(), "SortPreservingMergeExec", "input", )?; let input_schema = input.schema(); - let exprs = spm - .expr + let exprs = expr .iter() .map(|e| { let sort = match &e.expr_type { @@ -524,7 +542,7 @@ impl SortPreservingMergeExec { let Some(ordering) = LexOrdering::new(exprs) else { return internal_err!("SortPreservingMergeExec requires an ordering"); }; - let fetch = (spm.fetch >= 0).then_some(spm.fetch as usize); + let fetch = (*fetch >= 0).then_some(*fetch as usize); Ok(Arc::new( SortPreservingMergeExec::new(ordering, input).with_fetch(fetch), )) diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 8d77556509b9e..473c9a60281ea 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -558,7 +558,17 @@ impl ExecutionPlan for UnionExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let inputs = ctx.encode_children(self.inputs())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `UnionExec` is a compile error here until it is either serialized or + // explicitly documented as not needing to be. + let Self { + inputs, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Derived plan properties, recomputed on decode. + cache: _, + } = self; + let inputs = ctx.encode_children(inputs)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Union( @@ -581,8 +591,10 @@ impl UnionExec { protobuf::physical_plan_node::PhysicalPlanType::Union, "UnionExec", ); - let inputs = union - .inputs + // Destructure exhaustively so that a new field on `UnionExecNode` is a + // compile error here rather than a silently dropped field. + let protobuf::UnionExecNode { inputs } = union; + let inputs = inputs .iter() .map(|input| ctx.decode_child(input)) .collect::>>()?; @@ -805,7 +817,17 @@ impl ExecutionPlan for InterleaveExec { ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { use datafusion_proto_models::protobuf; - let inputs = ctx.encode_children(self.inputs())?; + // Destructure exhaustively (no `..`) so that adding a field to + // `InterleaveExec` is a compile error here until it is either + // serialized or explicitly documented as not needing to be. + let Self { + inputs, + // Runtime metrics, not part of the plan shape. + metrics: _, + // Derived plan properties, recomputed on decode. + cache: _, + } = self; + let inputs = ctx.encode_children(inputs)?; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some( protobuf::physical_plan_node::PhysicalPlanType::Interleave( @@ -828,8 +850,10 @@ impl InterleaveExec { protobuf::physical_plan_node::PhysicalPlanType::Interleave, "InterleaveExec", ); - let inputs = interleave - .inputs + // Destructure exhaustively so that a new field on `InterleaveExecNode` + // is a compile error here rather than a silently dropped field. + let protobuf::InterleaveExecNode { inputs } = interleave; + let inputs = inputs .iter() .map(|input| ctx.decode_child(input)) .collect::>>()?;