Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions datafusion/core/src/physical_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -274,15 +274,16 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>
let LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) = node else {
return Ok(Transformed::no(node));
};
let old_id = cte.id;
let (old_id, name) = (cte.id(), cte.name().to_string());
let new_id = MaterializedCteId::next();
let continuation = Arc::unwrap_or_clone(cte.continuation)
let (body, continuation) = cte.into_inputs();
let continuation = Arc::unwrap_or_clone(continuation)
.transform_down_with_subqueries(|node| match node {
// A nested occurrence of the same CTE owns the scans below
// it, and is renumbered on its own.
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
ref nested,
)) if nested.id == old_id => {
)) if nested.id() == old_id => {
Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
}
LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(mut scan))
Expand All @@ -297,11 +298,12 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>
})?
.data;
Ok(Transformed::yes(LogicalPlan::MaterializedCte(
MaterializedCtePlan::Cte(MaterializedCte {
id: new_id,
continuation: Arc::new(continuation),
..cte
}),
MaterializedCtePlan::Cte(MaterializedCte::try_new(
new_id,
name,
body,
Arc::new(continuation),
)?),
)))
})
.map(|t| Some(t.data))
Expand Down Expand Up @@ -2044,11 +2046,11 @@ impl DefaultPhysicalPlanner {
if body.boundedness().is_unbounded() {
return plan_err!(
"MATERIALIZED CTE {} reads an unbounded source, which cannot be materialized",
cte.name
cte.name()
);
}
let buffer =
Arc::new(MaterializedCteBuffer::new(cte.id.as_u64(), &cte.name));
Arc::new(MaterializedCteBuffer::new(cte.id().as_u64(), cte.name()));
Arc::new(MaterializedCteExec::new(body, continuation, buffer))
}

Expand Down
20 changes: 16 additions & 4 deletions datafusion/expr/src/logical_plan/materialized_cte.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,13 @@ pub enum MaterializedCtePlan {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct MaterializedCte {
/// Binds the scans in `continuation` to this node
pub id: MaterializedCteId,
pub(crate) id: MaterializedCteId,
/// Name of the CTE, used for display and error messages only
pub name: String,
pub(crate) name: String,
/// The body of the CTE
pub cte: Arc<LogicalPlan>,
pub(crate) cte: Arc<LogicalPlan>,
/// The query that declared the CTE and reads it
pub continuation: Arc<LogicalPlan>,
pub(crate) continuation: Arc<LogicalPlan>,
}

impl MaterializedCte {
Expand Down Expand Up @@ -151,6 +151,18 @@ impl MaterializedCte {
MaterializedCteScan::find_in(&self.continuation, self.id)
}

/// The body and the continuation, in this order, for a rule that changes
/// them in place. The schemas of the scans are not checked again.
pub fn inputs_mut(&mut self) -> (&mut Arc<LogicalPlan>, &mut Arc<LogicalPlan>) {
(&mut self.cte, &mut self.continuation)
}

/// The body and the continuation, in this order. Build a node again with
/// [`Self::try_new`].
pub fn into_inputs(self) -> (Arc<LogicalPlan>, Arc<LogicalPlan>) {
(self.cte, self.continuation)
}

/// Return a node with the same id and name, and new inputs. The schemas
/// of the scans are not checked again.
pub fn with_new_inputs(
Expand Down
10 changes: 4 additions & 6 deletions datafusion/expr/src/logical_plan/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -530,11 +530,9 @@ impl LogicalPlan {
LogicalPlan::Subquery(Subquery { subquery, .. }) => vec![subquery],
LogicalPlan::SubqueryAlias(SubqueryAlias { input, .. }) => vec![input],
LogicalPlan::Extension(extension) => extension.node.inputs(),
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
cte,
continuation,
..
})) => vec![cte, continuation],
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
vec![cte.body(), cte.continuation()]
}
LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => vec![],
LogicalPlan::Union(Union { inputs, .. }) => {
inputs.iter().map(|arc| arc.as_ref()).collect()
Expand Down Expand Up @@ -673,7 +671,7 @@ impl LogicalPlan {
}
// A materialized CTE outputs the output of its continuation.
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
cte.continuation.head_output_expr()
cte.continuation().head_output_expr()
}
LogicalPlan::Extension(_)
| LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => Ok(None),
Expand Down
21 changes: 12 additions & 9 deletions datafusion/optimizer/src/analyzer/type_coercion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,21 +128,24 @@ fn coerce_plan_and_subqueries(
// of a `NULL` column of a `UNION`. Coerce the body first, and give
// its new schema to the scans before the continuation is coerced.
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
let (id, name) = (cte.id(), cte.name().to_string());
let (body, continuation) = cte.into_inputs();
let body =
coerce_plan_and_subqueries(Arc::unwrap_or_clone(cte.cte), config)?;
coerce_plan_and_subqueries(Arc::unwrap_or_clone(body), config)?;
let continuation = with_materialized_cte_scan_schema(
Arc::unwrap_or_clone(cte.continuation),
cte.id,
Arc::unwrap_or_clone(continuation),
id,
body.schema(),
)?;
let continuation = coerce_plan_and_subqueries(continuation, config)?;
Ok(Transformed::new(
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
MaterializedCte {
cte: Arc::new(body),
continuation: Arc::new(continuation),
..cte
},
MaterializedCte::try_new(
id,
name,
Arc::new(body),
Arc::new(continuation),
)?,
)),
true,
TreeNodeRecursion::Jump,
Expand Down Expand Up @@ -173,7 +176,7 @@ fn with_materialized_cte_scan_schema(
.transform_down_with_subqueries(|node| match node {
// A nested occurrence of the same CTE owns the scans below it.
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref nested))
if nested.id == id =>
if nested.id() == id =>
{
Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
}
Expand Down
17 changes: 10 additions & 7 deletions datafusion/optimizer/src/decorrelate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -215,22 +215,25 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump))
}
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref cte))
if !cte.cte.all_out_ref_exprs().is_empty() =>
if !cte.body().all_out_ref_exprs().is_empty() =>
{
// A correlated body would have to run once per outer row.
Ok(self.stop_pull_up(plan))
}
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
// Only the continuation can hold correlated expressions. Rewrite
// it alone, so that the body shared by every scan keeps its schema.
let continuation =
Arc::unwrap_or_clone(cte.continuation).rewrite(self)?;
let (id, name) = (cte.id(), cte.name().to_string());
let (body, continuation) = cte.into_inputs();
let continuation = Arc::unwrap_or_clone(continuation).rewrite(self)?;
Ok(Transformed::new(
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
MaterializedCte {
continuation: Arc::new(continuation.data),
..cte
},
MaterializedCte::try_new(
id,
name,
body,
Arc::new(continuation.data),
)?,
)),
continuation.transformed,
TreeNodeRecursion::Jump,
Expand Down
10 changes: 3 additions & 7 deletions datafusion/optimizer/src/optimize_projections/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -528,12 +528,8 @@ fn optimize_materialized_cte(
config: &dyn OptimizerConfig,
indices: RequiredIndices,
) -> Result<Transformed<LogicalPlan>> {
let MaterializedCte {
id,
name,
cte: body,
continuation,
} = cte;
let (id, name) = (cte.id(), cte.name().to_string());
let (body, continuation) = cte.into_inputs();
// The output is the output of the continuation, so the continuation
// needs what the parent needs.
let continuation =
Expand Down Expand Up @@ -578,7 +574,7 @@ fn optimize_materialized_cte(
// A nested occurrence of the same CTE has its own body and
// owns the scans below it.
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref nested))
if nested.id == id =>
if nested.id() == id =>
{
Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
}
Expand Down
11 changes: 4 additions & 7 deletions datafusion/optimizer/src/optimizer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ use datafusion_expr::logical_plan::LogicalPlan;
use datafusion_expr::{
Aggregate, Analyze, CreateMemoryTable, CreateView, DdlStatement, Distinct,
DistinctOn, DmlStatement, Explain, Expr, Extension, Filter, Join, Limit,
MaterializedCte, MaterializedCtePlan, Projection, RecursiveQuery, Repartition, Sort,
Statement, Subquery, SubqueryAlias, Union, Unnest, Window,
MaterializedCtePlan, Projection, RecursiveQuery, Repartition, Sort, Statement,
Subquery, SubqueryAlias, Union, Unnest, Window,
};

use crate::common_subexpr_eliminate::CommonSubexprEliminate;
Expand Down Expand Up @@ -467,11 +467,8 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(
let r = f(Arc::make_mut(recursive_term))?;
s || r
}
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
cte,
continuation,
..
})) => {
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
let (cte, continuation) = cte.inputs_mut();
let b = f(Arc::make_mut(cte))?;
let c = f(Arc::make_mut(continuation))?;
b || c
Expand Down
5 changes: 3 additions & 2 deletions datafusion/optimizer/src/push_down_filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1403,9 +1403,10 @@ impl OptimizerRule for PushDownFilter {
// The output is the output of the continuation, so the filter
// moves onto the continuation. The body is shared by every scan
// of the CTE and is never filtered for one of them.
cte.continuation = Arc::new(LogicalPlan::Filter(Filter::new(
let (_, continuation) = cte.inputs_mut();
*continuation = Arc::new(LogicalPlan::Filter(Filter::new(
filter.predicate,
cte.continuation,
Arc::clone(continuation),
)));
Ok(Transformed::yes(LogicalPlan::MaterializedCte(
MaterializedCtePlan::Cte(cte),
Expand Down
7 changes: 7 additions & 0 deletions docs/source/library-user-guide/upgrading/57.0.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -168,3 +168,10 @@ schema.
has the names and types of the columns that it reads from the body. The
optimizer runs this check after each rule in debug builds, so a custom rule
that does not remap the scans now fails with an error that names the rule.

The fields of `MaterializedCte` are private. Read them with
`MaterializedCte::id`, `name`, `body` and `continuation`. Build a node with
`MaterializedCte::try_new`, which checks the scans of the CTE. A rule that
changes the inputs in place can use `MaterializedCte::inputs_mut` or
`MaterializedCte::with_new_inputs`, and `MaterializedCte::into_inputs` takes
the inputs by value.