Skip to content

Commit 7cb876a

Browse files
ggevayclaude
andcommitted
sql: bound INTERSECT plan size by ordering its inputs
Planning duplicates an INTERSECT's left input, so a left-deep chain doubles the plan at every level. A 35-way chain needs 2^35 HIR nodes, which exhausted environmentd's memory during planning and wedged the coordinator (incident-1160). INTERSECT is commutative, so put the cheaper input on the left. A subtree is then only duplicated when it is the smaller of the two, which bounds the plan at O(n^log2(3)) instead of O(2^n). The swap is not observable: column names come from the left scope either way, and both inputs are already cast to a common type before this point. The regression test uses a full 35-way chain because a shorter one would pass either way. The unfixed planner still returns the correct answer at 12 levels, so only a chain long enough to exhaust memory distinguishes the two. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011d7RFuATJRYwYDWjZEs2Sg
1 parent e15f613 commit 7cb876a

3 files changed

Lines changed: 36 additions & 1 deletion

File tree

‎src/sql/src/plan/hir.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1763,6 +1763,18 @@ impl HirRelationExpr {
17631763
}
17641764
}
17651765

1766+
/// The number of relation nodes in this expression.
1767+
///
1768+
/// Relations reached through scalar subqueries are included. The scalar
1769+
/// nodes themselves are not, so a large predicate over a small input still
1770+
/// counts as small. This is a structural size for comparing two
1771+
/// expressions against each other, not a cost estimate.
1772+
pub fn relation_node_count(&self) -> usize {
1773+
let mut count = 0;
1774+
self.visit_post(&mut |_| count += 1);
1775+
count
1776+
}
1777+
17661778
/// If self is a constant, return the value and the type, otherwise `None`.
17671779
pub fn as_const(&self) -> Option<(&Vec<Row>, &SqlRelationType)> {
17681780
match self {

‎src/sql/src/plan/query.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1969,8 +1969,21 @@ fn plan_set_expr(
19691969
}
19701970
SetOperator::Except => Hir::except(all, lhs, rhs),
19711971
SetOperator::Intersect => {
1972+
// Planning below duplicates whichever input ends up on the left, so a
1973+
// left-deep chain of INTERSECTs doubles the plan at every level, i.e. would be
1974+
// exponential. INTERSECT is commutative, so put the cheaper input on the left.
1975+
// A subtree is then only duplicated when it is the smaller of the two, so plan
1976+
// size obeys T(a + b) <= 2*T(a) + T(b) for input sizes a <= b. The worst case
1977+
// is balanced trees, where this solves to O(n^log2(3)) instead of O(2^n).
1978+
let (lhs, rhs) = if lhs.relation_node_count() > rhs.relation_node_count() {
1979+
(rhs, lhs)
1980+
} else {
1981+
(lhs, rhs)
1982+
};
19721983
// TODO: Let's not duplicate the left-hand expression into TWO dataflows!
1973-
// Though we believe that render() does The Right Thing (TM)
1984+
// The optimizer de-duplicates at some point, but it would be good to already
1985+
// not duplicate here.
1986+
//
19741987
// Also note that we do *not* need another threshold() at the end of the method chain
19751988
// because the right-hand side of the outer union only produces existing records,
19761989
// i.e., the record counts for differential data flow definitely remain non-negative.

‎test/sqllogictest/recursion_limit.slt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,13 @@ query I
1818
SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1;
1919
----
2020
1
21+
22+
# 35 chained INTERSECTs.
23+
# Planning duplicates an INTERSECT's left input, so a left-deep chain doubles
24+
# the plan at every level. `plan_set_expr` puts the cheaper input on the left to
25+
# keep that polynomial. Without that, this query needs 2^35 HIR nodes and
26+
# exhausts memory while planning.
27+
query I
28+
SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1 INTERSECT SELECT 1;
29+
----
30+
1

0 commit comments

Comments
 (0)