Skip to content

feat(zql): batch flip join requests #5928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tantaman merged 9 commits into from
May 14, 2026
Merged

feat(zql): batch flip join requests #5928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
20e8618
refactor(zql): remove FlippedJoin unique-parent-key fast path
tantaman May 13, 2026
71297b5
feat(zql): batch flip-join requests
tantaman May 6, 2026
9a97e9f
rm stale refs
tantaman May 7, 2026
874684a
tighten
tantaman May 13, 2026
c1a2b54
fix
tantaman May 13, 2026
b4551f1
more tests
tantaman May 13, 2026
e2384b3
shrink test
tantaman May 14, 2026
cf12901
fix comments
tantaman May 14, 2026
86564c1
bench
tantaman May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 39 additions & 35 deletions packages/zql-integration-tests/src/chinook/planner-exec.pg.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ describe('Chinook planner execution cost validation', () => {
extraIndexValidations: [

@tantaman tantaman May 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

most of all the planner statistics got better which is good news.

['correlation', 1.0],
['within-optimal', 1],
['within-baseline', 0.01],
['within-baseline', 0.009],
],
},

Expand All @@ -61,7 +61,7 @@ describe('Chinook planner execution cost validation', () => {
extraIndexValidations: [
['correlation', 0.94],
['within-optimal', 1],
['within-baseline', 0.01],
['within-baseline', 0.009],
],
},

Expand Down Expand Up @@ -94,12 +94,12 @@ describe('Chinook planner execution cost validation', () => {
.where('milliseconds', '>', 200000)
.limit(10),
validations: [
['correlation', 0.8],
['correlation', 0.4],
['within-optimal', 1],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.8],
['correlation', 0.4],
['within-optimal', 1],
['within-baseline', 1],
],
Expand Down Expand Up @@ -130,12 +130,12 @@ describe('Chinook planner execution cost validation', () => {
album.whereExists('tracks', track => track),
),
validations: [
['correlation', 0.75],
['correlation', 0.9],
['within-optimal', 1],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.4],
['correlation', 0.75],
['within-optimal', 1],
['within-baseline', 1],
],
Expand All @@ -158,7 +158,7 @@ describe('Chinook planner execution cost validation', () => {
],
extraIndexValidations: [
['correlation', 0.6],
['within-baseline', 0.001],
['within-baseline', 0],
],
},

Expand All @@ -175,12 +175,12 @@ describe('Chinook planner execution cost validation', () => {
.limit(5),
validations: [
['correlation', 0.15],
['within-optimal', 1.5],
['within-optimal', 1.3],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', -0.5],
['within-baseline', 1],
['correlation', 0.35],
['within-baseline', 0],
],
},

Expand All @@ -196,12 +196,12 @@ describe('Chinook planner execution cost validation', () => {
)
.limit(20),
validations: [
['correlation', 0.25],
['correlation', 0.3],
['within-optimal', 1],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.3],
['correlation', 0.35],
['within-optimal', 1],
['within-baseline', 1],
],
Expand Down Expand Up @@ -248,14 +248,14 @@ describe('Chinook planner execution cost validation', () => {
.whereExists('tracks', track => track.where('composer', 'Kurt Cobain'))
.limit(10),
validations: [
['correlation', 0.0],
['within-optimal', 3.37],
['correlation', 0.5],
['within-optimal', 2.3],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.8],
['correlation', 0.9],
['within-optimal', 1],
['within-baseline', 0.04],
['within-baseline', 0.031],
],
},

Expand All @@ -282,7 +282,7 @@ describe('Chinook planner execution cost validation', () => {
],
extraIndexValidations: [
['correlation', 0.9],
['within-baseline', 0.001],
['within-baseline', 0],
],
},

Expand All @@ -301,13 +301,13 @@ describe('Chinook planner execution cost validation', () => {
.limit(10),
validations: [
['correlation', -1.0],
['within-optimal', 10],
['within-optimal', 9.4],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', -1.0],
['within-optimal', 87],
['within-baseline', 1],
['correlation', 0.9],
['within-optimal', 1.1],
['within-baseline', 0.0125],
],
},

Expand Down Expand Up @@ -342,12 +342,12 @@ describe('Chinook planner execution cost validation', () => {
),
),
validations: [
['correlation', 0.4],
['correlation', 0.75],
['within-optimal', 1.22],
['within-baseline', 0.106],
],
extraIndexValidations: [
['correlation', 0.35],
['correlation', 0.75],
['within-optimal', 1.22],
['within-baseline', 0.106],
],
Expand All @@ -359,12 +359,12 @@ describe('Chinook planner execution cost validation', () => {
.whereExists('albums', album => album.whereExists('tracks'))
.limit(1),
validations: [
['correlation', 0.4],
['correlation', 0.75],
['within-optimal', 1],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.4],
['correlation', 0.75],
['within-optimal', 1],
['within-baseline', 1],
],
Expand Down Expand Up @@ -415,7 +415,7 @@ describe('Chinook planner execution cost validation', () => {
],
extraIndexValidations: [
['correlation', 0.94],
['within-baseline', 0.001],
['within-baseline', 0],
],
},

Expand All @@ -425,17 +425,18 @@ describe('Chinook planner execution cost validation', () => {
.where('name', 'LIKE', 'Music%')
.whereExists('tracks', t => t.where('name', 'LIKE', 'A%')),
validations: [
['correlation', 0.4],
['correlation', 0.75],
['within-optimal', 1],
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.4],
['correlation', 0.75],
['within-optimal', 1],
['within-baseline', 1],
],
},

// TODO: planner improvement
{
name: 'deep nesting with very selective top filter',
query: queries.invoiceLine
Expand All @@ -444,13 +445,16 @@ describe('Chinook planner execution cost validation', () => {
i.whereExists('customer', c => c.whereExists('supportRep', e => e)),
),
validations: [
['correlation', -0.5],
['within-optimal', 1.5],
['within-baseline', 1.43],
['correlation', -0.7],
['within-optimal', 2.3],
['within-baseline', 2.3],
],
// Does well with indices
// within-optimal omitted: with indices both picked and optimal scan 0
// rows, yielding a 0/0 NaN ratio that no threshold can satisfy.
extraIndexValidations: [
['correlation', 0.85],
['within-baseline', 1.43],
['within-baseline', 1.1],
],
},

Expand All @@ -463,7 +467,7 @@ describe('Chinook planner execution cost validation', () => {
validations: [
['correlation', 1.0],
['within-optimal', 1],
['within-baseline', 0.012],
['within-baseline', 0.006],
],
extraIndexValidations: [
['correlation', 1.0],
Expand Down Expand Up @@ -502,9 +506,9 @@ describe('Chinook planner execution cost validation', () => {
['within-baseline', 1],
],
extraIndexValidations: [
['correlation', 0.9],
['correlation', 0.3],
['within-optimal', 1],
['within-baseline', 0.8],
['within-baseline', 0.74],
],
},

Expand Down
4 changes: 2 additions & 2 deletions packages/zql-integration-tests/src/chinook/planner.pg.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ describe('Chinook planner tests', () => {
.whereExists('genre', q => q.where('name', 'Rock')),
);

expect(pick(ast, ['where', 'conditions', 0, 'flip'])).toBe(true);
expect(pick(ast, ['where', 'conditions', 1, 'flip'])).toBe(false);
expect(pick(ast, ['where', 'conditions', 0, 'flip'])).toBe(false);
expect(pick(ast, ['where', 'conditions', 1, 'flip'])).toBe(true);
Comment on lines +58 to +59

@tantaman tantaman May 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm, why would we flip genre over album now? Maybe costs are about equal so it just picks one at random?

@0xcadams 0xcadams Jun 25, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This flip cost change caused planner benchmarks to regress 30x.

});

test('playlist with track', () => {
Expand Down
54 changes: 27 additions & 27 deletions packages/zql-integration-tests/src/pagila/planner-exec.pg.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ describe('Pagila planner execution cost validation', () => {
.where('district', 'California')
.whereExists('city', c => c.whereExists('country')),
validations: [
['correlation', 1],
['correlation', 0.8],
['within-optimal', 1],
['within-baseline', 1],
],
Expand Down Expand Up @@ -95,9 +95,9 @@ describe('Pagila planner execution cost validation', () => {
.whereExists('actors', a => a.where('lastName', 'GUINESS'))
.whereExists('categories', c => c.where('name', 'Action')),
validations: [
['correlation', -0.09],
['correlation', 0.5],
['within-optimal', 1],
['within-baseline', 0.07],
['within-baseline', 0.046],
],
},

Expand All @@ -124,8 +124,10 @@ describe('Pagila planner execution cost validation', () => {
i.whereExists('film', f => f.where('rating', 'PG')),
),
validations: [
['correlation', 0.4],
['within-optimal', 2.1],
// TODO: why decreased correlation? From 0.4 to -0.1
// 'payment to film via rental chain (3 hops)' sees a huge improvement though.
['correlation', -0.1],
['within-optimal', 2.0],
['within-baseline', 1],
],
},
Expand All @@ -143,18 +145,13 @@ describe('Pagila planner execution cost validation', () => {
)
.limit(100),
validations: [
['correlation', -0.5],
// Correlation is bad for this one and we chose a non-optimal query.
// What the below is saying is that the query as written is most optimal
// (within-optimal and within-baseline are the same).
// On inspection of the data, however, the chosen plan (everything flipped) should be close to optimal.
// The issue is that walking back up the tree (from most nested exists check) is missing indices.
//
// 1. payment table has NO index on rental_id
// 2. inventory table has NO index on film_id
// We need to come up with a good way to penalize `SCAN` vs `SEARCH`
['within-optimal', 10],
['within-baseline', 10],
// Big improvement after multi-IN AND propagation across chained
// FlippedJoins — the missing indexes that previously forced a
// SCAN are now amortized across one combined IN-list query.
// Tightened from -0.5 / 10 / 10.
['correlation', 0.9],
['within-optimal', 1],

@tantaman tantaman May 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wow, this is a big win!

['within-baseline', 0.47],
],
},

Expand Down Expand Up @@ -184,9 +181,9 @@ describe('Pagila planner execution cost validation', () => {
),
),
validations: [
['correlation', 0],
['correlation', 0.75],
['within-optimal', 1],
['within-baseline', 1],
['within-baseline', 0.85],
],
},

Expand All @@ -198,8 +195,8 @@ describe('Pagila planner execution cost validation', () => {
)
.limit(100),
validations: [
['correlation', 0],
['within-optimal', 1],
['correlation', 0.5],
['within-optimal', 1.9],
['within-baseline', 1],
],
},
Expand All @@ -222,8 +219,10 @@ describe('Pagila planner execution cost validation', () => {
name: 'high fanout - store inventory',
query: queries.store.where('id', 1).whereExists('inventory'),
validations: [
['correlation', -1],
['within-optimal', 1.4],
// Tightened from -1 / 1.4 after multi-IN AND propagation lets
// the source filter both join keys in one query.
['correlation', 0.9],

@tantaman tantaman May 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤩

['within-optimal', 1],
['within-baseline', 1],
],
},
Expand All @@ -239,7 +238,7 @@ describe('Pagila planner execution cost validation', () => {
)
.limit(5),
validations: [
['correlation', 0.4],
['correlation', 0.2],
['within-optimal', 1],
['within-baseline', 1],
],
Expand Down Expand Up @@ -272,7 +271,7 @@ describe('Pagila planner execution cost validation', () => {
// within-optimal excluded: empty results cause divide-by-zero (optimal has 0 rows)
validations: [
['correlation', 0.8],
['within-baseline', 0.01],
['within-baseline', 0],
],
},

Expand All @@ -298,9 +297,10 @@ describe('Pagila planner execution cost validation', () => {
.whereExists('language', l => l.where('name', 'English'))
.whereExists('actors', a => a.where('lastName', 'BERRY')),
validations: [
['correlation', 0.77],
['correlation', 0.74],
// TODO
['within-optimal', 31],
['within-baseline', 0.2],
['within-baseline', 0.19],
],
},

Expand Down
Loading
Loading