Skip to content

Commit b6c6475

Browse files
tdcmeehantdcmeehan
authored
feat(planner): Add MaterializedViewScanNode (#26492)
1 parent 93fc2b8 commit b6c6475

File tree

14 files changed

+832
-11
lines changed

14 files changed

+832
-11
lines changed

‎presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.facebook.presto.spi.ColumnHandle;
2222
import com.facebook.presto.spi.ColumnMetadata;
2323
import com.facebook.presto.spi.ConnectorId;
24+
import com.facebook.presto.spi.MaterializedViewDefinition;
2425
import com.facebook.presto.spi.TableHandle;
2526
import com.facebook.presto.spi.analyzer.AccessControlInfo;
2627
import com.facebook.presto.spi.analyzer.AccessControlInfoForTable;
@@ -203,6 +204,8 @@ public class Analysis
203204

204205
private final Map<QualifiedObjectName, String> materializedViews = new LinkedHashMap<>();
205206

207+
private final Map<NodeRef<Table>, MaterializedViewInfo> materializedViewInfoMap = new LinkedHashMap<>();
208+
206209
private Optional<String> expandedQuery = Optional.empty();
207210

208211
// Keeps track of the subquery we are visiting, so we have access to base query information when processing materialized view status
@@ -817,6 +820,19 @@ public boolean hasTableRegisteredForMaterializedView(Table view, Table table)
817820
return tablesForMaterializedView.containsEntry(NodeRef.of(view), table);
818821
}
819822

823+
public void setMaterializedViewInfo(Table table, MaterializedViewInfo materializedViewInfo)
824+
{
825+
requireNonNull(table, "table is null");
826+
requireNonNull(materializedViewInfo, "materializedViewInfo is null");
827+
materializedViewInfoMap.put(NodeRef.of(table), materializedViewInfo);
828+
}
829+
830+
public Optional<MaterializedViewInfo> getMaterializedViewInfo(Table table)
831+
{
832+
requireNonNull(table, "table is null");
833+
return Optional.ofNullable(materializedViewInfoMap.get(NodeRef.of(table)));
834+
}
835+
820836
public void setSampleRatio(SampledRelation relation, double ratio)
821837
{
822838
sampleRatios.put(NodeRef.of(relation), ratio);
@@ -1211,6 +1227,47 @@ public Query getQuery()
12111227
}
12121228
}
12131229

1230+
@Immutable
1231+
public static final class MaterializedViewInfo
1232+
{
1233+
private final QualifiedObjectName materializedViewName;
1234+
private final Table dataTable;
1235+
private final Query viewQuery;
1236+
private final MaterializedViewDefinition materializedViewDefinition;
1237+
1238+
public MaterializedViewInfo(
1239+
QualifiedObjectName materializedViewName,
1240+
Table dataTable,
1241+
Query viewQuery,
1242+
MaterializedViewDefinition materializedViewDefinition)
1243+
{
1244+
this.materializedViewName = requireNonNull(materializedViewName, "materializedViewName is null");
1245+
this.dataTable = requireNonNull(dataTable, "dataTable is null");
1246+
this.viewQuery = requireNonNull(viewQuery, "viewQuery is null");
1247+
this.materializedViewDefinition = requireNonNull(materializedViewDefinition, "materializedViewDefinition is null");
1248+
}
1249+
1250+
public QualifiedObjectName getMaterializedViewName()
1251+
{
1252+
return materializedViewName;
1253+
}
1254+
1255+
public Table getDataTable()
1256+
{
1257+
return dataTable;
1258+
}
1259+
1260+
public Query getViewQuery()
1261+
{
1262+
return viewQuery;
1263+
}
1264+
1265+
public MaterializedViewDefinition getMaterializedViewDefinition()
1266+
{
1267+
return materializedViewDefinition;
1268+
}
1269+
}
1270+
12141271
public static final class JoinUsingAnalysis
12151272
{
12161273
private final List<Integer> leftJoinFields;

‎presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2303,21 +2303,57 @@ private Scope processMaterializedView(
23032303
analysis.getAccessControlReferences().addMaterializedViewDefinitionReference(materializedViewName, materializedViewDefinition);
23042304

23052305
analysis.registerMaterializedViewForAnalysis(materializedViewName, materializedView, materializedViewDefinition.getOriginalSql());
2306-
String newSql = getMaterializedViewSQL(materializedView, materializedViewName, materializedViewDefinition, scope);
23072306

2308-
Query query = (Query) sqlParser.createStatement(newSql, createParsingOptions(session, warningCollector));
2309-
analysis.registerNamedQuery(materializedView, query, true);
2307+
if (isLegacyMaterializedViews(session)) {
2308+
// Legacy SQL stitching approach: create UNION query with base tables
2309+
String newSql = getMaterializedViewSQL(materializedView, materializedViewName, materializedViewDefinition, scope);
23102310

2311-
Scope queryScope = process(query, scope);
2312-
RelationType relationType = queryScope.getRelationType().withAlias(materializedViewName.getObjectName(), null);
2313-
analysis.unregisterMaterializedViewForAnalysis(materializedView);
2311+
Query query = (Query) sqlParser.createStatement(newSql, createParsingOptions(session, warningCollector));
2312+
analysis.registerNamedQuery(materializedView, query, true);
23142313

2315-
Scope accessControlScope = Scope.builder()
2316-
.withRelationType(RelationId.anonymous(), relationType)
2317-
.build();
2318-
analyzeFiltersAndMasks(materializedView, materializedViewName, accessControlScope, relationType.getAllFields());
2314+
Scope queryScope = process(query, scope);
2315+
RelationType relationType = queryScope.getRelationType().withAlias(materializedViewName.getObjectName(), null);
2316+
analysis.unregisterMaterializedViewForAnalysis(materializedView);
2317+
2318+
Scope accessControlScope = Scope.builder()
2319+
.withRelationType(RelationId.anonymous(), relationType)
2320+
.build();
2321+
analyzeFiltersAndMasks(materializedView, materializedViewName, accessControlScope, relationType.getAllFields());
23192322

2320-
return createAndAssignScope(materializedView, scope, relationType);
2323+
return createAndAssignScope(materializedView, scope, relationType);
2324+
}
2325+
else {
2326+
Query viewQuery = (Query) sqlParser.createStatement(
2327+
materializedViewDefinition.getOriginalSql(),
2328+
createParsingOptions(session, warningCollector));
2329+
2330+
QualifiedName dataTableName = QualifiedName.of(
2331+
materializedViewName.getCatalogName(),
2332+
materializedViewName.getSchemaName(),
2333+
materializedViewDefinition.getTable());
2334+
Table dataTable = new Table(dataTableName);
2335+
2336+
Analysis.MaterializedViewInfo mvInfo = new Analysis.MaterializedViewInfo(
2337+
materializedViewName,
2338+
dataTable,
2339+
viewQuery,
2340+
materializedViewDefinition);
2341+
analysis.setMaterializedViewInfo(materializedView, mvInfo);
2342+
2343+
// Process the view query to analyze base tables for access control
2344+
process(viewQuery, scope);
2345+
2346+
Scope queryScope = process(dataTable, scope);
2347+
RelationType relationType = queryScope.getRelationType().withOnlyVisibleFields().withAlias(materializedViewName.getObjectName(), null);
2348+
analysis.unregisterMaterializedViewForAnalysis(materializedView);
2349+
2350+
Scope accessControlScope = Scope.builder()
2351+
.withRelationType(RelationId.anonymous(), relationType)
2352+
.build();
2353+
analyzeFiltersAndMasks(materializedView, materializedViewName, accessControlScope, relationType.getAllFields());
2354+
2355+
return createAndAssignScope(materializedView, scope, relationType);
2356+
}
23212357
}
23222358

23232359
private String getMaterializedViewSQL(

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import com.facebook.presto.sql.planner.iterative.rule.InlineSqlFunctions;
5656
import com.facebook.presto.sql.planner.iterative.rule.LeftJoinNullFilterToSemiJoin;
5757
import com.facebook.presto.sql.planner.iterative.rule.LeftJoinWithArrayContainsToEquiJoinCondition;
58+
import com.facebook.presto.sql.planner.iterative.rule.MaterializedViewRewrite;
5859
import com.facebook.presto.sql.planner.iterative.rule.MergeDuplicateAggregation;
5960
import com.facebook.presto.sql.planner.iterative.rule.MergeFilters;
6061
import com.facebook.presto.sql.planner.iterative.rule.MergeLimitWithDistinct;
@@ -314,6 +315,13 @@ public PlanOptimizers(
314315

315316
builder.add(new LogicalCteOptimizer(metadata));
316317

318+
builder.add(new IterativeOptimizer(
319+
metadata,
320+
ruleStats,
321+
statsCalculator,
322+
estimatedExchangesCostCalculator,
323+
ImmutableSet.of(new MaterializedViewRewrite(metadata))));
324+
317325
IterativeOptimizer inlineProjections = new IterativeOptimizer(
318326
metadata,
319327
ruleStats,

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import com.facebook.presto.Session;
1717
import com.facebook.presto.SystemSessionProperties;
18+
import com.facebook.presto.common.QualifiedObjectName;
1819
import com.facebook.presto.common.predicate.TupleDomain;
1920
import com.facebook.presto.common.type.ArrayType;
2021
import com.facebook.presto.common.type.MapType;
@@ -35,6 +36,7 @@
3536
import com.facebook.presto.spi.plan.FilterNode;
3637
import com.facebook.presto.spi.plan.IntersectNode;
3738
import com.facebook.presto.spi.plan.JoinNode;
39+
import com.facebook.presto.spi.plan.MaterializedViewScanNode;
3840
import com.facebook.presto.spi.plan.PlanNode;
3941
import com.facebook.presto.spi.plan.PlanNodeIdAllocator;
4042
import com.facebook.presto.spi.plan.ProjectNode;
@@ -182,6 +184,11 @@ public RelationPlan process(Node node, @Nullable SqlPlannerContext context)
182184
@Override
183185
protected RelationPlan visitTable(Table node, SqlPlannerContext context)
184186
{
187+
Optional<Analysis.MaterializedViewInfo> materializedViewInfo = analysis.getMaterializedViewInfo(node);
188+
if (materializedViewInfo.isPresent()) {
189+
return planMaterializedView(node, materializedViewInfo.get(), context);
190+
}
191+
185192
NamedQuery namedQuery = analysis.getNamedQuery(node);
186193
Scope scope = analysis.getScope(node);
187194

@@ -304,6 +311,59 @@ private RelationPlan addColumnMasks(Table table, RelationPlan plan, SqlPlannerCo
304311
return new RelationPlan(planBuilder.getRoot(), plan.getScope(), newMappings.build());
305312
}
306313

314+
private RelationPlan planMaterializedView(Table node, Analysis.MaterializedViewInfo materializedViewInfo, SqlPlannerContext context)
315+
{
316+
RelationPlan dataTablePlan = process(materializedViewInfo.getDataTable(), context);
317+
RelationPlan viewQueryPlan = process(materializedViewInfo.getViewQuery(), context);
318+
319+
Scope scope = analysis.getScope(node);
320+
321+
QualifiedObjectName materializedViewName = materializedViewInfo.getMaterializedViewName();
322+
323+
RelationType dataTableDescriptor = dataTablePlan.getDescriptor();
324+
List<VariableReferenceExpression> dataTableVariables = dataTablePlan.getFieldMappings();
325+
List<VariableReferenceExpression> viewQueryVariables = viewQueryPlan.getFieldMappings();
326+
327+
checkArgument(
328+
dataTableVariables.size() == viewQueryVariables.size(),
329+
"Materialized view %s has mismatched field counts: data table has %s fields but view query has %s fields",
330+
materializedViewName,
331+
dataTableVariables.size(),
332+
viewQueryVariables.size());
333+
334+
ImmutableList.Builder<VariableReferenceExpression> outputVariablesBuilder = ImmutableList.builder();
335+
ImmutableMap.Builder<VariableReferenceExpression, VariableReferenceExpression> dataTableMappingsBuilder = ImmutableMap.builder();
336+
ImmutableMap.Builder<VariableReferenceExpression, VariableReferenceExpression> viewQueryMappingsBuilder = ImmutableMap.builder();
337+
338+
for (Field field : dataTableDescriptor.getVisibleFields()) {
339+
int fieldIndex = dataTableDescriptor.indexOf(field);
340+
VariableReferenceExpression dataTableVar = dataTableVariables.get(fieldIndex);
341+
VariableReferenceExpression viewQueryVar = viewQueryVariables.get(fieldIndex);
342+
343+
VariableReferenceExpression outputVar = variableAllocator.newVariable(dataTableVar);
344+
outputVariablesBuilder.add(outputVar);
345+
346+
dataTableMappingsBuilder.put(outputVar, dataTableVar);
347+
viewQueryMappingsBuilder.put(outputVar, viewQueryVar);
348+
}
349+
350+
List<VariableReferenceExpression> outputVariables = outputVariablesBuilder.build();
351+
Map<VariableReferenceExpression, VariableReferenceExpression> dataTableMappings = dataTableMappingsBuilder.build();
352+
Map<VariableReferenceExpression, VariableReferenceExpression> viewQueryMappings = viewQueryMappingsBuilder.build();
353+
354+
MaterializedViewScanNode mvScanNode = new MaterializedViewScanNode(
355+
getSourceLocation(node.getLocation()),
356+
idAllocator.getNextId(),
357+
dataTablePlan.getRoot(),
358+
viewQueryPlan.getRoot(),
359+
materializedViewName,
360+
dataTableMappings,
361+
viewQueryMappings,
362+
outputVariables);
363+
364+
return new RelationPlan(mvScanNode, scope, outputVariables);
365+
}
366+
307367
@Override
308368
protected RelationPlan visitTableFunctionInvocation(TableFunctionInvocation node, SqlPlannerContext context)
309369
{

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MaterializedViewRewrite.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.sql.planner.iterative.rule;
15+
16+
import com.facebook.presto.Session;
17+
import com.facebook.presto.matching.Captures;
18+
import com.facebook.presto.matching.Pattern;
19+
import com.facebook.presto.metadata.Metadata;
20+
import com.facebook.presto.spi.MaterializedViewStatus;
21+
import com.facebook.presto.spi.analyzer.MetadataResolver;
22+
import com.facebook.presto.spi.plan.Assignments;
23+
import com.facebook.presto.spi.plan.MaterializedViewScanNode;
24+
import com.facebook.presto.spi.plan.PlanNode;
25+
import com.facebook.presto.spi.plan.ProjectNode;
26+
import com.facebook.presto.spi.relation.VariableReferenceExpression;
27+
import com.facebook.presto.sql.planner.iterative.Rule;
28+
29+
import java.util.Map;
30+
31+
import static com.facebook.presto.SystemSessionProperties.isLegacyMaterializedViews;
32+
import static com.facebook.presto.SystemSessionProperties.isMaterializedViewDataConsistencyEnabled;
33+
import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL;
34+
import static com.facebook.presto.sql.planner.plan.Patterns.materializedViewScan;
35+
import static com.google.common.base.Preconditions.checkState;
36+
import static java.util.Objects.requireNonNull;
37+
38+
public class MaterializedViewRewrite
39+
implements Rule<MaterializedViewScanNode>
40+
{
41+
private final Metadata metadata;
42+
43+
public MaterializedViewRewrite(Metadata metadata)
44+
{
45+
this.metadata = requireNonNull(metadata, "metadata is null");
46+
}
47+
48+
@Override
49+
public Pattern<MaterializedViewScanNode> getPattern()
50+
{
51+
return materializedViewScan();
52+
}
53+
54+
@Override
55+
public Result apply(MaterializedViewScanNode node, Captures captures, Context context)
56+
{
57+
Session session = context.getSession();
58+
checkState(!isLegacyMaterializedViews(session), "Materialized view rewrite rule should not fire when legacy materialized views are enabled");
59+
checkState(isMaterializedViewDataConsistencyEnabled(session), "Materialized view rewrite rule should not fire when materialized view data consistency is disabled");
60+
61+
MetadataResolver metadataResolver = metadata.getMetadataResolver(session);
62+
63+
MaterializedViewStatus status = metadataResolver.getMaterializedViewStatus(node.getMaterializedViewName());
64+
65+
boolean useDataTable = status.isFullyMaterialized();
66+
PlanNode chosenPlan = useDataTable ? node.getDataTablePlan() : node.getViewQueryPlan();
67+
Map<VariableReferenceExpression, VariableReferenceExpression> chosenMappings =
68+
useDataTable ? node.getDataTableMappings() : node.getViewQueryMappings();
69+
70+
Assignments.Builder assignments = Assignments.builder();
71+
for (VariableReferenceExpression outputVariable : node.getOutputVariables()) {
72+
VariableReferenceExpression sourceVariable = chosenMappings.get(outputVariable);
73+
requireNonNull(sourceVariable, "No mapping found for output variable: " + outputVariable);
74+
assignments.put(outputVariable, sourceVariable);
75+
}
76+
77+
return Result.ofPlanNode(new ProjectNode(
78+
node.getSourceLocation(),
79+
context.getIdAllocator().getNextId(),
80+
chosenPlan,
81+
assignments.build(),
82+
LOCAL));
83+
}
84+
}

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.facebook.presto.spi.plan.JoinNode;
3535
import com.facebook.presto.spi.plan.LimitNode;
3636
import com.facebook.presto.spi.plan.MarkDistinctNode;
37+
import com.facebook.presto.spi.plan.MaterializedViewScanNode;
3738
import com.facebook.presto.spi.plan.PlanNode;
3839
import com.facebook.presto.spi.plan.PlanNodeIdAllocator;
3940
import com.facebook.presto.spi.plan.ProjectNode;
@@ -89,6 +90,7 @@ public class ApplyConnectorOptimization
8990
ProjectNode.class,
9091
AggregationNode.class,
9192
MarkDistinctNode.class,
93+
MaterializedViewScanNode.class,
9294
UnionNode.class,
9395
IntersectNode.class,
9496
ExceptNode.class,

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.facebook.presto.spi.plan.JoinType;
2828
import com.facebook.presto.spi.plan.LimitNode;
2929
import com.facebook.presto.spi.plan.MarkDistinctNode;
30+
import com.facebook.presto.spi.plan.MaterializedViewScanNode;
3031
import com.facebook.presto.spi.plan.OutputNode;
3132
import com.facebook.presto.spi.plan.PlanNode;
3233
import com.facebook.presto.spi.plan.ProjectNode;
@@ -134,6 +135,11 @@ public static Pattern<MarkDistinctNode> markDistinct()
134135
return typeOf(MarkDistinctNode.class);
135136
}
136137

138+
public static Pattern<MaterializedViewScanNode> materializedViewScan()
139+
{
140+
return typeOf(MaterializedViewScanNode.class);
141+
}
142+
137143
public static Pattern<OutputNode> output()
138144
{
139145
return typeOf(OutputNode.class);

0 commit comments

Comments
 (0)