From a282293250a9278a05915cc11ee9ca1bdea83502 Mon Sep 17 00:00:00 2001 From: icetindil Date: Fri, 30 Aug 2013 02:12:37 -0700 Subject: [PATCH 01/15] changing PushAssignDownThroughProductRule to PushAssignDownThroughJoinRule --- .../rules/PushAssignDownThroughJoinRule.java | 0 .../PushAssignDownThroughProductRule.java | 87 ------------------- 2 files changed, 87 deletions(-) create mode 100644 algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java delete mode 100644 algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java new file mode 100644 index 000000000..e69de29bb diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java deleted file mode 100644 index d8523f61e..000000000 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2009-2013 by The Regents of the University of California - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * you may obtain a copy of the License from - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package edu.uci.ics.hyracks.algebricks.rewriter.rules; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang3.mutable.Mutable; - -import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; -import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; - -public class PushAssignDownThroughProductRule implements IAlgebraicRewriteRule { - - @Override - public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { - return false; - } - - @Override - public boolean rewritePost(Mutable opRef, IOptimizationContext context) - throws AlgebricksException { - AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); - if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN) { - return false; - } - Mutable op2Ref = op1.getInputs().get(0); - AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue(); - if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) { - return false; - } - AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2; - if (join.getCondition().getValue() != ConstantExpression.TRUE) { - return false; - } - - List used = new ArrayList(); - VariableUtilities.getUsedVariables(op1, used); - - Mutable b0Ref = op2.getInputs().get(0); - ILogicalOperator b0 = b0Ref.getValue(); - List b0Scm = new ArrayList(); - VariableUtilities.getLiveVariables(b0, b0Scm); - if (b0Scm.containsAll(used)) { - // push assign on left branch - op2Ref.setValue(b0); - b0Ref.setValue(op1); - opRef.setValue(op2); - return true; - } else { - Mutable b1Ref = op2.getInputs().get(1); - ILogicalOperator b1 = b1Ref.getValue(); - List b1Scm = new ArrayList(); - VariableUtilities.getLiveVariables(b1, b1Scm); - if (b1Scm.containsAll(used)) { - // push assign on right branch - op2Ref.setValue(b1); - b1Ref.setValue(op1); - opRef.setValue(op2); - return true; - } else { - return false; - } - } - } - -} From bb6ccbdfba58f97732c760bb022f36de4b3f3104 Mon Sep 17 00:00:00 2001 From: icetindil Date: Fri, 30 Aug 2013 02:59:22 -0700 Subject: [PATCH 02/15] fix for issue #624 --- .../rules/PushAssignDownThroughJoinRule.java | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java index e69de29bb..47975abb1 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java @@ -0,0 +1,87 @@ +/* + * Copyright 2009-2013 by The Regents of the University of California + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License from + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.uci.ics.hyracks.algebricks.rewriter.rules; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.mutable.Mutable; + +import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; +import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression; +import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator; +import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; +import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; +import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; + +public class PushAssignDownThroughJoinRule implements IAlgebraicRewriteRule { + + @Override + public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { + return false; + } + + @Override + public boolean rewritePost(Mutable opRef, IOptimizationContext context) + throws AlgebricksException { + AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); + if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN) { + return false; + } + Mutable op2Ref = op1.getInputs().get(0); + AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue(); + if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) { + return false; + } +// AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2; +// if (join.getCondition().getValue() != ConstantExpression.TRUE) { +// return false; +// } + + List used = new ArrayList(); + VariableUtilities.getUsedVariables(op1, used); + + Mutable b0Ref = op2.getInputs().get(0); + ILogicalOperator b0 = b0Ref.getValue(); + List b0Scm = new ArrayList(); + VariableUtilities.getLiveVariables(b0, b0Scm); + if (b0Scm.containsAll(used)) { + // push assign on left branch + op2Ref.setValue(b0); + b0Ref.setValue(op1); + opRef.setValue(op2); + return true; + } else { + Mutable b1Ref = op2.getInputs().get(1); + ILogicalOperator b1 = b1Ref.getValue(); + List b1Scm = new ArrayList(); + VariableUtilities.getLiveVariables(b1, b1Scm); + if (b1Scm.containsAll(used)) { + // push assign on right branch + op2Ref.setValue(b1); + b1Ref.setValue(op1); + opRef.setValue(op2); + return true; + } else { + return false; + } + } + } + +} From b1cf65d616ac7a86d9b54a415568d7f613952c2e Mon Sep 17 00:00:00 2001 From: icetindil Date: Tue, 3 Dec 2013 05:40:55 -0800 Subject: [PATCH 03/15] fixing primary key bug --- .../rewriter/rules/IntroduceGroupByForSubplanRule.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java index b918fc5aa..36da84803 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java @@ -240,9 +240,11 @@ protected Set computeGbyVars(AbstractLogicalOperator op, Set all = new ArrayList(); VariableUtilities.getLiveVariables(op, all); all.retainAll(freeVars); - for (FunctionalDependency fd : fdList) { - if (fd.getTail().containsAll(all)) { - return new HashSet(fd.getHead()); + if (!all.isEmpty()) { + for (FunctionalDependency fd : fdList) { + if (fd.getTail().containsAll(all)) { + return new HashSet(fd.getHead()); + } } } return null; From 3833d74a88aa7ff205670e6576322fb605fde28c Mon Sep 17 00:00:00 2001 From: icetindil Date: Mon, 10 Mar 2014 12:47:43 -0700 Subject: [PATCH 04/15] fixing a bug in unnest to product rule --- .../algebricks/rewriter/rules/ComplexUnnestToProductRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java index 5e6bcae32..910a8f9a4 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java @@ -264,7 +264,7 @@ private boolean findPlanPartition(AbstractLogicalOperator op, HashSet Date: Tue, 24 Jun 2014 04:00:28 -0700 Subject: [PATCH 05/15] modifying replicate operator to materialize dependent outputs --- .../core/algebra/base/ILogicalOperator.java | 5 + .../logical/AbstractBinaryJoinOperator.java | 13 ++ .../logical/AbstractLogicalOperator.java | 17 ++ .../operators/logical/GroupByOperator.java | 24 +++ .../operators/logical/OrderOperator.java | 12 ++ .../operators/logical/ReplicateOperator.java | 49 +++++ .../physical/ReplicatePOperator.java | 3 +- .../core/jobgen/impl/JobGenContext.java | 1 - .../core/jobgen/impl/PlanCompiler.java | 28 ++- .../rules/ExtractCommonOperatorsRule.java | 187 ++++++++++++------ .../std/base/AbstractOperatorDescriptor.java | 1 - .../std/misc/MaterializerTaskState.java | 82 ++++++++ .../misc/MaterializingOperatorDescriptor.java | 64 +----- .../std/misc/SplitOperatorDescriptor.java | 168 +++++++++++++--- 14 files changed, 493 insertions(+), 161 deletions(-) create mode 100644 hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java index d91cac057..061bfcb21 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; @@ -45,6 +46,10 @@ public interface ILogicalOperator { public List getSchema(); + public Pair getInputOutputDependencyLabels(); + + public boolean isBlocker(); + /* * * support for visitors diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java index 6c00412f9..326ec246e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; @@ -75,4 +76,16 @@ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform vi public boolean isMap() { return false; } + + @Override + public Pair getInputOutputDependencyLabels() { + int[] inputDependencyLabels = new int[] { 1, 0 }; + int[] outputDependencyLabels = new int[] { 1 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } + + @Override + public boolean isBlocker() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java index 78efd0db8..b9e626c5b 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; @@ -133,6 +134,22 @@ public final boolean hasInputs() { return !inputs.isEmpty(); } + /** + * @return labels (0 or 1) for each input and output indicating the dependency between them. + * The edges labeled as 1 must wait for the edges with label 0. + */ + @Override + public Pair getInputOutputDependencyLabels() { + int[] inputDependencyLabels = new int[inputs.size()]; // filled with 0's + int[] outputDependencyLabels = new int[] { 0 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } + + @Override + public boolean isBlocker() { + return false; + } + public boolean hasNestedPlans() { return false; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java index f738b506d..514da1fd3 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java @@ -29,6 +29,7 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy; @@ -274,4 +275,27 @@ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) return env; } + @Override + public Pair getInputOutputDependencyLabels() { + int[] inputDependencyLabels = new int[] { 0 }; + int[] outputDependencyLabels; + PhysicalOperatorTag physicalOpTag = this.getPhysicalOperator().getOperatorTag(); + if (physicalOpTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY + || physicalOpTag == PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY) { + outputDependencyLabels = new int[] { 0 }; + } else { + outputDependencyLabels = new int[] { 1 }; + } + return new Pair(inputDependencyLabels, outputDependencyLabels); + } + + @Override + public boolean isBlocker() { + PhysicalOperatorTag physicalOpTag = this.getPhysicalOperator().getOperatorTag(); + if (physicalOpTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY + || physicalOpTag == PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY) { + return false; + } + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java index 335fb36ce..eb5abdf9a 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java @@ -156,4 +156,16 @@ public boolean isMap() { public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); } + + @Override + public Pair getInputOutputDependencyLabels() { + int[] inputDependencyLabels = new int[] { 0 }; + int[] outputDependencyLabels = new int[] { 1 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } + + @Override + public boolean isBlocker() { + return true; + } } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java index 6dbd3f505..552fb6ccd 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java @@ -15,8 +15,13 @@ package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical; import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; @@ -28,9 +33,19 @@ public class ReplicateOperator extends AbstractLogicalOperator { private int outputArity = 2; + private boolean[] outputMaterializationFlags = new boolean[outputArity]; + private List> outputs; public ReplicateOperator(int outputArity) { this.outputArity = outputArity; + this.outputMaterializationFlags = new boolean[outputArity]; + this.outputs = new ArrayList>(); + } + + public ReplicateOperator(int outputArity, boolean[] outputMaterializationFlags) { + this.outputArity = outputArity; + this.outputMaterializationFlags = outputMaterializationFlags; + this.outputs = new ArrayList>(); } @Override @@ -71,9 +86,43 @@ public int getOutputArity() { return outputArity; } + public int setOutputArity(int outputArity) { + return this.outputArity = outputArity; + } + + public boolean[] getOutputMaterializationFlags() { + return outputMaterializationFlags; + } + + public List> getOutputs() { + return outputs; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); } + @Override + public Pair getInputOutputDependencyLabels() { + int[] inputDependencyLabels = new int[] { 0 }; + int[] outputDependencyLabels = new int[outputArity]; + // change the labels of outputs that requires materialization to 1 + for (int i = 0; i < outputArity; i++) { + if (outputMaterializationFlags[i]) { + outputDependencyLabels[i] = 1; + } + } + return new Pair(inputDependencyLabels, outputDependencyLabels); + } + + @Override + public boolean isBlocker() { + for (boolean requiresMaterialization : outputMaterializationFlags) { + if (requiresMaterialization) { + return true; + } + } + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java index c00e8f5e8..daee6706e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java @@ -64,8 +64,9 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext ReplicateOperator rop = (ReplicateOperator) op; int outputArity = rop.getOutputArity(); + boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags(); - SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity); + SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity, outputMaterializationFlags); contributeOpDesc(builder, (AbstractLogicalOperator) op, splitOpDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java index 4ae956ac3..e1de49525 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java @@ -41,7 +41,6 @@ import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider; import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider; import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory; -import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactory; import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider; public class JobGenContext { diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java index 961e5270b..3af57add3 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java @@ -26,7 +26,9 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; +import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator; import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory; import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry; import edu.uci.ics.hyracks.api.job.JobSpecification; @@ -43,7 +45,8 @@ public JobGenContext getContext() { return context; } - public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { + public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema, + IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { JobSpecification spec = new JobSpecification(context.getFrameSize()); if (jobEventListenerFactory != null) { spec.setJobletEventListenerFactory(jobEventListenerFactory); @@ -63,8 +66,8 @@ public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlan return spec; } - private void compileOpRef(Mutable opRef, IOperatorDescriptorRegistry spec, IHyracksJobBuilder builder, - IOperatorSchema outerPlanSchema) throws AlgebricksException { + private void compileOpRef(Mutable opRef, IOperatorDescriptorRegistry spec, + IHyracksJobBuilder builder, IOperatorSchema outerPlanSchema) throws AlgebricksException { ILogicalOperator op = opRef.getValue(); int n = op.getInputs().size(); IOperatorSchema[] schemas = new IOperatorSchema[n]; @@ -100,10 +103,21 @@ private void reviseEdges(IHyracksJobBuilder builder) { Mutable child = entry.getKey(); List> parents = entry.getValue(); if (parents.size() > 1) { - int i = 0; - for (Mutable parent : parents) { - builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0); - i++; + if (child.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) { + ReplicateOperator rop = (ReplicateOperator) child.getValue(); + if (rop.isBlocker()) { + // make the order of the graph edges consistent with the order of rop's outputs + List> outputs = rop.getOutputs(); + for (Mutable parent : parents) { + builder.contributeGraphEdge(child.getValue(), outputs.indexOf(parent), parent.getValue(), 0); + } + } else { + int i = 0; + for (Mutable parent : parents) { + builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0); + i++; + } + } } } } diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index 06b3c06b6..2ac511a6a 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -24,6 +25,7 @@ import org.apache.commons.lang3.mutable.MutableObject; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; @@ -48,8 +50,21 @@ public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule { private HashMap, List>> childrenToParents = new HashMap, List>>(); private List> roots = new ArrayList>(); - private List> joins = new ArrayList>(); private List>> equivalenceClasses = new ArrayList>>(); + private static HashSet opsWorthMaterialization = new HashSet(); + static { + opsWorthMaterialization.add(LogicalOperatorTag.SELECT); + opsWorthMaterialization.add(LogicalOperatorTag.INNERJOIN); + opsWorthMaterialization.add(LogicalOperatorTag.LEFTOUTERJOIN); + opsWorthMaterialization.add(LogicalOperatorTag.PROJECT); + opsWorthMaterialization.add(LogicalOperatorTag.GROUP); + opsWorthMaterialization.add(LogicalOperatorTag.ORDER); + opsWorthMaterialization.add(LogicalOperatorTag.AGGREGATE); + opsWorthMaterialization.add(LogicalOperatorTag.RUNNINGAGGREGATE); + opsWorthMaterialization.add(LogicalOperatorTag.DISTINCT); + opsWorthMaterialization.add(LogicalOperatorTag.LIMIT); + opsWorthMaterialization.add(LogicalOperatorTag.UNNEST_MAP); + } @Override public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { @@ -78,17 +93,14 @@ public boolean rewritePost(Mutable opRef, IOptimizationContext changed = false; // applying the rewriting until fixpoint topDownMaterialization(roots); - removeNonJoinBuildBranchCandidates(); genCandidates(context); removeTrivialShare(); - removeNonJoinBuildBranchCandidates(); if (equivalenceClasses.size() > 0) changed = rewrite(context); if (!rewritten) rewritten = changed; equivalenceClasses.clear(); childrenToParents.clear(); - joins.clear(); } while (changed); roots.clear(); } @@ -111,41 +123,6 @@ private void removeTrivialShare() { equivalenceClasses.remove(i); } - private void removeNonJoinBuildBranchCandidates() { - for (List> candidates : equivalenceClasses) { - for (int i = candidates.size() - 1; i >= 0; i--) { - Mutable opRef = candidates.get(i); - boolean reserve = false; - for (Mutable join : joins) - if (isInJoinBuildBranch(join, opRef)) { - reserve = true; - } - if (!reserve) - candidates.remove(i); - } - } - for (int i = equivalenceClasses.size() - 1; i >= 0; i--) - if (equivalenceClasses.get(i).size() < 2) - equivalenceClasses.remove(i); - } - - private boolean isInJoinBuildBranch(Mutable joinRef, Mutable opRef) { - Mutable buildBranch = joinRef.getValue().getInputs().get(1); - do { - if (buildBranch.equals(opRef)) { - return true; - } else { - AbstractLogicalOperator aop = (AbstractLogicalOperator) buildBranch.getValue(); - if (aop.getOperatorTag() == LogicalOperatorTag.INNERJOIN - || aop.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN - || buildBranch.getValue().getInputs().size() == 0) - return false; - else - buildBranch = buildBranch.getValue().getInputs().get(0); - } - } while (true); - } - private boolean rewrite(IOptimizationContext context) throws AlgebricksException { boolean changed = false; for (List> members : equivalenceClasses) { @@ -170,40 +147,47 @@ private boolean rewriteForOneEquivalentClass(List> mem members.remove(i); } } - AbstractLogicalOperator rop = new ReplicateOperator(group.size()); + boolean[] materializationFlags = computeMaterilizationFlags(group); + if (group.isEmpty()) { + continue; + } + candidate = group.get(0); + ReplicateOperator rop = new ReplicateOperator(group.size(), materializationFlags); rop.setPhysicalOperator(new ReplicatePOperator()); rop.setExecutionMode(ExecutionMode.PARTITIONED); Mutable ropRef = new MutableObject(rop); AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getValue(); + List> originalCandidateParents = childrenToParents.get(candidate); if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) { rop.getInputs().add(candidate); } else { AbstractLogicalOperator beforeExchange = new ExchangeOperator(); beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator()); + Mutable beforeExchangeRef = new MutableObject(beforeExchange); beforeExchange.getInputs().add(candidate); context.computeAndSetTypeEnvironmentForOperator(beforeExchange); - rop.getInputs().add(new MutableObject(beforeExchange)); + rop.getInputs().add(beforeExchangeRef); } context.computeAndSetTypeEnvironmentForOperator(rop); - List> parents = childrenToParents.get(candidate); - for (Mutable parentRef : parents) { + for (Mutable parentRef : originalCandidateParents) { AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getValue(); int index = parent.getInputs().indexOf(candidate); if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) { parent.getInputs().set(index, ropRef); + rop.getOutputs().add(parentRef); } else { AbstractLogicalOperator exchange = new ExchangeOperator(); exchange.setPhysicalOperator(new OneToOneExchangePOperator()); + MutableObject exchangeRef = new MutableObject(exchange); exchange.getInputs().add(ropRef); + rop.getOutputs().add(exchangeRef); context.computeAndSetTypeEnvironmentForOperator(exchange); - // parent.getInputs().get(index).setValue(exchange); - parent.getInputs().set(index, new MutableObject(exchange)); + parent.getInputs().set(index, exchangeRef); context.computeAndSetTypeEnvironmentForOperator(parent); } } - List liveVarsNew = new ArrayList(); VariableUtilities.getLiveVariables(candidate.getValue(), liveVarsNew); ArrayList> assignExprs = new ArrayList>(); @@ -226,9 +210,11 @@ private boolean rewriteForOneEquivalentClass(List> mem AbstractLogicalOperator exchOp = new ExchangeOperator(); exchOp.setPhysicalOperator(new OneToOneExchangePOperator()); exchOp.getInputs().add(ropRef); - - assignOperator.getInputs().add(new MutableObject(exchOp)); + MutableObject exchOpRef = new MutableObject(exchOp); + rop.getOutputs().add(exchOpRef); + assignOperator.getInputs().add(exchOpRef); projectOperator.getInputs().add(new MutableObject(assignOperator)); + // set the types context.computeAndSetTypeEnvironmentForOperator(exchOp); context.computeAndSetTypeEnvironmentForOperator(assignOperator); @@ -247,18 +233,18 @@ private boolean rewriteForOneEquivalentClass(List> mem } } - AbstractLogicalOperator exchg = new ExchangeOperator(); - exchg.setPhysicalOperator(new OneToOneExchangePOperator()); - ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator : projectOperator; if (parentOp.isMap()) { parentOp.getInputs().set(index, new MutableObject(childOp)); } else { + AbstractLogicalOperator exchg = new ExchangeOperator(); + exchg.setPhysicalOperator(new OneToOneExchangePOperator()); exchg.getInputs().add(new MutableObject(childOp)); parentOp.getInputs().set(index, new MutableObject(exchg)); + context.computeAndSetTypeEnvironmentForOperator(exchg); } - context.computeAndSetTypeEnvironmentForOperator(exchg); + context.computeAndSetTypeEnvironmentForOperator(parentOp); } } rewritten = true; @@ -302,11 +288,6 @@ private void topDownMaterialization(List> tops) { List> candidates = new ArrayList>(); List> nextLevel = new ArrayList>(); for (Mutable op : tops) { - AbstractLogicalOperator aop = (AbstractLogicalOperator) op.getValue(); - if ((aop.getOperatorTag() == LogicalOperatorTag.INNERJOIN || aop.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) - && !joins.contains(op)) { - joins.add(op); - } for (Mutable opRef : op.getValue().getInputs()) { List> opRefList = childrenToParents.get(opRef); if (opRefList == null) { @@ -390,4 +371,96 @@ private void prune(IOptimizationContext context) throws AlgebricksException { } } + private boolean[] computeMaterilizationFlags(List> group) { + List>> clusters = new ArrayList>>(); + List> blockedOps = new ArrayList>(); + for (Mutable opRef : group) { + List> cluster = new ArrayList>(); + computeDependency(null, opRef, blockedOps, cluster, false); + clusters.add(cluster); + } + boolean[] materializationFlags = new boolean[group.size()]; + boolean worthMaterialization = worthMaterialization(group.get(0)); + boolean requiresMaterialization; + for (int i = group.size() - 1; i >= 0; i--) { + requiresMaterialization = requiresMaterialization(clusters.get(i), blockedOps); + if (requiresMaterialization && !worthMaterialization) { + group.remove(i); + } + materializationFlags[i] = requiresMaterialization; + } + if (group.size() < 2) { + group.clear(); + } + // if does not worth materialization, the flags for the remaining candidates should be false + return worthMaterialization ? materializationFlags : new boolean[group.size()]; + } + + private void computeDependency(Mutable inputRef, Mutable opRef, + List> blockedOps, List> cluster, boolean foundBlocker) { + List> parentRefList = childrenToParents.get(opRef); + if (parentRefList != null) { + Pair labels = null; + int inputIndex = 0; + if (opRef.getValue().isBlocker() && inputRef != null) { + labels = opRef.getValue().getInputOutputDependencyLabels(); + List> inputs = opRef.getValue().getInputs(); + for (inputIndex = 0; inputIndex < inputs.size(); inputIndex++) { + if (inputs.get(inputIndex).equals(inputRef)) { + break; + } + } + } + for (Mutable parentRef : parentRefList) { + boolean foundBlockerForCurPath = foundBlocker; + if (opRef.getValue().isBlocker() && inputRef != null) { + // only replicate operator has multiple outputs + int outputIndex = 0; + if (opRef.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) { + ReplicateOperator rop = (ReplicateOperator) opRef.getValue(); + List> outputs = rop.getOutputs(); + for (outputIndex = 0; outputIndex < outputs.size(); outputIndex++) { + if (outputs.get(outputIndex).equals(parentRef)) { + break; + } + } + } + if (labels.second[outputIndex] == 1) { + if (labels.first[inputIndex] == 1) { // 1 -> 1 + if (!foundBlocker) { + cluster.add(opRef); + } + } else { // 0 -> 1 + blockedOps.add(opRef); + foundBlockerForCurPath = true; + } + } + } + computeDependency(opRef, parentRef, blockedOps, cluster, foundBlockerForCurPath); + } + } + } + + private boolean requiresMaterialization(List> cluster, + List> blockedOps) { + for (Mutable op : cluster) { + if (blockedOps.contains(op)) { + return true; + } + } + return false; + } + + private boolean worthMaterialization(Mutable candidate) { + if (opsWorthMaterialization.contains(candidate.getValue().getOperatorTag())) { + return true; + } + List> inputs = candidate.getValue().getInputs(); + for (Mutable inputRef : inputs) { + if (worthMaterialization(inputRef)) { + return true; + } + } + return false; + } } diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java index f3f8a2f95..700dec822 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java @@ -23,7 +23,6 @@ import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId; import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor; import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry; -import edu.uci.ics.hyracks.api.job.JobActivityGraph; public abstract class AbstractOperatorDescriptor implements IOperatorDescriptor { private static final long serialVersionUID = 1L; diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java new file mode 100644 index 000000000..02eb72a6e --- /dev/null +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java @@ -0,0 +1,82 @@ +/* + * Copyright 2009-2013 by The Regents of the University of California + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License from + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.uci.ics.hyracks.dataflow.std.misc; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; + +import edu.uci.ics.hyracks.api.comm.IFrameWriter; +import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; +import edu.uci.ics.hyracks.api.dataflow.TaskId; +import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; +import edu.uci.ics.hyracks.api.io.FileReference; +import edu.uci.ics.hyracks.api.job.JobId; +import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader; +import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter; +import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject; + +public class MaterializerTaskState extends AbstractStateObject { + private RunFileWriter out; + + public MaterializerTaskState(JobId jobId, TaskId taskId) { + super(jobId, taskId); + } + + @Override + public void toBytes(DataOutput out) throws IOException { + + } + + @Override + public void fromBytes(DataInput in) throws IOException { + + } + + public void open(IHyracksTaskContext ctx) throws HyracksDataException { + FileReference file = ctx.getJobletContext().createManagedWorkspaceFile( + MaterializerTaskState.class.getSimpleName()); + out = new RunFileWriter(file, ctx.getIOManager()); + out.open(); + } + + public void close() throws HyracksDataException { + out.close(); + } + + public void appendFrame(ByteBuffer buffer) throws HyracksDataException { + out.nextFrame(buffer); + } + + public void writeOut(IFrameWriter writer, ByteBuffer frame) throws HyracksDataException { + RunFileReader in = out.createReader(); + writer.open(); + try { + in.open(); + while (in.nextFrame(frame)) { + frame.flip(); + writer.nextFrame(frame); + frame.clear(); + } + in.close(); + } catch (Exception e) { + writer.fail(); + throw new HyracksDataException(e); + } finally { + writer.close(); + } + } +} \ No newline at end of file diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java index 89c20d6c3..3a405d059 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java @@ -14,12 +14,8 @@ */ package edu.uci.ics.hyracks.dataflow.std.misc; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.nio.ByteBuffer; -import edu.uci.ics.hyracks.api.comm.IFrameWriter; import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; import edu.uci.ics.hyracks.api.dataflow.ActivityId; import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder; @@ -28,14 +24,9 @@ import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider; import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor; import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; -import edu.uci.ics.hyracks.api.io.FileReference; import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry; -import edu.uci.ics.hyracks.api.job.JobId; -import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader; -import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter; import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode; import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor; -import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject; import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable; import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable; @@ -84,57 +75,6 @@ public void contributeActivities(IActivityGraphBuilder builder) { } - public static class MaterializerTaskState extends AbstractStateObject { - private RunFileWriter out; - - public MaterializerTaskState() { - } - - private MaterializerTaskState(JobId jobId, TaskId taskId) { - super(jobId, taskId); - } - - @Override - public void toBytes(DataOutput out) throws IOException { - - } - - @Override - public void fromBytes(DataInput in) throws IOException { - - } - - public void open(IHyracksTaskContext ctx) throws HyracksDataException { - FileReference file = ctx.getJobletContext().createManagedWorkspaceFile( - MaterializingOperatorDescriptor.class.getSimpleName()); - out = new RunFileWriter(file, ctx.getIOManager()); - out.open(); - } - - public void appendFrame(ByteBuffer buffer) throws HyracksDataException { - out.nextFrame(buffer); - } - - public void writeOut(IFrameWriter writer, ByteBuffer frame) throws HyracksDataException { - RunFileReader in = out.createReader(); - writer.open(); - try { - in.open(); - while (in.nextFrame(frame)) { - frame.flip(); - writer.nextFrame(frame); - frame.clear(); - } - in.close(); - } catch (Exception e) { - writer.fail(); - throw new HyracksDataException(e); - } finally { - writer.close(); - } - } - } - private final class MaterializerReaderActivityNode extends AbstractActivityNode { private static final long serialVersionUID = 1L; @@ -166,7 +106,7 @@ public void fail() throws HyracksDataException { @Override public void close() throws HyracksDataException { - state.out.close(); + state.close(); ByteBuffer frame = ctx.allocateFrame(); state.writeOut(writer, frame); } @@ -202,7 +142,7 @@ public void nextFrame(ByteBuffer buffer) throws HyracksDataException { @Override public void close() throws HyracksDataException { - state.out.close(); + state.close(); ctx.setStateObject(state); } diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java index 0e8f30301..990dfb832 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java @@ -18,64 +18,168 @@ import edu.uci.ics.hyracks.api.comm.IFrameWriter; import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; +import edu.uci.ics.hyracks.api.dataflow.ActivityId; +import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder; import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable; +import edu.uci.ics.hyracks.api.dataflow.TaskId; import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider; import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor; import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry; import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils; -import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; +import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode; +import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor; import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputOperatorNodePushable; +import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable; -public class SplitOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { +public class SplitOperatorDescriptor extends AbstractOperatorDescriptor { private static final long serialVersionUID = 1L; + private final static int SPLITTER_MATERIALIZER_ACTIVITY_ID = 0; + private final static int MATERIALIZE_READER_ACTIVITY_ID = 1; + + private boolean[] outputMaterializationFlags; + private boolean requiresMaterialization; + private int numberOfNonMaterializedOutputs = 0; + public SplitOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc, int outputArity) { + this(spec, rDesc, outputArity, new boolean[outputArity]); + } + + public SplitOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc, int outputArity, + boolean[] outputMaterializationFlags) { super(spec, 1, outputArity); for (int i = 0; i < outputArity; i++) { recordDescriptors[i] = rDesc; } + this.outputMaterializationFlags = outputMaterializationFlags; + requiresMaterialization = false; + for (boolean flag : outputMaterializationFlags) { + if (flag) { + requiresMaterialization = true; + break; + } + } + } @Override - public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, - final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) - throws HyracksDataException { - return new AbstractUnaryInputOperatorNodePushable() { - private final IFrameWriter[] writers = new IFrameWriter[outputArity]; - - @Override - public void close() throws HyracksDataException { - for (IFrameWriter writer : writers) { - writer.close(); - } + public void contributeActivities(IActivityGraphBuilder builder) { + SplitterMaterializerActivityNode sma = new SplitterMaterializerActivityNode(new ActivityId(odId, + SPLITTER_MATERIALIZER_ACTIVITY_ID)); + builder.addActivity(this, sma); + builder.addSourceEdge(0, sma, 0); + int taskOutputIndex = 0; + for (int i = 0; i < outputArity; i++) { + if (!outputMaterializationFlags[i]) { + builder.addTargetEdge(i, sma, taskOutputIndex); + taskOutputIndex++; } + } + numberOfNonMaterializedOutputs = taskOutputIndex; - @Override - public void fail() throws HyracksDataException { - for (IFrameWriter writer : writers) { - writer.fail(); + if (requiresMaterialization) { + int activityId = MATERIALIZE_READER_ACTIVITY_ID; + for (int i = 0; i < outputArity; i++) { + if (outputMaterializationFlags[i]) { + MaterializeReaderActivityNode mra = new MaterializeReaderActivityNode(new ActivityId(odId, + activityId)); + builder.addActivity(this, mra); + builder.addTargetEdge(i, mra, 0); + builder.addBlockingEdge(sma, mra); + activityId++; } } + } + } - @Override - public void nextFrame(ByteBuffer bufferAccessor) throws HyracksDataException { - for (IFrameWriter writer : writers) { - FrameUtils.flushFrame(bufferAccessor, writer); + private final class SplitterMaterializerActivityNode extends AbstractActivityNode { + private static final long serialVersionUID = 1L; + + public SplitterMaterializerActivityNode(ActivityId id) { + super(id); + } + + @Override + public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, + IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) { + return new AbstractUnaryInputOperatorNodePushable() { + private MaterializerTaskState state; + private final IFrameWriter[] writers = new IFrameWriter[numberOfNonMaterializedOutputs]; + + @Override + public void open() throws HyracksDataException { + if (requiresMaterialization) { + state = new MaterializerTaskState(ctx.getJobletContext().getJobId(), new TaskId( + getActivityId(), partition)); + state.open(ctx); + } + for (int i = 0; i < numberOfNonMaterializedOutputs; i++) { + writers[i].open(); + } } - } - @Override - public void open() throws HyracksDataException { - for (IFrameWriter writer : writers) { - writer.open(); + @Override + public void nextFrame(ByteBuffer bufferAccessor) throws HyracksDataException { + if (requiresMaterialization) { + state.appendFrame(bufferAccessor); + } + for (int i = 0; i < numberOfNonMaterializedOutputs; i++) { + FrameUtils.flushFrame(bufferAccessor, writers[i]); + } } - } - @Override - public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) { - writers[index] = writer; - } - }; + @Override + public void close() throws HyracksDataException { + if (requiresMaterialization) { + state.close(); + ctx.setStateObject(state); + } + for (int i = 0; i < numberOfNonMaterializedOutputs; i++) { + writers[i].close(); + } + } + + @Override + public void fail() throws HyracksDataException { + for (int i = 0; i < numberOfNonMaterializedOutputs; i++) { + writers[i].fail(); + } + } + + @Override + public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) { + writers[index] = writer; + } + }; + } + } + + private final class MaterializeReaderActivityNode extends AbstractActivityNode { + private static final long serialVersionUID = 1L; + + public MaterializeReaderActivityNode(ActivityId id) { + super(id); + } + + @Override + public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, + final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) + throws HyracksDataException { + return new AbstractUnaryOutputSourceOperatorNodePushable() { + + @Override + public void initialize() throws HyracksDataException { + ByteBuffer frame = ctx.allocateFrame(); + MaterializerTaskState state = (MaterializerTaskState) ctx.getStateObject(new TaskId(new ActivityId( + getOperatorId(), SPLITTER_MATERIALIZER_ACTIVITY_ID), partition)); + state.writeOut(writer, frame); + } + + @Override + public void deinitialize() throws HyracksDataException { + } + }; + } } } From b14a4dc22461e141a6b95dc190d8515b618a23d1 Mon Sep 17 00:00:00 2001 From: icetindil Date: Tue, 24 Jun 2014 04:03:04 -0700 Subject: [PATCH 06/15] fixing a bug on computing live variables for unnestmap operator --- .../operators/logical/visitors/SchemaVariableVisitor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java index 1d0fd7cf7..b8cc3b86f 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java @@ -224,7 +224,11 @@ public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksE @Override public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException { - standardLayout(op); + if (op.propagatesInput()) { + standardLayout(op); + } else { + VariableUtilities.getProducedVariables(op, schemaVariables); + } return null; } From c71ef28923fab852f7a876bb4d2f70148cb13294 Mon Sep 17 00:00:00 2001 From: icetindil Date: Thu, 26 Jun 2014 04:55:17 -0700 Subject: [PATCH 07/15] - added support to find shared plans containing operators with multiple inputs - removed redundant variables in the decor list of a group by operator" --- .../rules/ExtractCommonOperatorsRule.java | 36 ++++++++++---- .../rules/RemoveRedundantVariablesRule.java | 47 +++++++++++++------ 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index 2ac511a6a..64b8982cb 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -15,6 +15,7 @@ package edu.uci.ics.hyracks.algebricks.rewriter.rules; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -51,6 +52,7 @@ public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule { private HashMap, List>> childrenToParents = new HashMap, List>>(); private List> roots = new ArrayList>(); private List>> equivalenceClasses = new ArrayList>>(); + private HashMap, BitSet> opToCandidateInputs = new HashMap, BitSet>(); private static HashSet opsWorthMaterialization = new HashSet(); static { opsWorthMaterialization.add(LogicalOperatorTag.SELECT); @@ -101,6 +103,7 @@ public boolean rewritePost(Mutable opRef, IOptimizationContext rewritten = changed; equivalenceClasses.clear(); childrenToParents.clear(); + opToCandidateInputs.clear(); } while (changed); roots.clear(); } @@ -316,19 +319,34 @@ private void candidatesGrow(List> opList, List op : opList) { - for (Mutable inputRef : op.getValue().getInputs()) { + List> inputs = op.getValue().getInputs(); + for (int i = 0; i < inputs.size(); i++) { + Mutable inputRef = inputs.get(i); validCandidate = false; - // if current input is in candidates - for (Mutable candidate : previousCandidates) - if (inputRef.getValue().equals(candidate.getValue())) - validCandidate = true; - // if one input is not in candidates - if (!validCandidate) - break; + for (Mutable candidate : previousCandidates) { + // if current input is in candidates + if (inputRef.getValue().equals(candidate.getValue())) { + if (inputs.size() == 1) { + validCandidate = true; + } else { + BitSet candidateInputBitMap = opToCandidateInputs.get(op); + if (candidateInputBitMap == null) { + candidateInputBitMap = new BitSet(inputs.size()); + opToCandidateInputs.put(op, candidateInputBitMap); + } + candidateInputBitMap.set(i); + if (candidateInputBitMap.cardinality() == inputs.size()) { + validCandidate = true; + } + } + break; + } + } } if (!validCandidate) continue; - candidates.add(op); + if (!candidates.contains(op)) + candidates.add(op); } } diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java index bc0518182..c34a52f95 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java @@ -48,22 +48,20 @@ * Replaces redundant variable references with their bottom-most equivalent representative. * Does a DFS sweep over the plan keeping track of variable equivalence classes. * For example, this rule would perform the following rewrite. - * * Before Plan: * select (function-call: func, Args:[%0->$$11]) - * project [$11] - * assign [$$11] <- [$$10] - * assign [$$10] <- [$$9] - * assign [$$9] <- ... - * ... - * + * project [$11] + * assign [$$11] <- [$$10] + * assign [$$10] <- [$$9] + * assign [$$9] <- ... + * ... * After Plan: * select (function-call: func, Args:[%0->$$9]) - * project [$9] - * assign [$$11] <- [$$9] - * assign [$$10] <- [$$9] - * assign [$$9] <- ... - * ... + * project [$9] + * assign [$$11] <- [$$9] + * assign [$$10] <- [$$9] + * assign [$$9] <- ... + * ... */ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule { @@ -71,7 +69,7 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule { private final Map> equivalentVarsMap = new HashMap>(); protected boolean hasRun = false; - + @Override public boolean rewritePost(Mutable opRef, IOptimizationContext context) { return false; @@ -138,7 +136,7 @@ private boolean removeRedundantVariables(Mutable opRef, IOptim if (replaceProjectVars((ProjectOperator) op)) { modified = true; } - } else if(op.getOperatorTag() == LogicalOperatorTag.UNIONALL) { + } else if (op.getOperatorTag() == LogicalOperatorTag.UNIONALL) { // Replace redundant variables manually in the UnionAll operator. if (replaceUnionAllVars((UnionAllOperator) op)) { modified = true; @@ -205,6 +203,25 @@ private boolean handleGroupByVarRemapping(GroupByOperator groupOp) { } } } + // find the redundant variables within the decor list + Map variableToFirstDecorMap = new HashMap(); + Iterator>> iter = groupOp.getDecorList().iterator(); + while (iter.hasNext()) { + Pair> dp = iter.next(); + if (dp.first == null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) { + continue; + } + LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference(); + LogicalVariable firstDecor = variableToFirstDecorMap.get(dv); + if (firstDecor == null) { + variableToFirstDecorMap.put(dv, dp.first); + } else { + // The decor variable dp.first is redundant since firstDecor is exactly the same. + updateEquivalenceClassMap(dp.first, firstDecor); + iter.remove(); + modified = true; + } + } return modified; } @@ -251,7 +268,7 @@ private boolean replaceUnionAllVars(UnionAllOperator op) throws AlgebricksExcept } return modified; } - + private class VariableSubstitutionVisitor implements ILogicalExpressionReferenceTransform { @Override public boolean transform(Mutable exprRef) { From e58781032ce69a7886cade51bd4cbe32295dbbaa Mon Sep 17 00:00:00 2001 From: icetindil Date: Tue, 1 Jul 2014 11:21:43 -0700 Subject: [PATCH 08/15] fixed issue union operator was causing in dependency analysis --- .../rules/ExtractCommonOperatorsRule.java | 135 ++++++++++------- .../results/q7_volume_shipping.plan | 136 ++++++++++++------ .../results/u10_nestedloop_join.plan | 34 +++-- .../optimizerts/results/u3_union.plan | 32 +++-- 4 files changed, 219 insertions(+), 118 deletions(-) diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index 64b8982cb..f888a64be 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.commons.lang3.mutable.Mutable; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.mutable.MutableObject; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -53,6 +54,9 @@ public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule { private List> roots = new ArrayList>(); private List>> equivalenceClasses = new ArrayList>>(); private HashMap, BitSet> opToCandidateInputs = new HashMap, BitSet>(); + private HashMap, MutableInt> clusterMap = new HashMap, MutableInt>(); + private HashMap clusterWaitForMap = new HashMap(); + private int lastUsedClusterId = 0; private static HashSet opsWorthMaterialization = new HashSet(); static { opsWorthMaterialization.add(LogicalOperatorTag.SELECT); @@ -104,6 +108,9 @@ public boolean rewritePost(Mutable opRef, IOptimizationContext equivalenceClasses.clear(); childrenToParents.clear(); opToCandidateInputs.clear(); + clusterMap.clear(); + clusterWaitForMap.clear(); + lastUsedClusterId = 0; } while (changed); roots.clear(); } @@ -390,20 +397,23 @@ private void prune(IOptimizationContext context) throws AlgebricksException { } private boolean[] computeMaterilizationFlags(List> group) { - List>> clusters = new ArrayList>>(); - List> blockedOps = new ArrayList>(); - for (Mutable opRef : group) { - List> cluster = new ArrayList>(); - computeDependency(null, opRef, blockedOps, cluster, false); - clusters.add(cluster); + lastUsedClusterId = 0; + for (Mutable root : roots) { + computeClusters(null, root, new MutableInt(++lastUsedClusterId)); } boolean[] materializationFlags = new boolean[group.size()]; boolean worthMaterialization = worthMaterialization(group.get(0)); boolean requiresMaterialization; + // get clusterIds for each candidate in the group + List groupClusterIds = new ArrayList(group.size()); + for (int i = 0; i < group.size(); i++) { + groupClusterIds.add(clusterMap.get(group.get(i)).getValue()); + } for (int i = group.size() - 1; i >= 0; i--) { - requiresMaterialization = requiresMaterialization(clusters.get(i), blockedOps); + requiresMaterialization = requiresMaterialization(groupClusterIds, i); if (requiresMaterialization && !worthMaterialization) { group.remove(i); + groupClusterIds.remove(i); } materializationFlags[i] = requiresMaterialization; } @@ -414,59 +424,80 @@ private boolean[] computeMaterilizationFlags(List> gro return worthMaterialization ? materializationFlags : new boolean[group.size()]; } - private void computeDependency(Mutable inputRef, Mutable opRef, - List> blockedOps, List> cluster, boolean foundBlocker) { - List> parentRefList = childrenToParents.get(opRef); - if (parentRefList != null) { - Pair labels = null; - int inputIndex = 0; - if (opRef.getValue().isBlocker() && inputRef != null) { - labels = opRef.getValue().getInputOutputDependencyLabels(); - List> inputs = opRef.getValue().getInputs(); - for (inputIndex = 0; inputIndex < inputs.size(); inputIndex++) { - if (inputs.get(inputIndex).equals(inputRef)) { - break; - } + private boolean requiresMaterialization(List groupClusterIds, int index) { + Integer clusterId = groupClusterIds.get(index); + BitSet blockingClusters = new BitSet(); + getAllBlockingClusterIds(clusterId, blockingClusters); + if (!blockingClusters.isEmpty()) { + for (int i = 0; i < groupClusterIds.size(); i++) { + if (i == index) { + continue; } - } - for (Mutable parentRef : parentRefList) { - boolean foundBlockerForCurPath = foundBlocker; - if (opRef.getValue().isBlocker() && inputRef != null) { - // only replicate operator has multiple outputs - int outputIndex = 0; - if (opRef.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) { - ReplicateOperator rop = (ReplicateOperator) opRef.getValue(); - List> outputs = rop.getOutputs(); - for (outputIndex = 0; outputIndex < outputs.size(); outputIndex++) { - if (outputs.get(outputIndex).equals(parentRef)) { - break; - } - } - } - if (labels.second[outputIndex] == 1) { - if (labels.first[inputIndex] == 1) { // 1 -> 1 - if (!foundBlocker) { - cluster.add(opRef); - } - } else { // 0 -> 1 - blockedOps.add(opRef); - foundBlockerForCurPath = true; - } - } + if (blockingClusters.get(groupClusterIds.get(i))) { + return true; } - computeDependency(opRef, parentRef, blockedOps, cluster, foundBlockerForCurPath); } } + return false; } - private boolean requiresMaterialization(List> cluster, - List> blockedOps) { - for (Mutable op : cluster) { - if (blockedOps.contains(op)) { - return true; + private void getAllBlockingClusterIds(int clusterId, BitSet blockingClusters) { + BitSet waitFor = clusterWaitForMap.get(clusterId); + if (waitFor != null) { + for (int i = waitFor.nextSetBit(0); i >= 0; i = waitFor.nextSetBit(i + 1)) { + getAllBlockingClusterIds(i, blockingClusters); + } + blockingClusters.or(waitFor); + } + } + + private void computeClusters(Mutable parentRef, Mutable opRef, + MutableInt currentClusterId) { + // only replicate operator has multiple outputs + int outputIndex = 0; + if (opRef.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) { + ReplicateOperator rop = (ReplicateOperator) opRef.getValue(); + List> outputs = rop.getOutputs(); + for (outputIndex = 0; outputIndex < outputs.size(); outputIndex++) { + if (outputs.get(outputIndex).equals(parentRef)) { + break; + } + } + } + Pair labels = opRef.getValue().getInputOutputDependencyLabels(); + List> inputs = opRef.getValue().getInputs(); + for (int i = 0; i < inputs.size(); i++) { + Mutable inputRef = inputs.get(i); + if (labels.second[outputIndex] == 1 && labels.first[i] == 0) { // 1 -> 0 + if (labels.second.length == 1) { + clusterMap.put(opRef, currentClusterId); + // start a new cluster + MutableInt newClusterId = new MutableInt(++lastUsedClusterId); + computeClusters(opRef, inputRef, newClusterId); + BitSet waitForList = clusterWaitForMap.get(currentClusterId.getValue()); + if (waitForList == null) { + waitForList = new BitSet(); + clusterWaitForMap.put(currentClusterId.getValue(), waitForList); + } + waitForList.set(newClusterId.getValue()); + } + } else { // 0 -> 0 and 1 -> 1 + MutableInt prevClusterId = clusterMap.get(opRef); + if (prevClusterId == null || prevClusterId.getValue().equals(currentClusterId.getValue())) { + clusterMap.put(opRef, currentClusterId); + computeClusters(opRef, inputRef, currentClusterId); + } else { + // merge prevClusterId and currentClusterId: update all the map entries that has currentClusterId to prevClusterId + for (BitSet bs : clusterWaitForMap.values()) { + if (bs.get(currentClusterId.getValue())) { + bs.clear(currentClusterId.getValue()); + bs.set(prevClusterId.getValue()); + } + } + currentClusterId.setValue(prevClusterId.getValue()); + } } } - return false; } private boolean worthMaterialization(Mutable candidate) { diff --git a/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/q7_volume_shipping.plan b/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/q7_volume_shipping.plan index 9f62e7bd8..70d8e7b7a 100644 --- a/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/q7_volume_shipping.plan +++ b/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/q7_volume_shipping.plan @@ -14,38 +14,58 @@ write [%0->$$17, %0->$$18, %0->$$19, %0->$$20] -- NESTED_LOOP |PARTITIONED| exchange -- BROADCAST_EXCHANGE |PARTITIONED| - select (function-call: algebricks:eq, Args:[%0->$$2, GERMANY]) - -- STREAM_SELECT |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan [$$1, $$2]<-[$$1, $$2, $$3, $$4] <- default.nation - -- DATASOURCE_SCAN |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - project ([$$5, $$6]) + project ([$$1, $$2]) -- STREAM_PROJECT |PARTITIONED| - select (function-call: algebricks:eq, Args:[%0->$$6, FRANCE]) - -- STREAM_SELECT |PARTITIONED| - project ([$$5, $$6]) - -- STREAM_PROJECT |PARTITIONED| - assign [$$5, $$6] <- [%0->$$9, %0->$$10] - -- ASSIGN |PARTITIONED| + assign [$$1, $$2] <- [%0->$$9, %0->$$10] + -- ASSIGN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - replicate - -- SPLIT |PARTITIONED| + select (function-call: algebricks:eq, Args:[%0->$$10, GERMANY]) + -- STREAM_SELECT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan [$$9, $$10]<-[$$9, $$10, $$11, $$12] <- default.nation - -- DATASOURCE_SCAN |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + data-scan [$$9, $$10]<-[$$9, $$10, $$11, $$12] <- default.nation + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$5, $$6]) + -- STREAM_PROJECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + select (function-call: algebricks:eq, Args:[%0->$$6, FRANCE]) + -- STREAM_SELECT |PARTITIONED| + project ([$$5, $$6]) + -- STREAM_PROJECT |PARTITIONED| + assign [$$5, $$6] <- [%0->$$9, %0->$$10] + -- ASSIGN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan [$$9, $$10]<-[$$9, $$10, $$11, $$12] <- default.nation + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| project ([$$10, $$14, $$9, $$13]) @@ -56,34 +76,58 @@ write [%0->$$17, %0->$$18, %0->$$19, %0->$$20] -- NESTED_LOOP |PARTITIONED| exchange -- BROADCAST_EXCHANGE |PARTITIONED| - select (function-call: algebricks:eq, Args:[%0->$$14, FRANCE]) - -- STREAM_SELECT |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan [$$13, $$14]<-[$$13, $$14, $$15, $$16] <- default.nation - -- DATASOURCE_SCAN |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - project ([$$9, $$10]) + project ([$$13, $$14]) -- STREAM_PROJECT |PARTITIONED| - select (function-call: algebricks:eq, Args:[%0->$$10, GERMANY]) - -- STREAM_SELECT |PARTITIONED| + assign [$$13, $$14] <- [%0->$$5, %0->$$6] + -- ASSIGN |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| replicate -- SPLIT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan [$$9, $$10]<-[$$9, $$10, $$11, $$12] <- default.nation - -- DATASOURCE_SCAN |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + select (function-call: algebricks:eq, Args:[%0->$$6, FRANCE]) + -- STREAM_SELECT |PARTITIONED| + project ([$$5, $$6]) + -- STREAM_PROJECT |PARTITIONED| + assign [$$5, $$6] <- [%0->$$9, %0->$$10] + -- ASSIGN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan [$$9, $$10]<-[$$9, $$10, $$11, $$12] <- default.nation + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$9, $$10]) + -- STREAM_PROJECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + select (function-call: algebricks:eq, Args:[%0->$$10, GERMANY]) + -- STREAM_SELECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan [$$9, $$10]<-[$$9, $$10, $$11, $$12] <- default.nation + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| write [%0->$$47, %0->$$48, %0->$$49, %0->$$50] -- SINK_WRITE |PARTITIONED| project ([$$47, $$48, $$49, $$50]) diff --git a/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u10_nestedloop_join.plan b/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u10_nestedloop_join.plan index c86d57f28..a2bb4e48f 100644 --- a/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u10_nestedloop_join.plan +++ b/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u10_nestedloop_join.plan @@ -8,17 +8,31 @@ write [%0->$$6, %0->$$2, %0->$$5, %0->$$1] -- NESTED_LOOP |PARTITIONED| exchange -- BROADCAST_EXCHANGE |PARTITIONED| - data-scan [$$1, $$2]<-[$$1, $$2, $$3, $$4] <- default.nation - -- DATASOURCE_SCAN |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + project ([$$1, $$2]) + -- STREAM_PROJECT |PARTITIONED| + assign [$$1, $$2] <- [%0->$$5, %0->$$6] + -- ASSIGN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan [$$5, $$6]<-[$$5, $$6, $$7, $$8] <- default.nation + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan [$$5, $$6]<-[$$5, $$6, $$7, $$8] <- default.nation - -- DATASOURCE_SCAN |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + data-scan [$$5, $$6]<-[$$5, $$6, $$7, $$8] <- default.nation + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u3_union.plan b/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u3_union.plan index c4040f2b9..3923e19af 100644 --- a/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u3_union.plan +++ b/hivesterix/hivesterix-dist/src/test/resources/optimizerts/results/u3_union.plan @@ -12,14 +12,22 @@ write [%0->$$17, %0->$$18, %0->$$19, %0->$$20] -- ASSIGN |PARTITIONED| select (function-call: algebricks:gt, Args:[function-call: hive:org.apache.hadoop.hive.ql.udf.UDFOPMultiply, Args:[%0->$$1, 2], 50]) -- STREAM_SELECT |PARTITIONED| - exchange - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan []<-[$$1, $$2, $$3, $$4, $$5, $$6, $$7] <- default.supplier - -- DATASOURCE_SCAN |PARTITIONED| + project ([$$1, $$2, $$3, $$4, $$5, $$6, $$7]) + -- STREAM_PROJECT |PARTITIONED| + assign [$$1, $$2, $$3, $$4, $$5, $$6, $$7] <- [%0->$$9, %0->$$10, %0->$$11, %0->$$12, %0->$$13, %0->$$14, %0->$$15] + -- ASSIGN |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$9, $$10, $$11, $$12, $$13, $$14, $$15] <- default.supplier + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| project ([$$16, $$11, $$12, $$10]) @@ -30,9 +38,13 @@ write [%0->$$17, %0->$$18, %0->$$19, %0->$$20] -- STREAM_SELECT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - data-scan []<-[$$9, $$10, $$11, $$12, $$13, $$14, $$15] <- default.supplier - -- DATASOURCE_SCAN |PARTITIONED| + replicate + -- SPLIT |PARTITIONED| exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - empty-tuple-source - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + data-scan []<-[$$9, $$10, $$11, $$12, $$13, $$14, $$15] <- default.supplier + -- DATASOURCE_SCAN |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| From b661556c60b6c0ad8247c208beccc736dd88e0a8 Mon Sep 17 00:00:00 2001 From: icetindil Date: Thu, 10 Jul 2014 18:43:28 -0700 Subject: [PATCH 09/15] in HHJ handle the case when it spills and skipInMemoryHJ is set to false, check for memsize in NLJ and correctly set memsize in HHJ, make counthashed-ngram-token() to skip the bits for length & type --- .../dataflow/std/join/NestedLoopJoin.java | 3 + ...mizedHybridHashJoinOperatorDescriptor.java | 57 ++++++++++--------- .../NGramUTF8StringBinaryTokenizer.java | 5 +- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java index 52f1198a8..eab60bcd2 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java @@ -64,6 +64,9 @@ public NestedLoopJoin(IHyracksTaskContext ctx, FrameTupleAccessor accessor0, Fra this.appender.reset(outBuffer, true); this.outBuffers = new ArrayList(); this.memSize = memSize; + if (memSize < 3) { + throw new HyracksDataException("Not enough memory is available for Nested Loop Join"); + } this.predEvaluator = predEval; this.isReversed = false; this.ctx = ctx; diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java index 276f60f9b..744f93917 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java @@ -437,30 +437,33 @@ private void joinPartitionPair(OptimizedHybridHashJoin ohhj, RunFileReader build LOGGER.fine("\n>>>Joining Partition Pairs (pid "+pid+") - (level "+level+") - wasReversed "+wasReversed+" - BuildSize:\t"+buildPartSize+"\tProbeSize:\t"+probePartSize+" - MemForJoin "+(state.memForJoin)+" - LeftOuter is "+isLeftOuter); //Apply in-Mem HJ if possible - if(!skipInMemoryHJ){ - if ((buildPartSize < state.memForJoin) || (probePartSize < state.memForJoin && !isLeftOuter)) { - int tabSize = -1; - if (!forceRR && (isLeftOuter || (buildPartSize < probePartSize)) ) { //Case 1.1 - InMemHJ (wout Role-Reversal) - LOGGER.fine("\t>>>Case 1.1 (IsLeftOuter || buildSize>>Case 1.2. (NoIsLeftOuter || probe>>Case 1.1 (IsLeftOuter || buildSize>>Case 1.2. (NoIsLeftOuter || probe probeSideSize) { - applyNestedLoopJoin(buildRd, probeRd, state.memForJoin, rbrfw, rprfw, + applyNestedLoopJoin(buildRd, probeRd, memsize, rbrfw, rprfw, nljComparator0, true); //checked-modified } else { - applyNestedLoopJoin(probeRd, buildRd, state.memForJoin, rprfw, rbrfw, + applyNestedLoopJoin(probeRd, buildRd, memsize, rprfw, rbrfw, nljComparator1, true); //checked-modified } } diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java index 847a9235a..b1d722e1a 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java @@ -64,7 +64,10 @@ public void next() { // compute token count // ignore pre and post grams for duplicate detection if (!ignoreTokenCount && numPreChars == 0 && numPostChars == 0) { - int tmpIndex = start; + int tmpIndex = start + 2; // skip utf8 length indicator + if (sourceHasTypeTag) { + tmpIndex++; // skip type tag + } while (tmpIndex < currentTokenStart) { tokenCount++; // assume found int offset = 0; From bb51fa1525f35194e94a33647eeee7001783d566 Mon Sep 17 00:00:00 2001 From: icetindil Date: Fri, 18 Jul 2014 13:51:56 -0700 Subject: [PATCH 10/15] move dependency labeling from logical to physical --- .../core/algebra/base/ILogicalOperator.java | 5 ---- .../core/algebra/base/IPhysicalOperator.java | 7 ++++++ .../logical/AbstractBinaryJoinOperator.java | 13 ---------- .../logical/AbstractLogicalOperator.java | 17 ------------- .../operators/logical/GroupByOperator.java | 25 ------------------- .../operators/logical/OrderOperator.java | 12 --------- .../operators/logical/ReplicateOperator.java | 15 ----------- .../physical/AbstractJoinPOperator.java | 9 +++++++ .../physical/AbstractPhysicalOperator.java | 12 +++++++++ .../physical/AbstractStableSortPOperator.java | 6 +++++ .../physical/ExternalGroupByPOperator.java | 7 ++++++ .../physical/ReplicatePOperator.java | 16 ++++++++++++ .../physical/SortGroupByPOperator.java | 7 ++++++ .../rules/ExtractCommonOperatorsRule.java | 3 ++- 14 files changed, 66 insertions(+), 88 deletions(-) diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java index 061bfcb21..d91cac057 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; -import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; @@ -46,10 +45,6 @@ public interface ILogicalOperator { public List getSchema(); - public Pair getInputOutputDependencyLabels(); - - public boolean isBlocker(); - /* * * support for visitors diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java index f3f9837bf..6da26d2bd 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java @@ -15,6 +15,7 @@ package edu.uci.ics.hyracks.algebricks.core.algebra.base; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements; @@ -57,4 +58,10 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext public void setHostQueryContext(Object context); public Object getHostQueryContext(); + + /** + * @return labels (0 or 1) for each input and output indicating the dependency between them. + * The edges labeled as 1 must wait for the edges with label 0. + */ + public Pair getInputOutputDependencyLabels(ILogicalOperator op); } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java index 326ec246e..6c00412f9 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java @@ -19,7 +19,6 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; -import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; @@ -76,16 +75,4 @@ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform vi public boolean isMap() { return false; } - - @Override - public Pair getInputOutputDependencyLabels() { - int[] inputDependencyLabels = new int[] { 1, 0 }; - int[] outputDependencyLabels = new int[] { 1 }; - return new Pair(inputDependencyLabels, outputDependencyLabels); - } - - @Override - public boolean isBlocker() { - return true; - } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java index b9e626c5b..78efd0db8 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java @@ -22,7 +22,6 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; -import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; @@ -134,22 +133,6 @@ public final boolean hasInputs() { return !inputs.isEmpty(); } - /** - * @return labels (0 or 1) for each input and output indicating the dependency between them. - * The edges labeled as 1 must wait for the edges with label 0. - */ - @Override - public Pair getInputOutputDependencyLabels() { - int[] inputDependencyLabels = new int[inputs.size()]; // filled with 0's - int[] outputDependencyLabels = new int[] { 0 }; - return new Pair(inputDependencyLabels, outputDependencyLabels); - } - - @Override - public boolean isBlocker() { - return false; - } - public boolean hasNestedPlans() { return false; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java index 514da1fd3..a456e71f6 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java @@ -29,7 +29,6 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy; @@ -274,28 +273,4 @@ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) } return env; } - - @Override - public Pair getInputOutputDependencyLabels() { - int[] inputDependencyLabels = new int[] { 0 }; - int[] outputDependencyLabels; - PhysicalOperatorTag physicalOpTag = this.getPhysicalOperator().getOperatorTag(); - if (physicalOpTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY - || physicalOpTag == PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY) { - outputDependencyLabels = new int[] { 0 }; - } else { - outputDependencyLabels = new int[] { 1 }; - } - return new Pair(inputDependencyLabels, outputDependencyLabels); - } - - @Override - public boolean isBlocker() { - PhysicalOperatorTag physicalOpTag = this.getPhysicalOperator().getOperatorTag(); - if (physicalOpTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY - || physicalOpTag == PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY) { - return false; - } - return true; - } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java index eb5abdf9a..335fb36ce 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java @@ -156,16 +156,4 @@ public boolean isMap() { public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); } - - @Override - public Pair getInputOutputDependencyLabels() { - int[] inputDependencyLabels = new int[] { 0 }; - int[] outputDependencyLabels = new int[] { 1 }; - return new Pair(inputDependencyLabels, outputDependencyLabels); - } - - @Override - public boolean isBlocker() { - return true; - } } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java index 552fb6ccd..3f1397754 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; -import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; @@ -103,20 +102,6 @@ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) return createPropagatingAllInputsTypeEnvironment(ctx); } - @Override - public Pair getInputOutputDependencyLabels() { - int[] inputDependencyLabels = new int[] { 0 }; - int[] outputDependencyLabels = new int[outputArity]; - // change the labels of outputs that requires materialization to 1 - for (int i = 0; i < outputArity; i++) { - if (outputMaterializationFlags[i]) { - outputDependencyLabels[i] = 1; - } - } - return new Pair(inputDependencyLabels, outputDependencyLabels); - } - - @Override public boolean isBlocker() { for (boolean requiresMaterialization : outputMaterializationFlags) { if (requiresMaterialization) { diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java index 03f03fdc8..bd24b02f7 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java @@ -14,6 +14,8 @@ */ package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind; public abstract class AbstractJoinPOperator extends AbstractPhysicalOperator { @@ -37,4 +39,11 @@ public JoinKind getKind() { public JoinPartitioningType getPartitioningType() { return partitioningType; } + + @Override + public Pair getInputOutputDependencyLabels(ILogicalOperator op) { + int[] inputDependencyLabels = new int[] { 1, 0 }; + int[] outputDependencyLabels = new int[] { 1 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java index ac6488249..4407b750a 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java @@ -20,6 +20,7 @@ import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan; @@ -79,6 +80,17 @@ public boolean isJobGenDisabledBelowMe() { return disableJobGenBelow; } + /** + * @return labels (0 or 1) for each input and output indicating the dependency between them. + * The edges labeled as 1 must wait for the edges with label 0. + */ + @Override + public Pair getInputOutputDependencyLabels(ILogicalOperator op) { + int[] inputDependencyLabels = new int[op.getInputs().size()]; // filled with 0's + int[] outputDependencyLabels = new int[] { 0 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } + protected void contributeOpDesc(IHyracksJobBuilder builder, AbstractLogicalOperator op, IOperatorDescriptor opDesc) { if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) { AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java index ffc75c84c..047213466 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java @@ -127,4 +127,10 @@ public String toString() { } } + @Override + public Pair getInputOutputDependencyLabels(ILogicalOperator op) { + int[] inputDependencyLabels = new int[] { 0 }; + int[] outputDependencyLabels = new int[] { 1 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java index da70ba727..0358219f5 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java @@ -256,4 +256,11 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } + + @Override + public Pair getInputOutputDependencyLabels(ILogicalOperator op) { + int[] inputDependencyLabels = new int[] { 0 }; + int[] outputDependencyLabels = new int[] { 1 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java index daee6706e..c433f5c31 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java @@ -15,6 +15,7 @@ package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; +import edu.uci.ics.hyracks.algebricks.common.utils.Pair; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; @@ -71,4 +72,19 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } + + @Override + public Pair getInputOutputDependencyLabels(ILogicalOperator op) { + int[] inputDependencyLabels = new int[] { 0 }; + ReplicateOperator rop = (ReplicateOperator) op; + int[] outputDependencyLabels = new int[rop.getOutputArity()]; + // change the labels of outputs that requires materialization to 1 + boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags(); + for (int i = 0; i < rop.getOutputArity(); i++) { + if (outputMaterializationFlags[i]) { + outputDependencyLabels[i] = 1; + } + } + return new Pair(inputDependencyLabels, outputDependencyLabels); + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java index e92e3ee11..8416a8174 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java @@ -269,4 +269,11 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } + + @Override + public Pair getInputOutputDependencyLabels(ILogicalOperator op) { + int[] inputDependencyLabels = new int[] { 0 }; + int[] outputDependencyLabels = new int[] { 1 }; + return new Pair(inputDependencyLabels, outputDependencyLabels); + } } diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index f888a64be..cf0c5bda1 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -464,7 +464,8 @@ private void computeClusters(Mutable parentRef, Mutable labels = opRef.getValue().getInputOutputDependencyLabels(); + AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue(); + Pair labels = aop.getPhysicalOperator().getInputOutputDependencyLabels(opRef.getValue()); List> inputs = opRef.getValue().getInputs(); for (int i = 0; i < inputs.size(); i++) { Mutable inputRef = inputs.get(i); From efee860dc18f55a4cc2f6cfc13e3fdb6f6d73d5f Mon Sep 17 00:00:00 2001 From: icetindil Date: Fri, 18 Jul 2014 16:17:33 -0700 Subject: [PATCH 11/15] added a new method expensiveThanMaterialization() to logical operators to help with the decision whether it worths materializing --- .../core/algebra/base/ILogicalOperator.java | 7 +++++++ .../logical/AbstractBinaryJoinOperator.java | 5 +++++ .../logical/AbstractOperatorWithNestedPlans.java | 5 +++++ .../operators/logical/AggregateOperator.java | 5 +++++ .../operators/logical/AssignOperator.java | 5 +++++ .../logical/DataSourceScanOperator.java | 5 +++++ .../operators/logical/DistinctOperator.java | 5 +++++ .../logical/DistributeResultOperator.java | 5 +++++ .../logical/EmptyTupleSourceOperator.java | 5 +++++ .../operators/logical/ExchangeOperator.java | 5 +++++ .../operators/logical/ExtensionOperator.java | 5 +++++ .../logical/ExternalDataLookupOperator.java | 5 +++++ .../logical/IndexInsertDeleteOperator.java | 5 +++++ .../operators/logical/InsertDeleteOperator.java | 5 +++++ .../algebra/operators/logical/LimitOperator.java | 5 +++++ .../logical/NestedTupleSourceOperator.java | 5 +++++ .../algebra/operators/logical/OrderOperator.java | 5 +++++ .../logical/PartitioningSplitOperator.java | 5 +++++ .../operators/logical/ProjectOperator.java | 5 +++++ .../operators/logical/ReplicateOperator.java | 5 +++++ .../logical/RunningAggregateOperator.java | 5 +++++ .../operators/logical/ScriptOperator.java | 5 +++++ .../operators/logical/SelectOperator.java | 5 +++++ .../algebra/operators/logical/SinkOperator.java | 5 +++++ .../operators/logical/UnionAllOperator.java | 5 +++++ .../operators/logical/UnnestMapOperator.java | 4 ++++ .../operators/logical/UnnestOperator.java | 5 +++++ .../operators/logical/UpdateOperator.java | 5 +++++ .../algebra/operators/logical/WriteOperator.java | 5 +++++ .../operators/logical/WriteResultOperator.java | 5 +++++ .../rules/ExtractCommonOperatorsRule.java | 16 +--------------- .../dataflow/std/misc/MaterializerTaskState.java | 4 ++-- 32 files changed, 154 insertions(+), 17 deletions(-) diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java index d91cac057..377ab1a2c 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java @@ -56,6 +56,13 @@ public interface ILogicalOperator { public boolean isMap(); + /* + * This is needed to have a kind of cost based decision on whether to merge the shared subplans and materialize the result. + * If the subgraph whose result we would like to materialize has an operator that is computationally expensive, we assume + * it is cheaper to materialize the result of this subgraph and read from the file rather than recomputing it. + */ + public boolean expensiveThanMaterialization(); + public Map getAnnotations(); public void removeAnnotation(String annotationName); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java index 6c00412f9..56e690472 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java @@ -75,4 +75,9 @@ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform vi public boolean isMap() { return false; } + + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java index 417c959eb..d313dce0e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java @@ -83,6 +83,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + public abstract void getUsedVariablesExceptNestedPlans(Collection vars); public abstract void getProducedVariablesExceptNestedPlans(Collection vars); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java index 371da4cc9..99c8534b8 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java @@ -70,6 +70,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java index 0af3dbc12..a7374fbaa 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java @@ -76,6 +76,11 @@ public boolean isMap() { return true; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java index ba51f047a..c5abf0340 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java @@ -72,6 +72,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + public void addProjectVariables(Collection vars) { projectVars.addAll(vars); projectPushed = true; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java index e5ece690d..52e199f64 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java @@ -100,6 +100,11 @@ public R accept(ILogicalOperatorVisitor visitor, T arg) throws Alge public boolean isMap() { return false; } + + @Override + public boolean expensiveThanMaterialization() { + return true; + } public List getDistinctByVarList() { List varList = new ArrayList(expressions.size()); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java index 3e28d7329..a4632fa01 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java @@ -79,6 +79,11 @@ public boolean isMap() { return false; // actually depends on the physical op. } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java index 5872e6d64..b09846861 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java @@ -60,6 +60,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException { return new IVariableTypeEnvironment() { diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java index a3c26505e..1ea070113 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java @@ -68,6 +68,11 @@ public boolean isMap() { return true; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java index bdf6f1c17..b52e5d58d 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java @@ -64,6 +64,11 @@ public boolean isMap() { return this.delegate.isMap(); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java index e18c46155..75baf73fd 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java @@ -76,6 +76,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + public boolean isPropagateInput() { return propagateInput; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java index ea45cc624..fa061b877 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java @@ -82,6 +82,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java index 94fd19705..d71db1fd3 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java @@ -79,6 +79,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java index 79145f417..4f36bc6af 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java @@ -105,6 +105,11 @@ public boolean isMap() { return true; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java index be23f48e2..3ce2e0dcf 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java @@ -81,6 +81,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException { ITypeEnvPointer[] p = new ITypeEnvPointer[1]; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java index 335fb36ce..99f2cb2d6 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java @@ -152,6 +152,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java index a6c8f6f95..b6f1d83b9 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java @@ -107,6 +107,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java index bcba2800b..6ac4c0ef2 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java @@ -71,6 +71,11 @@ public boolean isMap() { return true; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + public List getVariables() { return variables; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java index 3f1397754..d3715b608 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java @@ -72,6 +72,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public void recomputeSchema() { schema = new ArrayList(inputs.get(0).getValue().getSchema()); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java index 4f608bbae..1c904b992 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java @@ -63,6 +63,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java index 46e98595f..3fa9ef656 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java @@ -88,6 +88,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public void recomputeSchema() { this.schema = outputVariables; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java index 5aa229c71..db7fe054e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java @@ -89,6 +89,11 @@ public boolean isMap() { return true; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { ITypeEnvPointer[] envPointers = new ITypeEnvPointer[1]; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java index bdad6d488..471848885 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java @@ -51,6 +51,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java index 97a77ecf0..0f470b997 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java @@ -99,6 +99,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java index 61e0b33de..e9cc1fd8e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java @@ -132,4 +132,8 @@ public boolean isMap() { } */ + @Override + public boolean expensiveThanMaterialization() { + return true; + } } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java index dba9e19bf..9384b16c5 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java @@ -141,4 +141,9 @@ public void propagateVariables(IOperatorSchema target, IOperatorSchema... source } }; } + + @Override + public boolean expensiveThanMaterialization() { + return false; + } } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java index 841f9c61c..f1bbe5d82 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java @@ -48,6 +48,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public VariablePropagationPolicy getVariablePropagationPolicy() { // TODO Auto-generated method stub diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java index e1c16403c..90f699c94 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java @@ -79,6 +79,11 @@ public boolean isMap() { return false; // actually depends on the physical op. } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java index ba402bcba..02e5d965a 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java @@ -87,6 +87,11 @@ public boolean isMap() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index cf0c5bda1..2545a1b23 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -57,20 +57,6 @@ public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule { private HashMap, MutableInt> clusterMap = new HashMap, MutableInt>(); private HashMap clusterWaitForMap = new HashMap(); private int lastUsedClusterId = 0; - private static HashSet opsWorthMaterialization = new HashSet(); - static { - opsWorthMaterialization.add(LogicalOperatorTag.SELECT); - opsWorthMaterialization.add(LogicalOperatorTag.INNERJOIN); - opsWorthMaterialization.add(LogicalOperatorTag.LEFTOUTERJOIN); - opsWorthMaterialization.add(LogicalOperatorTag.PROJECT); - opsWorthMaterialization.add(LogicalOperatorTag.GROUP); - opsWorthMaterialization.add(LogicalOperatorTag.ORDER); - opsWorthMaterialization.add(LogicalOperatorTag.AGGREGATE); - opsWorthMaterialization.add(LogicalOperatorTag.RUNNINGAGGREGATE); - opsWorthMaterialization.add(LogicalOperatorTag.DISTINCT); - opsWorthMaterialization.add(LogicalOperatorTag.LIMIT); - opsWorthMaterialization.add(LogicalOperatorTag.UNNEST_MAP); - } @Override public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { @@ -502,7 +488,7 @@ private void computeClusters(Mutable parentRef, Mutable candidate) { - if (opsWorthMaterialization.contains(candidate.getValue().getOperatorTag())) { + if (candidate.getValue().expensiveThanMaterialization()) { return true; } List> inputs = candidate.getValue().getInputs(); diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java index 02eb72a6e..ee857aba4 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 0d1e14d6e05f7e88cbdf71526607c167433eaeff Mon Sep 17 00:00:00 2001 From: icetindil Date: Tue, 5 Aug 2014 20:21:40 -0700 Subject: [PATCH 12/15] addressing Yingyi's code review comments - moved expensiveThanMaterialization() function into physical layer, - deleted the file once the latest output of replicate operator is done. --- .../algebricks/core/algebra/base/ILogicalOperator.java | 7 ------- .../algebricks/core/algebra/base/IPhysicalOperator.java | 8 ++++++++ .../operators/logical/AbstractBinaryJoinOperator.java | 5 ----- .../logical/AbstractOperatorWithNestedPlans.java | 5 ----- .../core/algebra/operators/logical/AggregateOperator.java | 5 ----- .../core/algebra/operators/logical/AssignOperator.java | 5 ----- .../algebra/operators/logical/DataSourceScanOperator.java | 5 ----- .../core/algebra/operators/logical/DistinctOperator.java | 5 ----- .../operators/logical/DistributeResultOperator.java | 5 ----- .../operators/logical/EmptyTupleSourceOperator.java | 5 ----- .../core/algebra/operators/logical/ExchangeOperator.java | 5 ----- .../core/algebra/operators/logical/ExtensionOperator.java | 5 ----- .../operators/logical/ExternalDataLookupOperator.java | 5 ----- .../operators/logical/IndexInsertDeleteOperator.java | 5 ----- .../algebra/operators/logical/InsertDeleteOperator.java | 5 ----- .../core/algebra/operators/logical/LimitOperator.java | 5 ----- .../operators/logical/NestedTupleSourceOperator.java | 5 ----- .../core/algebra/operators/logical/OrderOperator.java | 5 ----- .../operators/logical/PartitioningSplitOperator.java | 5 ----- .../core/algebra/operators/logical/ProjectOperator.java | 5 ----- .../core/algebra/operators/logical/ReplicateOperator.java | 5 ----- .../operators/logical/RunningAggregateOperator.java | 5 ----- .../core/algebra/operators/logical/ScriptOperator.java | 5 ----- .../core/algebra/operators/logical/SelectOperator.java | 5 ----- .../core/algebra/operators/logical/SinkOperator.java | 5 ----- .../core/algebra/operators/logical/UnionAllOperator.java | 5 ----- .../core/algebra/operators/logical/UnnestMapOperator.java | 5 ----- .../core/algebra/operators/logical/UnnestOperator.java | 5 ----- .../core/algebra/operators/logical/UpdateOperator.java | 5 ----- .../core/algebra/operators/logical/WriteOperator.java | 5 ----- .../algebra/operators/logical/WriteResultOperator.java | 5 ----- .../operators/physical/AbstractExchangePOperator.java | 5 +++++ .../algebra/operators/physical/AbstractJoinPOperator.java | 5 +++++ .../physical/AbstractPreclusteredGroupByPOperator.java | 4 ++++ .../algebra/operators/physical/AbstractScanPOperator.java | 4 ++++ .../operators/physical/AbstractStableSortPOperator.java | 5 +++++ .../algebra/operators/physical/AggregatePOperator.java | 4 ++++ .../core/algebra/operators/physical/AssignPOperator.java | 4 ++++ .../operators/physical/DistributeResultPOperator.java | 5 +++++ .../operators/physical/EmptyTupleSourcePOperator.java | 4 ++++ .../operators/physical/ExternalGroupByPOperator.java | 5 +++++ .../operators/physical/IndexInsertDeletePOperator.java | 4 ++++ .../algebra/operators/physical/InsertDeletePOperator.java | 4 ++++ .../operators/physical/NestedTupleSourcePOperator.java | 4 ++++ .../operators/physical/PreSortedDistinctByPOperator.java | 4 ++++ .../algebra/operators/physical/ReplicatePOperator.java | 5 +++++ .../operators/physical/RunningAggregatePOperator.java | 6 ++++-- .../core/algebra/operators/physical/SinkPOperator.java | 4 ++++ .../algebra/operators/physical/SinkWritePOperator.java | 4 ++++ .../algebra/operators/physical/SortGroupByPOperator.java | 5 +++++ .../algebra/operators/physical/StreamLimitPOperator.java | 4 ++++ .../operators/physical/StreamProjectPOperator.java | 4 ++++ .../algebra/operators/physical/StreamSelectPOperator.java | 4 ++++ .../physical/StringStreamingScriptPOperator.java | 4 ++++ .../core/algebra/operators/physical/SubplanPOperator.java | 4 ++++ .../algebra/operators/physical/UnionAllPOperator.java | 4 ++++ .../algebra/operators/physical/WriteResultPOperator.java | 5 +++++ .../rewriter/rules/ExtractCommonOperatorsRule.java | 3 ++- .../hyracks/dataflow/std/misc/MaterializerTaskState.java | 4 ++++ .../dataflow/std/misc/SplitOperatorDescriptor.java | 8 ++++++++ 60 files changed, 134 insertions(+), 155 deletions(-) diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java index 377ab1a2c..d91cac057 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java @@ -56,13 +56,6 @@ public interface ILogicalOperator { public boolean isMap(); - /* - * This is needed to have a kind of cost based decision on whether to merge the shared subplans and materialize the result. - * If the subgraph whose result we would like to materialize has an operator that is computationally expensive, we assume - * it is cheaper to materialize the result of this subgraph and read from the file rather than recomputing it. - */ - public boolean expensiveThanMaterialization(); - public Map getAnnotations(); public void removeAnnotation(String annotationName); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java index 6da26d2bd..dafe300b1 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java @@ -64,4 +64,12 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext * The edges labeled as 1 must wait for the edges with label 0. */ public Pair getInputOutputDependencyLabels(ILogicalOperator op); + + /* + * This is needed to have a kind of cost based decision on whether to merge the shared subplans and materialize the result. + * If the subgraph whose result we would like to materialize has an operator that is computationally expensive, we assume + * it is cheaper to materialize the result of this subgraph and read from the file rather than recomputing it. + */ + public boolean expensiveThanMaterialization(); + } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java index 56e690472..6c00412f9 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java @@ -75,9 +75,4 @@ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform vi public boolean isMap() { return false; } - - @Override - public boolean expensiveThanMaterialization() { - return true; - } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java index d313dce0e..417c959eb 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java @@ -83,11 +83,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - public abstract void getUsedVariablesExceptNestedPlans(Collection vars); public abstract void getProducedVariablesExceptNestedPlans(Collection vars); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java index 99c8534b8..371da4cc9 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java @@ -70,11 +70,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java index a7374fbaa..0af3dbc12 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java @@ -76,11 +76,6 @@ public boolean isMap() { return true; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java index c5abf0340..ba51f047a 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java @@ -72,11 +72,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - public void addProjectVariables(Collection vars) { projectVars.addAll(vars); projectPushed = true; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java index 52e199f64..3d8c4b6ba 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java @@ -99,11 +99,6 @@ public R accept(ILogicalOperatorVisitor visitor, T arg) throws Alge @Override public boolean isMap() { return false; - } - - @Override - public boolean expensiveThanMaterialization() { - return true; } public List getDistinctByVarList() { diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java index a4632fa01..3e28d7329 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java @@ -79,11 +79,6 @@ public boolean isMap() { return false; // actually depends on the physical op. } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java index b09846861..5872e6d64 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java @@ -60,11 +60,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException { return new IVariableTypeEnvironment() { diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java index 1ea070113..a3c26505e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java @@ -68,11 +68,6 @@ public boolean isMap() { return true; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java index b52e5d58d..bdf6f1c17 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java @@ -64,11 +64,6 @@ public boolean isMap() { return this.delegate.isMap(); } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java index 75baf73fd..e18c46155 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java @@ -76,11 +76,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - public boolean isPropagateInput() { return propagateInput; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java index fa061b877..ea45cc624 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java @@ -82,11 +82,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java index d71db1fd3..94fd19705 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java @@ -79,11 +79,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java index 4f36bc6af..79145f417 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java @@ -105,11 +105,6 @@ public boolean isMap() { return true; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java index 3ce2e0dcf..be23f48e2 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java @@ -81,11 +81,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException { ITypeEnvPointer[] p = new ITypeEnvPointer[1]; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java index 99f2cb2d6..335fb36ce 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java @@ -152,11 +152,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java index b6f1d83b9..a6c8f6f95 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java @@ -107,11 +107,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java index 6ac4c0ef2..bcba2800b 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java @@ -71,11 +71,6 @@ public boolean isMap() { return true; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - public List getVariables() { return variables; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java index d3715b608..3f1397754 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java @@ -72,11 +72,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public void recomputeSchema() { schema = new ArrayList(inputs.get(0).getValue().getSchema()); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java index 1c904b992..4f608bbae 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java @@ -63,11 +63,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java index 3fa9ef656..46e98595f 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java @@ -88,11 +88,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public void recomputeSchema() { this.schema = outputVariables; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java index db7fe054e..5aa229c71 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java @@ -89,11 +89,6 @@ public boolean isMap() { return true; } - @Override - public boolean expensiveThanMaterialization() { - return true; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { ITypeEnvPointer[] envPointers = new ITypeEnvPointer[1]; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java index 471848885..bdad6d488 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java @@ -51,11 +51,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public VariablePropagationPolicy getVariablePropagationPolicy() { return VariablePropagationPolicy.ALL; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java index 0f470b997..97a77ecf0 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java @@ -99,11 +99,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java index e9cc1fd8e..b7a56bec8 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java @@ -131,9 +131,4 @@ public boolean isMap() { return !propagateInput; } */ - - @Override - public boolean expensiveThanMaterialization() { - return true; - } } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java index 9384b16c5..dba9e19bf 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java @@ -141,9 +141,4 @@ public void propagateVariables(IOperatorSchema target, IOperatorSchema... source } }; } - - @Override - public boolean expensiveThanMaterialization() { - return false; - } } \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java index f1bbe5d82..841f9c61c 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java @@ -48,11 +48,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public VariablePropagationPolicy getVariablePropagationPolicy() { // TODO Auto-generated method stub diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java index 90f699c94..e1c16403c 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java @@ -79,11 +79,6 @@ public boolean isMap() { return false; // actually depends on the physical op. } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java index 02e5d965a..ba402bcba 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java @@ -87,11 +87,6 @@ public boolean isMap() { return false; } - @Override - public boolean expensiveThanMaterialization() { - return false; - } - @Override public void recomputeSchema() { schema = new ArrayList(); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java index 25fb453ad..42d964d75 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java @@ -40,6 +40,11 @@ public boolean isMicroOperator() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } + public abstract Pair createConnectorDescriptor( IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException; diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java index bd24b02f7..fcc04ab92 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java @@ -46,4 +46,9 @@ public Pair getInputOutputDependencyLabels(ILogicalOperator op) { int[] outputDependencyLabels = new int[] { 1 }; return new Pair(inputDependencyLabels, outputDependencyLabels); } + + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java index c375a3e14..74adf5bf7 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java @@ -249,4 +249,8 @@ private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable return null; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java index 18b442bbf..299f51951 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java @@ -26,4 +26,8 @@ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op return emptyUnaryRequirements(); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java index 047213466..8cfe067e6 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java @@ -133,4 +133,9 @@ public Pair getInputOutputDependencyLabels(ILogicalOperator op) { int[] outputDependencyLabels = new int[] { 1 }; return new Pair(inputDependencyLabels, outputDependencyLabels); } + + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java index 99aec265d..8b1b447c6 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java @@ -114,4 +114,8 @@ public boolean isMicroOperator() { return true; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java index 55da00ecd..3e87281bc 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java @@ -99,4 +99,8 @@ public void setRapidFrameFlush(boolean flushFramesRapidly) { this.flushFramesRapidly = flushFramesRapidly; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java index 1ba07be6f..54fdf2c35 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java @@ -106,4 +106,9 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext ILogicalOperator src = resultOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, resultOp, 0); } + + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java index 107aebd10..b71497977 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java @@ -65,4 +65,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeMicroOperator(op, runtime, recDesc); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java index 0358219f5..fe438a586 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java @@ -263,4 +263,9 @@ public Pair getInputOutputDependencyLabels(ILogicalOperator op) { int[] outputDependencyLabels = new int[] { 1 }; return new Pair(inputDependencyLabels, outputDependencyLabels); } + + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java index 0c07c4cf1..b56d638ae 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java @@ -124,4 +124,8 @@ public boolean isMicroOperator() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java index 582991146..d25415132 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java @@ -114,4 +114,8 @@ public boolean isMicroOperator() { return false; } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java index f2b52daaf..f68b32772 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java @@ -71,4 +71,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeMicroOperator(op, runtime, recDesc); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java index cd99155fb..3f7879302 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java @@ -136,4 +136,8 @@ public PhysicalOperatorTag getOperatorTag() { return PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java index c433f5c31..d8f7d99ca 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java @@ -87,4 +87,9 @@ public Pair getInputOutputDependencyLabels(ILogicalOperator op) { } return new Pair(inputDependencyLabels, outputDependencyLabels); } + + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java index 956c74f9b..3b2387ac7 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java @@ -30,8 +30,6 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty; -import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector; @@ -99,4 +97,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java index c5fa192fa..aa24221f6 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java @@ -64,4 +64,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeGraphEdge(src, 0, op, 0); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java index fea9701f2..9105c1fc3 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java @@ -104,4 +104,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeGraphEdge(src, 0, write, 0); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java index 8416a8174..48a0fa6ad 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java @@ -276,4 +276,9 @@ public Pair getInputOutputDependencyLabels(ILogicalOperator op) { int[] outputDependencyLabels = new int[] { 1 }; return new Pair(inputDependencyLabels, outputDependencyLabels); } + + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java index 324709afc..63ddffe60 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java @@ -100,4 +100,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeGraphEdge(src, 0, limit, 0); } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java index fa7d34ec4..2ad5fdab0 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java @@ -81,4 +81,8 @@ public void setRapidFrameFlush(boolean flushFramesRapidly) { this.flushFramesRapidly = flushFramesRapidly; } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java index bbf8e9ae9..3999fa6ba 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java @@ -73,4 +73,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeGraphEdge(src, 0, select, 0); } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java index 39271bb36..e6a0be712 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java @@ -75,4 +75,8 @@ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext computeDeliveredPropertiesForUsedVariables(s, s.getInputVariables()); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java index 28df9fe02..efc609af0 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java @@ -104,4 +104,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeGraphEdge(src, 0, op, 0); } + @Override + public boolean expensiveThanMaterialization() { + return true; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java index 7fb6ddcdf..cda3b0096 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java @@ -98,4 +98,8 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext builder.contributeGraphEdge(src2, 0, op, 1); } + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java index 8b241d35d..f399184e4 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java @@ -103,4 +103,9 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext ILogicalOperator src = writeResultOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, writeResultOp, 0); } + + @Override + public boolean expensiveThanMaterialization() { + return false; + } } diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index 2545a1b23..691006fc2 100644 --- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -488,7 +488,8 @@ private void computeClusters(Mutable parentRef, Mutable candidate) { - if (candidate.getValue().expensiveThanMaterialization()) { + AbstractLogicalOperator aop = (AbstractLogicalOperator) candidate.getValue(); + if (aop.getPhysicalOperator().expensiveThanMaterialization()) { return true; } List> inputs = candidate.getValue().getInputs(); diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java index ee857aba4..48de837c2 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializerTaskState.java @@ -79,4 +79,8 @@ public void writeOut(IFrameWriter writer, ByteBuffer frame) throws HyracksDataEx writer.close(); } } + + public void deleteFile() { + out.getFileReference().delete(); + } } \ No newline at end of file diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java index 990dfb832..b8e1ac81d 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java @@ -41,6 +41,7 @@ public class SplitOperatorDescriptor extends AbstractOperatorDescriptor { private boolean[] outputMaterializationFlags; private boolean requiresMaterialization; private int numberOfNonMaterializedOutputs = 0; + private int numberOfActiveMaterializeReaders = 0; public SplitOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc, int outputArity) { this(spec, rDesc, outputArity, new boolean[outputArity]); @@ -87,6 +88,7 @@ public void contributeActivities(IActivityGraphBuilder builder) { builder.addActivity(this, mra); builder.addTargetEdge(i, mra, 0); builder.addBlockingEdge(sma, mra); + numberOfActiveMaterializeReaders++; activityId++; } } @@ -178,6 +180,12 @@ public void initialize() throws HyracksDataException { @Override public void deinitialize() throws HyracksDataException { + numberOfActiveMaterializeReaders--; + MaterializerTaskState state = (MaterializerTaskState) ctx.getStateObject(new TaskId(new ActivityId( + getOperatorId(), SPLITTER_MATERIALIZER_ACTIVITY_ID), partition)); + if (numberOfActiveMaterializeReaders == 0) { + state.deleteFile(); + } } }; } From b61dee62ad5de1a243af64482bcc9568eaf2a9e1 Mon Sep 17 00:00:00 2001 From: icetindil Date: Thu, 7 Aug 2014 16:12:28 -0700 Subject: [PATCH 13/15] fixed issue 794 --- .../expressions/INullableTypeComputer.java | 4 +++ .../expressions/IVariableTypeEnvironment.java | 3 +- .../logical/AbstractLogicalOperator.java | 3 +- .../operators/logical/AssignOperator.java | 14 +++++++- .../logical/EmptyTupleSourceOperator.java | 4 +-- .../logical/LeftOuterJoinOperator.java | 13 ++++++-- .../properties/TypePropagationPolicy.java | 24 +++++++++++--- .../typing/NonPropagatingTypeEnvironment.java | 3 +- ...ropagateOperatorInputsTypeEnvironment.java | 14 ++++---- .../typing/PropagatingTypeEnvironment.java | 32 +++++++++++++++---- .../expression/HiveNullableTypeComputer.java | 9 ++++++ 11 files changed, 96 insertions(+), 27 deletions(-) diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java index 877d36367..d3f6d19d0 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java @@ -18,4 +18,8 @@ public interface INullableTypeComputer { public Object makeNullableType(Object type) throws AlgebricksException; + + public boolean canBeNull(Object type); + + public Object getNonOptionalType(Object type); } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java index 3ba438a32..8bcc575ab 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java @@ -23,7 +23,8 @@ public interface IVariableTypeEnvironment { public Object getVarType(LogicalVariable var) throws AlgebricksException; - public Object getVarType(LogicalVariable var, List nonNullVariables) throws AlgebricksException; + public Object getVarType(LogicalVariable var, List nonNullVariables, + List> correlatedNullableVariableLists) throws AlgebricksException; public void setVarType(LogicalVariable var, Object type); diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java index 78efd0db8..d4459bc2d 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java @@ -173,8 +173,7 @@ public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) return createPropagatingAllInputsTypeEnvironment(ctx); } - protected IVariableTypeEnvironment createPropagatingAllInputsTypeEnvironment(ITypingContext ctx) { - // return createPropagatingAllInputsTypeEnvironment(ctx); + protected PropagatingTypeEnvironment createPropagatingAllInputsTypeEnvironment(ITypingContext ctx) { int n = inputs.size(); ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n]; for (int i = 0; i < n; i++) { diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java index 0af3dbc12..0d40307ee 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java @@ -20,11 +20,14 @@ import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy; import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext; +import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment; import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; /** @@ -78,13 +81,22 @@ public boolean isMap() { @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { - IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); + PropagatingTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); int n = variables.size(); for (int i = 0; i < n; i++) { env.setVarType( variables.get(i), ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(), env)); + if (expressions.get(i).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) { + LogicalVariable var = ((VariableReferenceExpression) expressions.get(i).getValue()) + .getVariableReference(); + for (List list : env.getCorrelatedNullableVariableLists()) { + if (list.contains(var)) { + list.add(variables.get(i)); + } + } + } } return env; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java index 5872e6d64..7d16ed1f5 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java @@ -80,8 +80,8 @@ public Object getType(ILogicalExpression expr) throws AlgebricksException { } @Override - public Object getVarType(LogicalVariable var, List nonNullVariables) - throws AlgebricksException { + public Object getVarType(LogicalVariable var, List nonNullVariables, + List> correlatedNullableVariableLists) throws AlgebricksException { return null; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java index 8c3709935..1504039c1 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java @@ -14,13 +14,18 @@ */ package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical; +import java.util.ArrayList; +import java.util.List; + import org.apache.commons.lang3.mutable.Mutable; import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; +import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy; import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer; import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext; @@ -56,8 +61,12 @@ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) for (int i = 0; i < n; i++) { envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx); } - return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(), - ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers); + PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), + ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers); + List liveVars = new ArrayList(); + VariableUtilities.getLiveVariables(inputs.get(1).getValue(), liveVars); // live variables from outer branch can be null together + env.getCorrelatedNullableVariableLists().add(liveVars); + return env; } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java index 16fed48b2..29d496c1a 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java @@ -27,14 +27,26 @@ public abstract class TypePropagationPolicy { @Override public Object getVarType(LogicalVariable var, INullableTypeComputer ntc, - List nonNullVariableList, ITypeEnvPointer... typeEnvs) throws AlgebricksException { + List nonNullVariableList, List> correlatedNullableVariableLists, + ITypeEnvPointer... typeEnvs) throws AlgebricksException { for (ITypeEnvPointer p : typeEnvs) { IVariableTypeEnvironment env = p.getTypeEnv(); if (env == null) { throw new AlgebricksException("Null environment for pointer " + p + " in getVarType for var=" + var); } - Object t = env.getVarType(var, nonNullVariableList); + Object t = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists); if (t != null) { + if (ntc.canBeNull(t)) { + for (List list : correlatedNullableVariableLists) { + if (list.contains(var)) { + for (LogicalVariable v : list) { + if (nonNullVariableList.contains(v)) { + return ntc.getNonOptionalType(t); + } + } + } + } + } return t; } } @@ -46,10 +58,12 @@ public Object getVarType(LogicalVariable var, INullableTypeComputer ntc, @Override public Object getVarType(LogicalVariable var, INullableTypeComputer ntc, - List nonNullVariableList, ITypeEnvPointer... typeEnvs) throws AlgebricksException { + List nonNullVariableList, List> correlatedNullableVariableLists, + ITypeEnvPointer... typeEnvs) throws AlgebricksException { int n = typeEnvs.length; for (int i = 0; i < n; i++) { - Object t = typeEnvs[i].getTypeEnv().getVarType(var, nonNullVariableList); + Object t = typeEnvs[i].getTypeEnv().getVarType(var, nonNullVariableList, + correlatedNullableVariableLists); if (t != null) { if (i == 0) { // inner branch return t; @@ -78,5 +92,5 @@ public Object getVarType(LogicalVariable var, INullableTypeComputer ntc, }; public abstract Object getVarType(LogicalVariable var, INullableTypeComputer ntc, - List nonNullVariableList, ITypeEnvPointer... typeEnvs) throws AlgebricksException; + List nonNullVariableList, List> correlatedNullableVariableLists, ITypeEnvPointer... typeEnvs) throws AlgebricksException; } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java index 7e744dd71..eb0eb7102 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java @@ -34,7 +34,8 @@ public Object getVarType(LogicalVariable var) throws AlgebricksException { } @Override - public Object getVarType(LogicalVariable var, List nonNullVariables) throws AlgebricksException { + public Object getVarType(LogicalVariable var, List nonNullVariables, + List> correlatedNullableVariableLists) throws AlgebricksException { return getVarType(var); } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java index 71bc891c1..87ee385c1 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java @@ -29,6 +29,7 @@ public class PropagateOperatorInputsTypeEnvironment extends AbstractTypeEnvironment { private final List nonNullVariables = new ArrayList(); + private final List> correlatedNullableVariableLists = new ArrayList>(); private final ILogicalOperator op; private final ITypingContext ctx; @@ -44,13 +45,14 @@ public List getNonNullVariables() { } @Override - public Object getVarType(LogicalVariable var, List nonNullVariableList) throws AlgebricksException { + public Object getVarType(LogicalVariable var, List nonNullVariableList, + List> correlatedNullableVariableLists) throws AlgebricksException { nonNullVariableList.addAll(nonNullVariables); - return getVarTypeFullList(var, nonNullVariableList); + return getVarTypeFullList(var, nonNullVariableList, correlatedNullableVariableLists); } - private Object getVarTypeFullList(LogicalVariable var, List nonNullVariableList) - throws AlgebricksException { + private Object getVarTypeFullList(LogicalVariable var, List nonNullVariableList, + List> correlatedNullableVariableLists) throws AlgebricksException { Object t = varTypeMap.get(var); if (t != null) { return t; @@ -58,7 +60,7 @@ private Object getVarTypeFullList(LogicalVariable var, List non for (Mutable r : op.getInputs()) { ILogicalOperator c = r.getValue(); IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(c); - Object t2 = env.getVarType(var, nonNullVariableList); + Object t2 = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists); if (t2 != null) { return t2; } @@ -68,7 +70,7 @@ private Object getVarTypeFullList(LogicalVariable var, List non @Override public Object getVarType(LogicalVariable var) throws AlgebricksException { - return getVarTypeFullList(var, nonNullVariables); + return getVarTypeFullList(var, nonNullVariables, correlatedNullableVariableLists); } } diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java index 429457100..d167cd7a6 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java @@ -34,6 +34,8 @@ public class PropagatingTypeEnvironment extends AbstractTypeEnvironment { private final List nonNullVariables = new ArrayList(); + private final List> correlatedNullableVariableLists = new ArrayList>(); + public PropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer, IMetadataProvider metadataProvider, TypePropagationPolicy policy, ITypeEnvPointer[] envPointers) { @@ -45,25 +47,41 @@ public PropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer @Override public Object getVarType(LogicalVariable var) throws AlgebricksException { - return getVarTypeFullList(var, nonNullVariables); + return getVarTypeFullList(var, nonNullVariables, correlatedNullableVariableLists); } public List getNonNullVariables() { return nonNullVariables; } + public List> getCorrelatedNullableVariableLists() { + return correlatedNullableVariableLists; + } + @Override - public Object getVarType(LogicalVariable var, List nonNullVariableList) throws AlgebricksException { - nonNullVariableList.addAll(nonNullVariables); - return getVarTypeFullList(var, nonNullVariableList); + public Object getVarType(LogicalVariable var, List nonNullVariableList, + List> correlatedNullableVariableLists) throws AlgebricksException { + for (LogicalVariable v : nonNullVariables) { + if (!nonNullVariableList.contains(v)) { + nonNullVariableList.add(v); + } + } + Object t = getVarTypeFullList(var, nonNullVariableList, correlatedNullableVariableLists); + for (List list : this.correlatedNullableVariableLists) { + if (!correlatedNullableVariableLists.contains(list)) { + correlatedNullableVariableLists.add(list); + } + } + return t; } - private Object getVarTypeFullList(LogicalVariable var, List nonNullVariableList) - throws AlgebricksException { + private Object getVarTypeFullList(LogicalVariable var, List nonNullVariableList, + List> correlatedNullableVariableLists) throws AlgebricksException { Object t = varTypeMap.get(var); if (t != null) { return t; } - return policy.getVarType(var, nullableTypeComputer, nonNullVariableList, envPointers); + return policy.getVarType(var, nullableTypeComputer, nonNullVariableList, correlatedNullableVariableLists, + envPointers); } } diff --git a/hivesterix/hivesterix-common/src/main/java/edu/uci/ics/hivesterix/logical/expression/HiveNullableTypeComputer.java b/hivesterix/hivesterix-common/src/main/java/edu/uci/ics/hivesterix/logical/expression/HiveNullableTypeComputer.java index 75e23a71f..79ee90743 100644 --- a/hivesterix/hivesterix-common/src/main/java/edu/uci/ics/hivesterix/logical/expression/HiveNullableTypeComputer.java +++ b/hivesterix/hivesterix-common/src/main/java/edu/uci/ics/hivesterix/logical/expression/HiveNullableTypeComputer.java @@ -26,4 +26,13 @@ public Object makeNullableType(Object type) throws AlgebricksException { return type; } + @Override + public boolean canBeNull(Object type) { + return false; + } + + @Override + public Object getNonOptionalType(Object type) { + return type; + } } From ded9af79daaacd27017f7cd30b4ceb2799a6bba0 Mon Sep 17 00:00:00 2001 From: icetindil Date: Thu, 7 Aug 2014 17:55:13 -0700 Subject: [PATCH 14/15] fixed NPE for piglet examples --- .../core/algebra/properties/TypePropagationPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java index 29d496c1a..e1824d65e 100644 --- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java +++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java @@ -36,7 +36,7 @@ public Object getVarType(LogicalVariable var, INullableTypeComputer ntc, } Object t = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists); if (t != null) { - if (ntc.canBeNull(t)) { + if (ntc != null && ntc.canBeNull(t)) { for (List list : correlatedNullableVariableLists) { if (list.contains(var)) { for (LogicalVariable v : list) { From 0e868b53ab5c301c9c953c100b6f8295fbdb21c9 Mon Sep 17 00:00:00 2001 From: Ian Maxon Date: Mon, 19 Oct 2015 16:38:17 -0700 Subject: [PATCH 15/15] Rename rule --- .../algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename algebricks/algebricks-rewriter/src/main/java/{edu/uci/ics => org/apache}/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java (100%) diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java similarity index 100% rename from algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java rename to algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignDownThroughJoinRule.java