From d373b7fc3251be501bae3c17394c124699e9ea50 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Fri, 31 Oct 2025 11:25:00 +0000 Subject: [PATCH] feat: Add RemoveInsensitiveAggregateDistinct optimizer rule --- .../builtin/tools/WorkerFunctionUtil.java | 2 +- ...actSqlInvokedFunctionNamespaceManager.java | 1 + .../HiveAggregationFunctionDescription.java | 10 +- ...HiveAggregationFunctionImplementation.java | 6 + ...regationFunctionImplementationFactory.java | 3 +- ...uiltInSpecialFunctionNamespaceManager.java | 1 + .../AbstractMinMaxAggregationFunction.java | 2 +- .../AbstractMinMaxNAggregationFunction.java | 2 +- .../AggregationFromAnnotationsParser.java | 2 + .../aggregation/AggregationHeader.java | 9 +- .../AlternativeApproxPercentile.java | 2 +- .../ArbitraryAggregationFunction.java | 2 +- .../aggregation/BitwiseAndAggregation.java | 2 +- .../aggregation/BitwiseOrAggregation.java | 2 +- .../aggregation/BooleanAndAggregation.java | 2 +- .../aggregation/BooleanOrAggregation.java | 2 +- ...ltInAggregationFunctionImplementation.java | 10 + .../ChecksumAggregationFunction.java | 2 +- .../operator/aggregation/CountColumn.java | 2 +- .../DecimalAverageAggregation.java | 2 +- .../aggregation/DecimalSumAggregation.java | 2 +- .../aggregation/MapAggregationFunction.java | 2 +- .../aggregation/MapUnionAggregation.java | 2 +- .../aggregation/MapUnionSumAggregation.java | 2 +- .../MergeStatisticalDigestFunction.java | 2 +- .../aggregation/ParametricAggregation.java | 1 + .../aggregation/RealAverageAggregation.java | 1 + .../ReduceAggregationFunction.java | 1 + .../StatisticalDigestAggregationFunction.java | 2 +- .../arrayagg/ArrayAggregationFunction.java | 2 +- .../arrayagg/SetAggregationFunction.java | 6 +- .../arrayagg/SetUnionFunction.java | 6 +- .../aggregation/histogram/Histogram.java | 2 +- .../minmaxby/AbstractMinMaxBy.java | 2 +- .../AbstractMinMaxByNAggregationFunction.java | 2 +- .../MultimapAggregationFunction.java | 2 +- .../NoisyAverageGaussianAggregation.java | 2 +- ...isyAverageGaussianClippingAggregation.java | 2 +- ...GaussianClippingRandomSeedAggregation.java | 2 +- ...yAverageGaussianRandomSeedAggregation.java | 2 +- .../NoisyCountGaussianColumnAggregation.java | 2 +- ...ntGaussianColumnRandomSeedAggregation.java | 2 +- .../NoisySumGaussianAggregation.java | 2 +- .../NoisySumGaussianClippingAggregation.java | 2 +- ...GaussianClippingRandomSeedAggregation.java | 2 +- ...NoisySumGaussianRandomSeedAggregation.java | 2 +- .../presto/sql/planner/PlanOptimizers.java | 4 + .../RemoveInsensitiveAggregateDistinct.java | 93 ++++++++++ ...perLogLogWithLimitAggregationFunction.java | 2 +- ...perLogLogWithLimitAggregationFunction.java | 2 +- .../presto/sql/TestExpressionInterpreter.java | 2 +- ...estRemoveInsensitiveAggregateDistinct.java | 174 ++++++++++++++++++ .../main/functions/FunctionMetadata.cpp | 2 + .../tests/data/ApproxMostFrequent.json | 21 ++- .../main/functions/tests/data/CovarSamp.json | 6 +- .../main/functions/tests/data/SetAgg.json | 3 +- .../main/functions/tests/data/StddevSamp.json | 15 +- .../main/functions/tests/data/Variance.json | 18 +- .../iceberg/presto_protocol_iceberg.cpp | 6 +- .../core/presto_protocol_core.cpp | 15 ++ .../core/presto_protocol_core.h | 1 + .../TestPrestoNativeBuiltInFunctions.java | 2 +- .../test/resources/external_functions.json | 12 +- .../NativeFunctionNamespaceManager.java | 1 + .../spi/function/AggregationFunction.java | 2 + .../AggregationFunctionImplementation.java | 2 + .../function/AggregationFunctionMetadata.java | 17 +- ...okedAggregationFunctionImplementation.java | 10 + 68 files changed, 454 insertions(+), 79 deletions(-) create mode 100644 presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveInsensitiveAggregateDistinct.java create mode 100644 presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveInsensitiveAggregateDistinct.java diff --git a/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java b/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java index d880429f7489f..75d40baa43685 100644 --- a/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java +++ b/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java @@ -62,7 +62,7 @@ public static synchronized SqlInvokedFunction createSqlInvokedFunction(String fu jsonBasedUdfFunctionMetaData.getAggregateMetadata() .map(metadata -> new AggregationFunctionMetadata( convertApplicableTypeToVariable(metadata.getIntermediateType()), - metadata.isOrderSensitive())); + metadata.isOrderSensitive(), metadata.isDistinctSensitive())); return new SqlInvokedFunction( qualifiedFunctionName, diff --git a/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java index e612926e7c85b..fb80fcfb3518a 100644 --- a/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java +++ b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java @@ -367,6 +367,7 @@ protected AggregationFunctionImplementation sqlInvokedFunctionToAggregationImple typeManager.getType(aggregationMetadata.getIntermediateType()), typeManager.getType(function.getSignature().getReturnType()), aggregationMetadata.isOrderSensitive(), + aggregationMetadata.isDistinctSensitive(), parameters); default: throw new IllegalStateException(format("Unknown function implementation type: %s", implementationType)); diff --git a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionDescription.java b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionDescription.java index 91ee527013bae..4aa1f7d5e0b4c 100644 --- a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionDescription.java +++ b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionDescription.java @@ -29,6 +29,7 @@ public class HiveAggregationFunctionDescription private final Type finalType; private final boolean decomposable; private final boolean orderSensitive; + private final boolean distinctSensitive; public HiveAggregationFunctionDescription( QualifiedObjectName name, @@ -36,7 +37,8 @@ public HiveAggregationFunctionDescription( List intermediateTypes, Type finalType, boolean decomposable, - boolean orderSensitive) + boolean orderSensitive, + boolean distinctSensitive) { this.name = requireNonNull(name); this.parameterTypes = requireNonNull(parameterTypes); @@ -44,6 +46,7 @@ public HiveAggregationFunctionDescription( this.finalType = requireNonNull(finalType); this.decomposable = decomposable; this.orderSensitive = orderSensitive; + this.distinctSensitive = distinctSensitive; } public String getName() @@ -75,4 +78,9 @@ public boolean isOrderSensitive() { return orderSensitive; } + + public boolean isDistinctSensitive() + { + return distinctSensitive; + } } diff --git a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementation.java b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementation.java index ab588acec6585..02f387738bd86 100644 --- a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementation.java +++ b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementation.java @@ -107,6 +107,12 @@ public boolean isOrderSensitive() return aggregationFunctionDescription.isOrderSensitive(); } + @Override + public boolean isDistinctSensitive() + { + return aggregationFunctionDescription.isDistinctSensitive(); + } + public AggregationMetadata getAggregationMetadata() { return aggregationMetadata; diff --git a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementationFactory.java b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementationFactory.java index 9610d6c901d66..829be43257032 100644 --- a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementationFactory.java +++ b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/aggregation/HiveAggregationFunctionImplementationFactory.java @@ -76,7 +76,8 @@ public HiveAggregationFunctionImplementation create() ImmutableList.of(intermediateType), outputType, true, - false); + false, + true); HiveAccumulatorInvoker invocationContext = new HiveAccumulatorInvoker( partialEvaluatorSupplier, diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java index 6a4e716076cd9..0e9ff9feb8d49 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java @@ -253,6 +253,7 @@ private AggregationFunctionImplementation sqlInvokedFunctionToAggregationImpleme typeManager.getType(aggregationMetadata.getIntermediateType()), typeManager.getType(function.getSignature().getReturnType()), aggregationMetadata.isOrderSensitive(), + aggregationMetadata.isDistinctSensitive(), parameters); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java index c146d3849dada..103dae8d14295 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java @@ -163,7 +163,7 @@ else if (type.getJavaType() == boolean.class) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(intermediateType), - type, true, false, metadata, accumulatorClass, groupedAccumulatorClass); + type, true, false, false, metadata, accumulatorClass, groupedAccumulatorClass); } protected AccumulatorStateSerializer getStateSerializer(Class stateInterface, DynamicClassLoader classLoader) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java index 15329ee8013b4..5d9daacb0dfa0 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java @@ -117,7 +117,7 @@ protected BuiltInAggregationFunctionImplementation generateAggregation(Type type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(intermediateType), outputType, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } public static void input(BlockComparator comparator, Type type, MinMaxNState state, Block block, long n, int blockIndex) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java index 696303dcb014c..a2e2b39547ffd 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java @@ -142,6 +142,7 @@ private static AggregationHeader parseHeader(AnnotatedElement aggregationDefinit parseDescription(aggregationDefinition), aggregationAnnotation.decomposable(), aggregationAnnotation.isOrderSensitive(), + aggregationAnnotation.isDistinctSensitive(), aggregationAnnotation.visibility(), aggregationAnnotation.isCalledOnNullInput()); } @@ -157,6 +158,7 @@ private static List parseHeaders(AnnotatedElement aggregation parseDescription(aggregationDefinition, toParse), aggregationAnnotation.decomposable(), aggregationAnnotation.isOrderSensitive(), + aggregationAnnotation.isDistinctSensitive(), aggregationAnnotation.visibility(), aggregationAnnotation.isCalledOnNullInput())) .collect(toImmutableList()); diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java index 876e757705ad1..478b7cc9b52c9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java @@ -25,15 +25,17 @@ public class AggregationHeader private final Optional description; private final boolean decomposable; private final boolean orderSensitive; + private final boolean distinctSensitive; private final SqlFunctionVisibility visibility; private final boolean isCalledOnNullInput; - public AggregationHeader(String name, Optional description, boolean decomposable, boolean orderSensitive, SqlFunctionVisibility visibility, boolean isCalledOnNullInput) + public AggregationHeader(String name, Optional description, boolean decomposable, boolean orderSensitive, boolean distinctSensitive, SqlFunctionVisibility visibility, boolean isCalledOnNullInput) { this.name = requireNonNull(name, "name cannot be null"); this.description = requireNonNull(description, "description cannot be null"); this.decomposable = decomposable; this.orderSensitive = orderSensitive; + this.distinctSensitive = distinctSensitive; this.visibility = visibility; this.isCalledOnNullInput = isCalledOnNullInput; } @@ -58,6 +60,11 @@ public boolean isOrderSensitive() return orderSensitive; } + public boolean isDistinctSensitive() + { + return distinctSensitive; + } + public SqlFunctionVisibility getVisibility() { return visibility; diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AlternativeApproxPercentile.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AlternativeApproxPercentile.java index 958cc893b7193..eed36cccd2bf7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AlternativeApproxPercentile.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AlternativeApproxPercentile.java @@ -120,7 +120,7 @@ public BuiltInAggregationFunctionImplementation specialize( new RowType.Field(Optional.of("levels"), new ArrayType(IntegerType.INTEGER)))); return new BuiltInAggregationFunctionImplementation( NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, false, null, null, null); + true, false, true, null, null, null); } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ArbitraryAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ArbitraryAggregationFunction.java index 5bdc95104f728..07eeebf349381 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ArbitraryAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ArbitraryAggregationFunction.java @@ -169,7 +169,7 @@ else if (type.getJavaType() == boolean.class) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), type, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, false, metadata, accumulatorClass, groupedAccumulatorClass); } protected Type overrideIntermediateType(Type inputType, Type defaultIntermediateType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseAndAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseAndAggregation.java index a8cfd21b6f6e4..e670475ad5efd 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseAndAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseAndAggregation.java @@ -24,7 +24,7 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; -@AggregationFunction("bitwise_and_agg") +@AggregationFunction(value = "bitwise_and_agg", isDistinctSensitive = false) public final class BitwiseAndAggregation { private BitwiseAndAggregation() {} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseOrAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseOrAggregation.java index 67b93e4b02d6e..716d001fc8530 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseOrAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BitwiseOrAggregation.java @@ -24,7 +24,7 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; -@AggregationFunction("bitwise_or_agg") +@AggregationFunction(value = "bitwise_or_agg", isDistinctSensitive = false) public class BitwiseOrAggregation { private BitwiseOrAggregation() {} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanAndAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanAndAggregation.java index 9b065b3287f13..b25f9de0d7b89 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanAndAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanAndAggregation.java @@ -28,7 +28,7 @@ import static com.facebook.presto.operator.aggregation.state.TriStateBooleanState.NULL_VALUE; import static com.facebook.presto.operator.aggregation.state.TriStateBooleanState.TRUE_VALUE; -@AggregationFunction(value = "bool_and", alias = "every") +@AggregationFunction(value = "bool_and", alias = "every", isDistinctSensitive = false) public final class BooleanAndAggregation { private BooleanAndAggregation() {} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanOrAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanOrAggregation.java index 24fb1cc3e38ea..90a2530b75f5f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanOrAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BooleanOrAggregation.java @@ -28,7 +28,7 @@ import static com.facebook.presto.operator.aggregation.state.TriStateBooleanState.NULL_VALUE; import static com.facebook.presto.operator.aggregation.state.TriStateBooleanState.TRUE_VALUE; -@AggregationFunction(value = "bool_or") +@AggregationFunction(value = "bool_or", isDistinctSensitive = false) public final class BooleanOrAggregation { private BooleanOrAggregation() {} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BuiltInAggregationFunctionImplementation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BuiltInAggregationFunctionImplementation.java index fcc749d43c427..f08923c0b3260 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BuiltInAggregationFunctionImplementation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/BuiltInAggregationFunctionImplementation.java @@ -37,6 +37,7 @@ public final class BuiltInAggregationFunctionImplementation private final List lambdaInterfaces; private final boolean decomposable; private final boolean orderSensitive; + private final boolean distinctSensitive; private final AggregationMetadata aggregationMetadata; @@ -51,6 +52,7 @@ public BuiltInAggregationFunctionImplementation( Type finalType, boolean decomposable, boolean orderSensitive, + boolean distinctSensitive, AggregationMetadata aggregationMetadata, Class accumulatorClass, Class groupedAccumulatorClass) @@ -62,6 +64,7 @@ public BuiltInAggregationFunctionImplementation( finalType, decomposable, orderSensitive, + distinctSensitive, aggregationMetadata, accumulatorClass, groupedAccumulatorClass, @@ -75,6 +78,7 @@ public BuiltInAggregationFunctionImplementation( Type finalType, boolean decomposable, boolean orderSensitive, + boolean distinctSensitive, AggregationMetadata aggregationMetadata, Class accumulatorClass, Class groupedAccumulatorClass, @@ -87,6 +91,7 @@ public BuiltInAggregationFunctionImplementation( this.finalType = requireNonNull(finalType, "finalType is null"); this.decomposable = decomposable; this.orderSensitive = orderSensitive; + this.distinctSensitive = distinctSensitive; this.aggregationMetadata = aggregationMetadata; this.accumulatorClass = accumulatorClass; this.groupedAccumulatorClass = groupedAccumulatorClass; @@ -139,6 +144,11 @@ public boolean isOrderSensitive() return orderSensitive; } + public boolean isDistinctSensitive() + { + return distinctSensitive; + } + public AggregationMetadata getAggregationMetadata() { return aggregationMetadata; diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ChecksumAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ChecksumAggregationFunction.java index 47ab886b844f5..6453a86545263 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ChecksumAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ChecksumAggregationFunction.java @@ -106,7 +106,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(BIGINT), VARBINARY, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/CountColumn.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/CountColumn.java index d21d66917b98e..2e2d4d3b86f09 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/CountColumn.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/CountColumn.java @@ -107,7 +107,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), BIGINT, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalAverageAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalAverageAggregation.java index 0753fa911c181..92b87b1522ac2 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalAverageAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalAverageAggregation.java @@ -146,7 +146,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), type, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalSumAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalSumAggregation.java index 877cf97cb3c08..16e2546e6a92f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalSumAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/DecimalSumAggregation.java @@ -135,7 +135,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapAggregationFunction.java index 1e68c0c3814bd..d8c6cf43f6941 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapAggregationFunction.java @@ -111,7 +111,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + true, true, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type keyType, Type valueType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionAggregation.java index 6f96f6842e8ad..5ad2c6d68e61f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionAggregation.java @@ -104,7 +104,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + true, true, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type inputType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionSumAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionSumAggregation.java index 87757e7906b0c..fa79c7ad807bb 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionSumAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MapUnionSumAggregation.java @@ -110,7 +110,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type inputType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java index a47776c97c954..3f9ab4b8c9fc2 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java @@ -116,7 +116,7 @@ private BuiltInAggregationFunctionImplementation generateAggregation(Type type) metadata, classLoader); return new BuiltInAggregationFunctionImplementation(name, ImmutableList.of(type), ImmutableList.of(intermediateType), type, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + true, true, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type valueType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java index c61b9b69d46d5..fdcfdac1930a6 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java @@ -127,6 +127,7 @@ public BuiltInAggregationFunctionImplementation specialize(BoundVariables variab outputType, details.isDecomposable(), details.isOrderSensitive(), + details.isDistinctSensitive(), metadata, accumulatorClass, groupedAccumulatorClass); diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/RealAverageAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/RealAverageAggregation.java index 828a31798b7cf..0c4cee3ae3422 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/RealAverageAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/RealAverageAggregation.java @@ -114,6 +114,7 @@ public BuiltInAggregationFunctionImplementation specialize(BoundVariables boundV REAL, true, false, + true, metadata, accumulatorClass, groupedAccumulatorClass); diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java index 8312d36ce6410..a73061a666f3d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java @@ -143,6 +143,7 @@ private BuiltInAggregationFunctionImplementation generateAggregation(Type inputT stateType, true, false, + true, metadata, accumulatorClass, groupedAccumulatorClass, diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java index ba2cfe3b4cacb..67c722b407dc7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java @@ -161,7 +161,7 @@ private BuiltInAggregationFunctionImplementation generateAggregation(String name metadata, classLoader); return new BuiltInAggregationFunctionImplementation(name, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + true, true, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List getInputTypes(Type valueType, int arity) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/ArrayAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/ArrayAggregationFunction.java index 438f199f09ca6..dd9509ec8ac45 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/ArrayAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/ArrayAggregationFunction.java @@ -121,7 +121,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + true, true, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type value, boolean legacyArrayAgg) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetAggregationFunction.java index 3158ab14d83dd..c68ce5594e522 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetAggregationFunction.java @@ -114,10 +114,8 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type GroupedAccumulator.class, metadata, classLoader); - return new - - BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, + true, true, false, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type valueType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetUnionFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetUnionFunction.java index 1d4547dc937f0..f7af827bc07c2 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetUnionFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/arrayagg/SetUnionFunction.java @@ -115,10 +115,8 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type GroupedAccumulator.class, metadata, classLoader); - return new - - BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, + true, true, false, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type valueType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/histogram/Histogram.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/histogram/Histogram.java index 613c7e505a6b0..c08e4dd8dbb2a 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/histogram/Histogram.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/histogram/Histogram.java @@ -116,7 +116,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation( metadata, classLoader); return new BuiltInAggregationFunctionImplementation(functionName, inputTypes, ImmutableList.of(intermediateType), - outputType, true, false, metadata, accumulatorClass, groupedAccumulatorClass); + outputType, true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type keyType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java index 32d5edd92a5b0..9645773e07417 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java @@ -176,7 +176,7 @@ private BuiltInAggregationFunctionImplementation generateAggregation(Type valueT metadata, classLoader); return new BuiltInAggregationFunctionImplementation(getSignature().getNameSuffix(), inputTypes, - ImmutableList.of(intermediateType), valueType, true, false, metadata, + ImmutableList.of(intermediateType), valueType, true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxByNAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxByNAggregationFunction.java index 4d5d20a12e40a..0a3ffad7f00bf 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxByNAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxByNAggregationFunction.java @@ -193,6 +193,6 @@ protected BuiltInAggregationFunctionImplementation generateAggregation(Type valu classLoader); return new BuiltInAggregationFunctionImplementation(name, inputTypes, ImmutableList.of(intermediateType), outputType, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/multimapagg/MultimapAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/multimapagg/MultimapAggregationFunction.java index a8a50bc1658c3..e58e4abe18bf7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/multimapagg/MultimapAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/multimapagg/MultimapAggregationFunction.java @@ -114,7 +114,7 @@ private BuiltInAggregationFunctionImplementation generateAggregation(Type keyTyp metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, - true, true, metadata, accumulatorClass, groupedAccumulatorClass); + true, true, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type keyType, Type valueType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianAggregation.java index 97d599fa01b26..e50cf0f267616 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianAggregation.java @@ -166,7 +166,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingAggregation.java index cabf3240c11a7..ed56bc291d7d7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingAggregation.java @@ -168,7 +168,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingRandomSeedAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingRandomSeedAggregation.java index 2e0c23826a525..e09a559e36e37 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingRandomSeedAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianClippingRandomSeedAggregation.java @@ -169,7 +169,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianRandomSeedAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianRandomSeedAggregation.java index 4c9aa8d1b5025..1d13bbdedc460 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianRandomSeedAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyAverageGaussianRandomSeedAggregation.java @@ -167,7 +167,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnAggregation.java index 90687d519193f..2f290d912f037 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnAggregation.java @@ -128,7 +128,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), BIGINT, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnRandomSeedAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnRandomSeedAggregation.java index a45783bcdf023..656a2eaa44950 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnRandomSeedAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyCountGaussianColumnRandomSeedAggregation.java @@ -112,7 +112,7 @@ private static BuiltInAggregationFunctionImplementation generateAggregation(Type metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), BIGINT, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianAggregation.java index fdc16db8a4d06..42c0525e2d909 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianAggregation.java @@ -166,7 +166,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingAggregation.java index 5309908832985..f0b0a3c52d1d9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingAggregation.java @@ -168,7 +168,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingRandomSeedAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingRandomSeedAggregation.java index 12dff6534b931..7887fc2ad4a80 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingRandomSeedAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianClippingRandomSeedAggregation.java @@ -169,7 +169,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianRandomSeedAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianRandomSeedAggregation.java index 3c2737afa5737..bb6109135e946 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianRandomSeedAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisySumGaussianRandomSeedAggregation.java @@ -167,7 +167,7 @@ else if (type instanceof RealType) { metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), DOUBLE, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static List createInputParameterMetadata(Type type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 2d7c8be053645..1caa5167e4625 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -111,6 +111,7 @@ import com.facebook.presto.sql.planner.iterative.rule.RemoveEmptyDelete; import com.facebook.presto.sql.planner.iterative.rule.RemoveFullSample; import com.facebook.presto.sql.planner.iterative.rule.RemoveIdentityProjectionsBelowProjection; +import com.facebook.presto.sql.planner.iterative.rule.RemoveInsensitiveAggregateDistinct; import com.facebook.presto.sql.planner.iterative.rule.RemoveMapCastRule; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantAggregateDistinct; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantCastToVarcharInJoinClause; @@ -424,6 +425,7 @@ public PlanOptimizers( new PushLimitThroughUnion(), new RemoveTrivialFilters(), new ImplementFilteredAggregations(metadata.getFunctionAndTypeManager()), + new RemoveInsensitiveAggregateDistinct(metadata.getFunctionAndTypeManager()), new SingleDistinctAggregationToGroupBy(), new MultipleDistinctAggregationToMarkDistinct(), new ImplementBernoulliSampleAsFilter(metadata.getFunctionAndTypeManager()), @@ -632,6 +634,7 @@ public PlanOptimizers( new RemoveRedundantSortColumns(), new RemoveRedundantLimit(), new RemoveRedundantDistinctLimit(), + new RemoveInsensitiveAggregateDistinct(metadata.getFunctionAndTypeManager()), new RemoveRedundantAggregateDistinct(), new RemoveRedundantIdentityProjections(), new PushAggregationThroughOuterJoin(metadata.getFunctionAndTypeManager()))), @@ -832,6 +835,7 @@ public PlanOptimizers( new RemoveRedundantSort(), new RemoveRedundantLimit(), new RemoveRedundantDistinctLimit(), + new RemoveInsensitiveAggregateDistinct(metadata.getFunctionAndTypeManager()), new RemoveRedundantAggregateDistinct(), new RemoveRedundantIdentityProjections(), new PushAggregationThroughOuterJoin(metadata.getFunctionAndTypeManager())))); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveInsensitiveAggregateDistinct.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveInsensitiveAggregateDistinct.java new file mode 100644 index 0000000000000..7b3e253c1b7ed --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveInsensitiveAggregateDistinct.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; + +public class RemoveInsensitiveAggregateDistinct + implements Rule +{ + private final Pattern pattern = aggregation() + .matching(this::canRemoveDistinct); + + private final FunctionAndTypeManager functionAndTypeManager; + + public RemoveInsensitiveAggregateDistinct(FunctionAndTypeManager functionAndTypeManager) + { + this.functionAndTypeManager = functionAndTypeManager; + } + + @Override + public Pattern getPattern() + { + return pattern; + } + + @Override + public Result apply(AggregationNode node, Captures captures, Context context) + { + ImmutableMap.Builder aggregations = ImmutableMap.builder(); + for (Map.Entry entry : node.getAggregations().entrySet()) { + Aggregation aggregation = entry.getValue(); + if (canRemoveDistinct(aggregation)) { + aggregations.put(entry.getKey(), + new Aggregation( + aggregation.getCall(), + aggregation.getFilter(), + aggregation.getOrderBy(), + false, + aggregation.getMask())); + } + else { + aggregations.put(entry); + } + } + return Result.ofPlanNode( + new AggregationNode( + node.getSourceLocation(), + node.getId(), + node.getSource(), + aggregations.build(), + node.getGroupingSets(), + node.getPreGroupedVariables(), + node.getStep(), + node.getHashVariable(), + node.getGroupIdVariable(), + node.getAggregationId())); + } + + private boolean canRemoveDistinct(AggregationNode aggregationNode) + { + return aggregationNode.getAggregations().values().stream() + .anyMatch(this::canRemoveDistinct); + } + + private boolean canRemoveDistinct(Aggregation aggregation) + { + return aggregation.isDistinct() + && (!functionAndTypeManager.getAggregateFunctionImplementation(aggregation.getFunctionHandle()).isDistinctSensitive()) + && (!functionAndTypeManager.getAggregateFunctionImplementation(aggregation.getFunctionHandle()).isOrderSensitive()); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogWithLimitAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogWithLimitAggregationFunction.java index d87b8fc69f2bc..465e206fd0f8d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogWithLimitAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogWithLimitAggregationFunction.java @@ -109,7 +109,7 @@ private BuiltInAggregationFunctionImplementation generateAggregation(Type firstI metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), K_HYPER_LOG_LOG, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } private static MethodHandle getMethodHandle(Type firstInputType, Type secondInputType) diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogWithLimitAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogWithLimitAggregationFunction.java index 3f692d9bb66b9..c169e230e571f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogWithLimitAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogWithLimitAggregationFunction.java @@ -121,7 +121,7 @@ public BuiltInAggregationFunctionImplementation specialize(BoundVariables boundV metadata, classLoader); return new BuiltInAggregationFunctionImplementation(NAME, inputTypes, ImmutableList.of(intermediateType), K_HYPER_LOG_LOG, - true, false, metadata, accumulatorClass, groupedAccumulatorClass); + true, false, true, metadata, accumulatorClass, groupedAccumulatorClass); } @Override diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main-base/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index c4d9d050dbc3d..ddaabcb8865f6 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -141,7 +141,7 @@ public class TestExpressionInterpreter "", notVersioned(), FunctionKind.AGGREGATE, - Optional.of(new AggregationFunctionMetadata(parseTypeSignature("ROW(double, int)"), false))); + Optional.of(new AggregationFunctionMetadata(parseTypeSignature("ROW(double, int)"), false, true))); private static final int TEST_VARCHAR_TYPE_LENGTH = 17; private static final TypeProvider SYMBOL_TYPES = TypeProvider.viewOf(ImmutableMap.builder() diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveInsensitiveAggregateDistinct.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveInsensitiveAggregateDistinct.java new file mode 100644 index 0000000000000..a036067c7c32e --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveInsensitiveAggregateDistinct.java @@ -0,0 +1,174 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.spi.plan.AggregationNode.Step; +import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert; +import com.facebook.presto.sql.tree.FunctionCall; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.globalAggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.symbol; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; + +public class TestRemoveInsensitiveAggregateDistinct + extends BaseRuleTest +{ + @Test + public void testNoDistinct() + { + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2"))) + .addAggregation(p.variable("output"), p.rowExpression("max(input1)")))) + .doesNotFire(); + } + + @Test + public void testSensitiveDistinct() + { + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2"))) + .addAggregation(p.variable("output"), p.rowExpression("count(input1)"), true))) + .doesNotFire(); + + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2"))) + .addAggregation(p.variable("output"), p.rowExpression("min(input1, 2)"), true))) + .doesNotFire(); + } + + @Test + public void testSensitiveOrder() + { + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2"))) + .addAggregation(p.variable("output"), p.rowExpression("set_agg(input1)"), true))) + .doesNotFire(); + + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1", new ArrayType(BIGINT)), p.variable("input2"))) + .addAggregation(p.variable("output"), p.rowExpression("set_union(input1)"), true))) + .doesNotFire(); + } + + @Test + public void testInsensitiveDistinct() + { + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2"))) + .addAggregation(p.variable("output"), p.rowExpression("max(input1)"), true))) + .matches( + aggregation( + globalAggregation(), + ImmutableMap.of( + Optional.of("output"), + functionCall("max", false, ImmutableList.of(symbol("input1")))), + ImmutableMap.of(), + Optional.empty(), + Step.SINGLE, + values("input1", "input2"))); + } + + @Test + public void testMultipleInsensitiveDistinct() + { + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2", BOOLEAN))) + .addAggregation(p.variable("output1"), p.rowExpression("any_value(input1)"), true) + .addAggregation(p.variable("output2"), p.rowExpression("arbitrary(input1)"), true) + .addAggregation(p.variable("output3"), p.rowExpression("bitwise_and_agg(input1)"), true) + .addAggregation(p.variable("output4"), p.rowExpression("bitwise_or_agg(input1)"), true) + .addAggregation(p.variable("output5"), p.rowExpression("bool_and(input2)"), true) + .addAggregation(p.variable("output6"), p.rowExpression("bool_or(input2)"), true) + .addAggregation(p.variable("output7"), p.rowExpression("every(input2)"), true) + .addAggregation(p.variable("output8"), p.rowExpression("max(input1)"), true) + .addAggregation(p.variable("output9"), p.rowExpression("min(input1)"), true))) + .matches( + aggregation( + globalAggregation(), + ImmutableMap., ExpectedValueProvider>builder() + .put(Optional.of("output1"), functionCall("any_value", false, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output2"), functionCall("arbitrary", false, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output3"), functionCall("bitwise_and_agg", false, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output4"), functionCall("bitwise_or_agg", false, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output5"), functionCall("bool_and", false, ImmutableList.of(symbol("input2")))) + .put(Optional.of("output6"), functionCall("bool_or", false, ImmutableList.of(symbol("input2")))) + .put(Optional.of("output7"), functionCall("every", false, ImmutableList.of(symbol("input2")))) + .put(Optional.of("output8"), functionCall("max", false, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output9"), functionCall("min", false, ImmutableList.of(symbol("input1")))) + .build(), + ImmutableMap.of(), + Optional.empty(), + Step.SINGLE, + values("input1", "input2"))); + } + + @Test + public void testMixedDistinct() + { + assertRuleApplication() + .on(p -> p.aggregation(builder -> builder + .globalGrouping() + .source(p.values(p.variable("input1"), p.variable("input2", BOOLEAN))) + .addAggregation(p.variable("output1"), p.rowExpression("count(input1)"), true) + .addAggregation(p.variable("output2"), p.rowExpression("sum(input1)"), true) + .addAggregation(p.variable("output3"), p.rowExpression("avg(input1)"), true) + .addAggregation(p.variable("output4"), p.rowExpression("max(input1)"), true) + .addAggregation(p.variable("output5"), p.rowExpression("min(input1)"), true))) + .matches( + aggregation( + globalAggregation(), + ImmutableMap., ExpectedValueProvider>builder() + .put(Optional.of("output1"), functionCall("count", true, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output2"), functionCall("sum", true, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output3"), functionCall("avg", true, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output4"), functionCall("max", false, ImmutableList.of(symbol("input1")))) + .put(Optional.of("output5"), functionCall("min", false, ImmutableList.of(symbol("input1")))) + .build(), + ImmutableMap.of(), + Optional.empty(), + Step.SINGLE, + values("input1", "input2"))); + } + + private RuleAssert assertRuleApplication() + { + return tester().assertThat(new RemoveInsensitiveAggregateDistinct(getFunctionManager())); + } +} diff --git a/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp b/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp index 761e3500b1fa1..1c267345e4e0f 100644 --- a/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp +++ b/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp @@ -63,6 +63,8 @@ const protocol::AggregationFunctionMetadata getAggregationFunctionMetadata( boost::algorithm::to_lower_copy(signature.intermediateType().toString()); metadata.isOrderSensitive = getAggregateFunctionEntry(name)->metadata.orderSensitive; + metadata.isDistinctSensitive = + !getAggregateFunctionEntry(name)->metadata.ignoreDuplicates; return metadata; } diff --git a/presto-native-execution/presto_cpp/main/functions/tests/data/ApproxMostFrequent.json b/presto-native-execution/presto_cpp/main/functions/tests/data/ApproxMostFrequent.json index 793e7e811e0b8..0d0e0ade6e377 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/data/ApproxMostFrequent.json +++ b/presto-native-execution/presto_cpp/main/functions/tests/data/ApproxMostFrequent.json @@ -3,7 +3,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(boolean),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", @@ -26,7 +27,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(tinyint),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", @@ -49,7 +51,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(smallint),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", @@ -72,7 +75,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(integer),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", @@ -95,7 +99,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(bigint),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", @@ -118,7 +123,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(varchar),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", @@ -141,7 +147,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,bigint,array(json),array(bigint))", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.approx_most_frequent", "functionKind": "AGGREGATE", diff --git a/presto-native-execution/presto_cpp/main/functions/tests/data/CovarSamp.json b/presto-native-execution/presto_cpp/main/functions/tests/data/CovarSamp.json index eb9125d444ad9..b56edcde84740 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/data/CovarSamp.json +++ b/presto-native-execution/presto_cpp/main/functions/tests/data/CovarSamp.json @@ -3,7 +3,8 @@ { "aggregateMetadata": { "intermediateType": "row(double,bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.covar_samp", "functionKind": "AGGREGATE", @@ -25,7 +26,8 @@ { "aggregateMetadata": { "intermediateType": "row(double,bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.covar_samp", "functionKind": "AGGREGATE", diff --git a/presto-native-execution/presto_cpp/main/functions/tests/data/SetAgg.json b/presto-native-execution/presto_cpp/main/functions/tests/data/SetAgg.json index f487aced8a942..478e39328cd44 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/data/SetAgg.json +++ b/presto-native-execution/presto_cpp/main/functions/tests/data/SetAgg.json @@ -3,7 +3,8 @@ { "aggregateMetadata": { "intermediateType": "array(t)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": false }, "docString": "presto.default.set_agg", "functionKind": "AGGREGATE", diff --git a/presto-native-execution/presto_cpp/main/functions/tests/data/StddevSamp.json b/presto-native-execution/presto_cpp/main/functions/tests/data/StddevSamp.json index 07f1f52222d6b..ad78769d7f329 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/data/StddevSamp.json +++ b/presto-native-execution/presto_cpp/main/functions/tests/data/StddevSamp.json @@ -3,7 +3,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.stddev_samp", "functionKind": "AGGREGATE", @@ -24,7 +25,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.stddev_samp", "functionKind": "AGGREGATE", @@ -45,7 +47,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.stddev_samp", "functionKind": "AGGREGATE", @@ -66,7 +69,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.stddev_samp", "functionKind": "AGGREGATE", @@ -87,7 +91,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.stddev_samp", "functionKind": "AGGREGATE", diff --git a/presto-native-execution/presto_cpp/main/functions/tests/data/Variance.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Variance.json index 633d5d89e95a4..b04be81c0258a 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/data/Variance.json +++ b/presto-native-execution/presto_cpp/main/functions/tests/data/Variance.json @@ -3,7 +3,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.variance", "functionKind": "AGGREGATE", @@ -24,7 +25,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.variance", "functionKind": "AGGREGATE", @@ -45,7 +47,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.variance", "functionKind": "AGGREGATE", @@ -66,7 +69,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.variance", "functionKind": "AGGREGATE", @@ -87,7 +91,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.variance", "functionKind": "AGGREGATE", @@ -108,7 +113,8 @@ { "aggregateMetadata": { "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true + "isOrderSensitive": true, + "isDistinctSensitive": true }, "docString": "presto.default.variance", "functionKind": "WINDOW", diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index 6d03a5ce52b12..d5dd246832bcf 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -512,10 +512,10 @@ static const std::pair PartitionTransformType_enum_table[] = { // NOLINT: cert-err58-cpp {PartitionTransformType::IDENTITY, "IDENTITY"}, - {PartitionTransformType::YEAR, "YEAR"}, - {PartitionTransformType::MONTH, "MONTH"}, - {PartitionTransformType::DAY, "DAY"}, {PartitionTransformType::HOUR, "HOUR"}, + {PartitionTransformType::DAY, "DAY"}, + {PartitionTransformType::MONTH, "MONTH"}, + {PartitionTransformType::YEAR, "YEAR"}, {PartitionTransformType::BUCKET, "BUCKET"}, {PartitionTransformType::TRUNCATE, "TRUNCATE"}}; void to_json(json& j, const PartitionTransformType& e) { diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index 874e2275577c1..0c533fe97688a 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -535,6 +535,13 @@ void to_json(json& j, const AggregationFunctionMetadata& p) { "AggregationFunctionMetadata", "bool", "isOrderSensitive"); + to_json_key( + j, + "isDistinctSensitive", + p.isDistinctSensitive, + "AggregationFunctionMetadata", + "bool", + "isDistinctSensitive"); } void from_json(const json& j, AggregationFunctionMetadata& p) { @@ -552,6 +559,13 @@ void from_json(const json& j, AggregationFunctionMetadata& p) { "AggregationFunctionMetadata", "bool", "isOrderSensitive"); + from_json_key( + j, + "isDistinctSensitive", + p.isDistinctSensitive, + "AggregationFunctionMetadata", + "bool", + "isDistinctSensitive"); } } // namespace facebook::presto::protocol namespace facebook::presto::protocol { @@ -1119,6 +1133,7 @@ void from_json(const json& j, std::shared_ptr& p) { } } // namespace facebook::presto::protocol // dependency TpchTransactionHandle +// dependency TpcdsTransactionHandle // dependency ArrowTransactionHandle namespace facebook::presto::protocol { diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index 2b1e4eb66c14e..743356cf93f91 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -415,6 +415,7 @@ namespace facebook::presto::protocol { struct AggregationFunctionMetadata { TypeSignature intermediateType = {}; bool isOrderSensitive = {}; + bool isDistinctSensitive = {}; }; void to_json(json& j, const AggregationFunctionMetadata& p); void from_json(const json& j, AggregationFunctionMetadata& p); diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java index 59263a302113c..844b40fe76852 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java @@ -121,7 +121,7 @@ private List getTestAggregationFunctions() "default", false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), - Optional.of(new AggregationFunctionMetadata(new TypeSignature("varbinary"), true)), + Optional.of(new AggregationFunctionMetadata(new TypeSignature("varbinary"), true, true)), Optional.empty(), Optional.empty(), Optional.of(ImmutableList.of()), diff --git a/presto-native-execution/src/test/resources/external_functions.json b/presto-native-execution/src/test/resources/external_functions.json index c3c1754d5d090..0011941bce9f1 100644 --- a/presto-native-execution/src/test/resources/external_functions.json +++ b/presto-native-execution/src/test/resources/external_functions.json @@ -82,7 +82,8 @@ }, "aggregateMetadata": { "intermediateType": "BIGINT", - "isOrderSensitive": false + "isOrderSensitive": false, + "isDistinctSensitive": true } } ], @@ -102,7 +103,8 @@ }, "aggregateMetadata": { "intermediateType": "ROW(DOUBLE, BIGINT)", - "isOrderSensitive": false + "isOrderSensitive": false, + "isDistinctSensitive": true } } ], @@ -122,7 +124,8 @@ }, "aggregateMetadata": { "intermediateType": "ROW(DOUBLE, BIGINT)", - "isOrderSensitive": false + "isOrderSensitive": false, + "isDistinctSensitive": true } } ], @@ -142,7 +145,8 @@ }, "aggregateMetadata": { "intermediateType": "ROW(DOUBLE, BIGINT)", - "isOrderSensitive": false + "isOrderSensitive": false, + "isDistinctSensitive": true } } ], diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java index f1b7142a188ee..17a171efe0bd4 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java @@ -172,6 +172,7 @@ private AggregationFunctionImplementation processNativeFunctionHandle(NativeFunc typeManager.getType(resolvedIntermediateType), typeManager.getType(signature.getReturnType()), aggregationMetadata.isOrderSensitive(), + aggregationMetadata.isDistinctSensitive(), parameters)); return aggregationImplementationByHandle.get(nativeFunctionHandle); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java index 9a8f86fab045a..27df72fb127c3 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java @@ -35,6 +35,8 @@ */ boolean isOrderSensitive() default false; + boolean isDistinctSensitive() default true; + SqlFunctionVisibility visibility() default PUBLIC; String[] alias() default {}; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionImplementation.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionImplementation.java index 14a6e4389041e..a38028922ece8 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionImplementation.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionImplementation.java @@ -27,4 +27,6 @@ public interface AggregationFunctionImplementation boolean isDecomposable(); boolean isOrderSensitive(); + + boolean isDistinctSensitive(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java index 240ae5d5241f8..9e91ffe7d6f82 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java @@ -36,13 +36,17 @@ public class AggregationFunctionMetadata */ private final boolean isOrderSensitive; + private final boolean isDistinctSensitive; + @JsonCreator public AggregationFunctionMetadata( @JsonProperty("intermediateType") TypeSignature intermediateType, - @JsonProperty("isOrderSensitive") boolean isOrderSensitive) + @JsonProperty("isOrderSensitive") boolean isOrderSensitive, + @JsonProperty("isDistinctSensitive") boolean isDistinctSensitive) { this.intermediateType = requireNonNull(intermediateType, "intermediateType is null"); this.isOrderSensitive = isOrderSensitive; + this.isDistinctSensitive = isDistinctSensitive; } @JsonProperty @@ -57,13 +61,20 @@ public boolean isOrderSensitive() return isOrderSensitive; } + @JsonProperty + public boolean isDistinctSensitive() + { + return isDistinctSensitive; + } + @Override public String toString() { return format( - "%s(%s,%s)", + "%s(%s,%s,%s)", getClass().getSimpleName(), getIntermediateType(), - isOrderSensitive()); + isOrderSensitive(), + isDistinctSensitive()); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java index 2386caba550dd..362ee607f98bf 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java @@ -35,17 +35,21 @@ public class SqlInvokedAggregationFunctionImplementation private final boolean isOrderSensitive; + private final boolean isDistinctSensitive; + private final List parameterTypes; public SqlInvokedAggregationFunctionImplementation( Type intermediateType, Type finalType, boolean isOrderSensitive, + boolean isDistinctSensitive, List parameterTypes) { this.intermediateType = requireNonNull(intermediateType, "intermediateType is null"); this.finalType = requireNonNull(finalType, "finalType is null"); this.isOrderSensitive = isOrderSensitive; + this.isDistinctSensitive = isDistinctSensitive; this.parameterTypes = requireNonNull(parameterTypes, "parameterTypes is null"); } @@ -74,6 +78,12 @@ public boolean isOrderSensitive() return isOrderSensitive; } + @Override + public boolean isDistinctSensitive() + { + return isDistinctSensitive; + } + @Override public List getParameterTypes() {