From 10b3f532704fa526b32f96e9ef9fd4e11c698704 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Tue, 12 Mar 2024 17:41:06 +0100 Subject: [PATCH] [CALCITE-6429] Arrow adapter should default to the Enumerable convention for unsupported filters --- .../calcite/adapter/arrow/ArrowRules.java | 9 ++- .../adapter/arrow/ArrowTranslator.java | 8 +- .../adapter/arrow/ArrowAdapterTest.java | 79 +++++++++++++------ .../java/org/apache/calcite/util/Bug.java | 20 +++++ 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index 42e47e8be2c..b70e7096483 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -84,8 +84,13 @@ protected ArrowFilterRule(Config config) { final Filter filter = call.rel(0); if (filter.getTraitSet().contains(Convention.NONE)) { - final RelNode converted = convert(filter); - call.transformTo(converted); + try { + final RelNode converted = convert(filter); + call.transformTo(converted); + } catch (UnsupportedOperationException e) { + // skip rule application when hitting an unsupported feature, + // allowing a plan in the Enumerable convention to be generated + } } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 5b68a130dac..900e4705713 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -67,7 +67,7 @@ List translateMatch(RexNode condition) { if (disjunctions.size() == 1) { return translateAnd(disjunctions.get(0)); } else { - throw new AssertionError("cannot translate " + condition); + throw new UnsupportedOperationException("Unsupported disjunctive condition " + condition); } } @@ -128,7 +128,7 @@ private String translateMatch2(RexNode node) { case GREATER_THAN_OR_EQUAL: return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node); default: - throw new AssertionError("cannot translate " + node); + throw new UnsupportedOperationException("Unsupported binary operator " + node); } } @@ -147,7 +147,7 @@ private String translateBinary(String op, String rop, RexCall call) { if (expression != null) { return expression; } - throw new AssertionError("cannot translate op " + op + " call " + call); + throw new UnsupportedOperationException("Unsupported binary operator " + call); } /** Translates a call to a binary operator. Returns null on failure. */ @@ -197,6 +197,6 @@ private static String getLiteralType(Object literal) { } else if (String.class.equals(literal.getClass())) { return "string"; } - throw new AssertionError("Invalid literal"); + throw new UnsupportedOperationException("Unsupported literal " + literal); } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 19e7bdfd5d5..6dd8b75cd40 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.schema.Table; import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.util.Bug; import org.apache.calcite.util.Sources; import com.google.common.collect.ImmutableList; @@ -238,15 +239,24 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException .explainContains(plan); } - @Disabled("OR is not supported yet") @Test void testArrowProjectFieldsWithDisjunctiveFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\"=12 or \"stringField\"='12'"; - String plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String plan; + if (Bug.CALCITE_6293_FIXED) { + plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } else { + plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[12], " + + "expr#3=[=($t0, $t2)], expr#4=['12':VARCHAR], expr#5=[=($t1, $t4)], " + + "expr#6=[OR($t3, $t5)], proj#0..1=[{exprs}], $condition=[$t6])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } String result = "intField=12; stringField=12\n"; CalciteAssert.that() @@ -256,15 +266,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException .explainContains(plan); } - @Disabled("IN is not supported as OR is not supported yet") @Test void testArrowProjectFieldsWithInFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\" in (0, 1, 2)"; - String plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String plan; + if (Bug.CALCITE_6294_FIXED) { + plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } else { + plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[Sarg[0, 1, 2]], " + + "expr#3=[SEARCH($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } String result = "intField=0; stringField=0\n" + "intField=1; stringField=1\n" + "intField=2; stringField=2\n"; @@ -276,19 +294,28 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException .explainContains(plan); } - @Disabled("IS NOT NULL is not supported yet") @Test void testArrowProjectFieldsWithIsNotNullFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\" is not null\n" + "order by \"intField\"\n" + "limit 1"; - String plan = "PLAN=EnumerableLimit(fetch=[1])\n" - + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" - + " ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[IS NOT NULL($0)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String plan; + if (Bug.CALCITE_6295_FIXED) { + plan = "PLAN=EnumerableLimit(fetch=[1])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[IS NOT NULL($0)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } else { + plan = "PLAN=EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])\n" + + " EnumerableLimit(fetch=[1])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t0)], proj#0..3=[{exprs}], $condition=[$t4])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } String result = "intField=0; stringField=0\n"; CalciteAssert.that() @@ -298,15 +325,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException .explainContains(plan); } - @Disabled("IS NULL is not supported yet") @Test void testArrowProjectFieldsWithIsNullFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\" is null"; - String plan = "ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[IS NOT NULL($0)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String plan; + if (Bug.CALCITE_6296_FIXED) { + plan = "ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[IS NOT NULL($0)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } else { + plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}]," + + " expr#2=[IS NULL($t0)], proj#0..1=[{exprs}], $condition=[$t2])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + } CalciteAssert.that() .with(arrow) diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 331922dce54..3e5385ee020 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -221,6 +221,26 @@ public abstract class Bug { * is fixed. */ public static final boolean CALCITE_6391_FIXED = false; + /** Whether + * + * [CALCITE-6293] Support OR condition in Arrow adapter is fixed. */ + public static final boolean CALCITE_6293_FIXED = false; + + /** Whether + * + * [CALCITE-6294] Support IN filter in Arrow adapter is fixed. */ + public static final boolean CALCITE_6294_FIXED = false; + + /** Whether + * + * [CALCITE-6295] Support IS NOT NULL in Arrow adapter is fixed. */ + public static final boolean CALCITE_6295_FIXED = false; + + /** Whether + * + * [CALCITE-6296] Support IS NULL in Arrow adapter is fixed. */ + public static final boolean CALCITE_6296_FIXED = false; + /** * Use this to flag temporary code. */