[CALCITE-6429] Arrow adapter should default to the Enumerable convent…

…ion for unsupported filters
asolimando · Jun 5, 2024 · 748f34d · 748f34d
1 parent be9b860
commit 748f34d
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 28 deletions.
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
@@ -84,8 +84,13 @@ protected ArrowFilterRule(Config config) {
       final Filter filter = call.rel(0);
 
       if (filter.getTraitSet().contains(Convention.NONE)) {
-        final RelNode converted = convert(filter);
-        call.transformTo(converted);
+        try {
+          final RelNode converted = convert(filter);
+          call.transformTo(converted);
+        } catch (UnsupportedOperationException e) {
+          // skip rule application when hitting an unsupported feature,
+          // allowing a plan in the Enumerable convention to be generated
+        }
       }
     }
 

diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
@@ -67,7 +67,7 @@ List<String> translateMatch(RexNode condition) {
     if (disjunctions.size() == 1) {
       return translateAnd(disjunctions.get(0));
     } else {
-      throw new AssertionError("cannot translate " + condition);
+      throw new UnsupportedOperationException("Unsupported disjunctive condition " + condition);
     }
   }
 
@@ -128,7 +128,7 @@ private String translateMatch2(RexNode node) {
     case GREATER_THAN_OR_EQUAL:
       return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node);
     default:
-      throw new AssertionError("cannot translate " + node);
+      throw new UnsupportedOperationException("Unsupported binary operator " + node);
     }
   }
 
@@ -147,7 +147,7 @@ private String translateBinary(String op, String rop, RexCall call) {
     if (expression != null) {
       return expression;
     }
-    throw new AssertionError("cannot translate op " + op + " call " + call);
+    throw new UnsupportedOperationException("Unsupported binary operator " + call);
   }
 
   /** Translates a call to a binary operator. Returns null on failure. */
@@ -197,6 +197,6 @@ private static String getLiteralType(Object literal) {
     } else if (String.class.equals(literal.getClass())) {
       return "string";
     }
-    throw new AssertionError("Invalid literal");
+    throw new UnsupportedOperationException("Unsupported literal " + literal);
   }
 }
diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
@@ -22,6 +22,7 @@
 import org.apache.calcite.rel.type.RelDataTypeSystem;
 import org.apache.calcite.schema.Table;
 import org.apache.calcite.test.CalciteAssert;
+import org.apache.calcite.util.Bug;
 import org.apache.calcite.util.Sources;
 
 import com.google.common.collect.ImmutableList;
@@ -238,15 +239,24 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
         .explainContains(plan);
   }
 
-  @Disabled("OR is not supported yet")
   @Test void testArrowProjectFieldsWithDisjunctiveFilter() {
     String sql = "select \"intField\", \"stringField\"\n"
         + "from arrowdata\n"
         + "where \"intField\"=12 or \"stringField\"='12'";
-    String plan = "PLAN=ArrowToEnumerableConverter\n"
-        + "  ArrowProject(intField=[$0], stringField=[$1])\n"
-        + "    ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n"
-        + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    String plan;
+    if (Bug.CALCITE_6293_FIXED) {
+      plan = "PLAN=ArrowToEnumerableConverter\n"
+          + "  ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "    ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n"
+          + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    } else {
+      plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[12], "
+          + "expr#3=[=($t0, $t2)], expr#4=['12':VARCHAR], expr#5=[=($t1, $t4)], "
+          + "expr#6=[OR($t3, $t5)], proj#0..1=[{exprs}], $condition=[$t6])\n"
+          + "  ArrowToEnumerableConverter\n"
+          + "    ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    }
     String result = "intField=12; stringField=12\n";
 
     CalciteAssert.that()
@@ -256,15 +266,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
         .explainContains(plan);
   }
 
-  @Disabled("IN is not supported as OR is not supported yet")
   @Test void testArrowProjectFieldsWithInFilter() {
     String sql = "select \"intField\", \"stringField\"\n"
         + "from arrowdata\n"
         + "where \"intField\" in (0, 1, 2)";
-    String plan = "PLAN=ArrowToEnumerableConverter\n"
-        + "  ArrowProject(intField=[$0], stringField=[$1])\n"
-        + "    ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n"
-        + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    String plan;
+    if (Bug.CALCITE_6294_FIXED) {
+      plan = "PLAN=ArrowToEnumerableConverter\n"
+          + "  ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "    ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n"
+          + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    } else {
+      plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[Sarg[0, 1, 2]], "
+          + "expr#3=[SEARCH($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n"
+          + "  ArrowToEnumerableConverter\n"
+          + "    ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    }
     String result = "intField=0; stringField=0\n"
         + "intField=1; stringField=1\n"
         + "intField=2; stringField=2\n";
@@ -276,19 +294,28 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
         .explainContains(plan);
   }
 
-  @Disabled("IS NOT NULL is not supported yet")
   @Test void testArrowProjectFieldsWithIsNotNullFilter() {
     String sql = "select \"intField\", \"stringField\"\n"
         + "from arrowdata\n"
         + "where \"intField\" is not null\n"
         + "order by \"intField\"\n"
         + "limit 1";
-    String plan = "PLAN=EnumerableLimit(fetch=[1])\n"
-        + "  EnumerableSort(sort0=[$0], dir0=[ASC])\n"
-        + "    ArrowToEnumerableConverter\n"
-        + "      ArrowProject(intField=[$0], stringField=[$1])\n"
-        + "        ArrowFilter(condition=[IS NOT NULL($0)])\n"
-        + "          ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    String plan;
+    if (Bug.CALCITE_6295_FIXED) {
+      plan = "PLAN=EnumerableLimit(fetch=[1])\n"
+          + "  EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+          + "    ArrowToEnumerableConverter\n"
+          + "      ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "        ArrowFilter(condition=[IS NOT NULL($0)])\n"
+          + "          ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    } else {
+      plan = "PLAN=EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])\n"
+          + "  EnumerableLimit(fetch=[1])\n"
+          + "    EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+          + "      EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t0)], proj#0..3=[{exprs}], $condition=[$t4])\n"
+          + "        ArrowToEnumerableConverter\n"
+          + "          ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    }
     String result = "intField=0; stringField=0\n";
 
     CalciteAssert.that()
@@ -298,15 +325,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
         .explainContains(plan);
   }
 
-  @Disabled("IS NULL is not supported yet")
   @Test void testArrowProjectFieldsWithIsNullFilter() {
     String sql = "select \"intField\", \"stringField\"\n"
         + "from arrowdata\n"
         + "where \"intField\" is null";
-    String plan = "ArrowToEnumerableConverter\n"
-        + "  ArrowProject(intField=[$0], stringField=[$1])\n"
-        + "    ArrowFilter(condition=[IS NOT NULL($0)])\n"
-        + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    String plan;
+    if (Bug.CALCITE_6296_FIXED) {
+      plan = "ArrowToEnumerableConverter\n"
+          + "  ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "    ArrowFilter(condition=[IS NOT NULL($0)])\n"
+          + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    } else {
+      plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}],"
+          + " expr#2=[IS NULL($t0)], proj#0..1=[{exprs}], $condition=[$t2])\n"
+          + "  ArrowToEnumerableConverter\n"
+          + "    ArrowProject(intField=[$0], stringField=[$1])\n"
+          + "      ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+    }
 
     CalciteAssert.that()
         .with(arrow)

diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java
@@ -221,6 +221,26 @@ public abstract class Bug {
    * is fixed. */
   public static final boolean CALCITE_6391_FIXED = false;
 
+  /** Whether
+   * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6293">
+   * [CALCITE-6293] Support OR condition in Arrow adapter</a> is fixed. */
+  public static final boolean CALCITE_6293_FIXED = false;
+
+  /** Whether
+   * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6294">
+   * [CALCITE-6294] Support IN filter in Arrow adapter</a> is fixed. */
+  public static final boolean CALCITE_6294_FIXED = false;
+
+  /** Whether
+   * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6295">
+   * [CALCITE-6295] Support IS NOT NULL in Arrow adapter</a> is fixed. */
+  public static final boolean CALCITE_6295_FIXED = false;
+
+  /** Whether
+   * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6296">
+   * [CALCITE-6296] Support IS NULL in Arrow adapter</a> is fixed. */
+  public static final boolean CALCITE_6296_FIXED = false;
+
   /**
    * Use this to flag temporary code.
    */