Skip to content

Commit

Permalink
[CALCITE-6429] Arrow adapter should default to the Enumerable convent…
Browse files Browse the repository at this point in the history
…ion for unsupported filters
  • Loading branch information
asolimando committed Jun 5, 2024
1 parent be9b860 commit 748f34d
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,13 @@ protected ArrowFilterRule(Config config) {
final Filter filter = call.rel(0);

if (filter.getTraitSet().contains(Convention.NONE)) {
final RelNode converted = convert(filter);
call.transformTo(converted);
try {
final RelNode converted = convert(filter);
call.transformTo(converted);
} catch (UnsupportedOperationException e) {
// skip rule application when hitting an unsupported feature,
// allowing a plan in the Enumerable convention to be generated
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ List<String> translateMatch(RexNode condition) {
if (disjunctions.size() == 1) {
return translateAnd(disjunctions.get(0));
} else {
throw new AssertionError("cannot translate " + condition);
throw new UnsupportedOperationException("Unsupported disjunctive condition " + condition);
}
}

Expand Down Expand Up @@ -128,7 +128,7 @@ private String translateMatch2(RexNode node) {
case GREATER_THAN_OR_EQUAL:
return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node);
default:
throw new AssertionError("cannot translate " + node);
throw new UnsupportedOperationException("Unsupported binary operator " + node);
}
}

Expand All @@ -147,7 +147,7 @@ private String translateBinary(String op, String rop, RexCall call) {
if (expression != null) {
return expression;
}
throw new AssertionError("cannot translate op " + op + " call " + call);
throw new UnsupportedOperationException("Unsupported binary operator " + call);
}

/** Translates a call to a binary operator. Returns null on failure. */
Expand Down Expand Up @@ -197,6 +197,6 @@ private static String getLiteralType(Object literal) {
} else if (String.class.equals(literal.getClass())) {
return "string";
}
throw new AssertionError("Invalid literal");
throw new UnsupportedOperationException("Unsupported literal " + literal);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.schema.Table;
import org.apache.calcite.test.CalciteAssert;
import org.apache.calcite.util.Bug;
import org.apache.calcite.util.Sources;

import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -238,15 +239,24 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
.explainContains(plan);
}

@Disabled("OR is not supported yet")
@Test void testArrowProjectFieldsWithDisjunctiveFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\"=12 or \"stringField\"='12'";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
String plan;
if (Bug.CALCITE_6293_FIXED) {
plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
} else {
plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[12], "
+ "expr#3=[=($t0, $t2)], expr#4=['12':VARCHAR], expr#5=[=($t1, $t4)], "
+ "expr#6=[OR($t3, $t5)], proj#0..1=[{exprs}], $condition=[$t6])\n"
+ " ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
}
String result = "intField=12; stringField=12\n";

CalciteAssert.that()
Expand All @@ -256,15 +266,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
.explainContains(plan);
}

@Disabled("IN is not supported as OR is not supported yet")
@Test void testArrowProjectFieldsWithInFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\" in (0, 1, 2)";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
String plan;
if (Bug.CALCITE_6294_FIXED) {
plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
} else {
plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[Sarg[0, 1, 2]], "
+ "expr#3=[SEARCH($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n"
+ " ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
}
String result = "intField=0; stringField=0\n"
+ "intField=1; stringField=1\n"
+ "intField=2; stringField=2\n";
Expand All @@ -276,19 +294,28 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
.explainContains(plan);
}

@Disabled("IS NOT NULL is not supported yet")
@Test void testArrowProjectFieldsWithIsNotNullFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\" is not null\n"
+ "order by \"intField\"\n"
+ "limit 1";
String plan = "PLAN=EnumerableLimit(fetch=[1])\n"
+ " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ " ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[IS NOT NULL($0)])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
String plan;
if (Bug.CALCITE_6295_FIXED) {
plan = "PLAN=EnumerableLimit(fetch=[1])\n"
+ " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ " ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[IS NOT NULL($0)])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
} else {
plan = "PLAN=EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])\n"
+ " EnumerableLimit(fetch=[1])\n"
+ " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ " EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t0)], proj#0..3=[{exprs}], $condition=[$t4])\n"
+ " ArrowToEnumerableConverter\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
}
String result = "intField=0; stringField=0\n";

CalciteAssert.that()
Expand All @@ -298,15 +325,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException
.explainContains(plan);
}

@Disabled("IS NULL is not supported yet")
@Test void testArrowProjectFieldsWithIsNullFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\" is null";
String plan = "ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[IS NOT NULL($0)])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
String plan;
if (Bug.CALCITE_6296_FIXED) {
plan = "ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowFilter(condition=[IS NOT NULL($0)])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
} else {
plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}],"
+ " expr#2=[IS NULL($t0)], proj#0..1=[{exprs}], $condition=[$t2])\n"
+ " ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$0], stringField=[$1])\n"
+ " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
}

CalciteAssert.that()
.with(arrow)
Expand Down
20 changes: 20 additions & 0 deletions core/src/main/java/org/apache/calcite/util/Bug.java
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,26 @@ public abstract class Bug {
* is fixed. */
public static final boolean CALCITE_6391_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6293">
* [CALCITE-6293] Support OR condition in Arrow adapter</a> is fixed. */
public static final boolean CALCITE_6293_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6294">
* [CALCITE-6294] Support IN filter in Arrow adapter</a> is fixed. */
public static final boolean CALCITE_6294_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6295">
* [CALCITE-6295] Support IS NOT NULL in Arrow adapter</a> is fixed. */
public static final boolean CALCITE_6295_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6296">
* [CALCITE-6296] Support IS NULL in Arrow adapter</a> is fixed. */
public static final boolean CALCITE_6296_FIXED = false;

/**
* Use this to flag temporary code.
*/
Expand Down

0 comments on commit 748f34d

Please sign in to comment.