From a90ddb8bac43ac1716f46716d47de5e9a431c902 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Sun, 3 Mar 2024 16:31:08 +0100 Subject: [PATCH] improved tests --- .../adapter/arrow/ArrowFilterEnumerator.java | 4 +- .../adapter/arrow/ArrowAdapterTest.java | 303 ++++++++++++++---- .../calcite/adapter/arrow/ArrowExtension.java | 70 ++++ .../calcite/config/CalciteSystemProperty.java | 6 + 4 files changed, 311 insertions(+), 72 deletions(-) create mode 100644 arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java index 326c3086d01..d6015ca6cd0 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java @@ -88,7 +88,9 @@ void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { @Override public void close() { try { - buf.close(); + if (buf != null) { + buf.close(); + } filter.close(); } catch (GandivaException e) { throw Util.toUnchecked(e); diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 526b4584220..fde437bb138 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -16,6 +16,8 @@ */ package org.apache.calcite.adapter.arrow; +import com.google.common.collect.ImmutableList; + import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -24,16 +26,15 @@ import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.util.Sources; -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.gandiva.expression.ExpressionTree; -import org.apache.arrow.vector.types.pojo.Schema; - import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import java.io.File; import java.io.IOException; @@ -41,48 +42,19 @@ import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Objects; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; /** * Tests for the Apache Arrow adapter. */ +@Execution(ExecutionMode.SAME_THREAD) +@ExtendWith(ArrowExtension.class) class ArrowAdapterTest { private static Map arrow; private static File arrowDataDirectory; - private static boolean hasGandivaSupport = detectGandivaSupport(); - - ArrowAdapterTest() { - assumeTrue(hasGandivaSupport, "gandiva not supported on this platform, skipping tests"); - } - - /** - * Gandiva (used to implement arrow filtering / projection) does not currently distribute - * a binary that is compatible with M1 macs on maven central. - * see ARROW-16608. - * - * @return true if we believe that gandiva is supported on this platform and we can run the tests - */ - private static boolean detectGandivaSupport() { - try { - Schema emptySchema = new Schema(new ArrayList<>(), null); - List expressions = new ArrayList<>(); - Projector.make(emptySchema, expressions); - } catch (GandivaException e) { - // this is ok -- we'll always hit this because of an empty expression - // the fact that we got here, is indicative that the JNI library was loaded properly - return true; - } catch (UnsatisfiedLinkError e) { - return false; - } - return true; - } @BeforeAll static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException, SQLException { @@ -112,9 +84,8 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); RelDataType relDataType = tableMap.get("ARROWDATA").getRowType(typeFactory); - assertThat(relDataType.getFieldNames().get(0), is("intField")); - assertThat(relDataType.getFieldNames().get(1), is("stringField")); - assertThat(relDataType.getFieldNames().get(2), is("floatField")); + assertEquals(relDataType.getFieldNames(), + ImmutableList.of("intField", "stringField", "floatField", "longField")); } @Test void testArrowProjectAllFields() { @@ -127,6 +98,46 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + "intField=3; stringField=3; floatField=3.0; longField=3\n" + "intField=4; stringField=4; floatField=4.0; longField=4\n" + "intField=5; stringField=5; floatField=5.0; longField=5\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(6) + .returns(result) + .explainContains(plan); + } + + @Test void testArrowProjectAllFieldsExplicitly() { + String sql = "select \"intField\", \"stringField\", \"floatField\", \"longField\" from arrowdata\n"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=0; stringField=0; floatField=0.0; longField=0\n" + + "intField=1; stringField=1; floatField=1.0; longField=1\n" + + "intField=2; stringField=2; floatField=2.0; longField=2\n" + + "intField=3; stringField=3; floatField=3.0; longField=3\n" + + "intField=4; stringField=4; floatField=4.0; longField=4\n" + + "intField=5; stringField=5; floatField=5.0; longField=5\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(6) + .returns(result) + .explainContains(plan); + } + + @Test void testArrowProjectAllFieldsExplicitlyPermutation() { + String sql = "select \"stringField\", \"intField\", \"longField\", \"floatField\" from arrowdata\n"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(stringField=[$1], intField=[$0], longField=[$3], floatField=[$2])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "stringField=0; intField=0; longField=0; floatField=0.0\n" + + "stringField=1; intField=1; longField=1; floatField=1.0\n" + + "stringField=2; intField=2; longField=2; floatField=2.0\n" + + "stringField=3; intField=3; longField=3; floatField=3.0\n" + + "stringField=4; intField=4; longField=4; floatField=4.0\n" + + "stringField=5; intField=5; longField=5; floatField=5.0\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -137,11 +148,12 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException @Test void testArrowProjectSingleField() { String sql = "select \"intField\" from arrowdata\n"; - String result = "intField=0\nintField=1\nintField=2\n" - + "intField=3\nintField=4\nintField=5\n"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$0])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=0\nintField=1\nintField=2\n" + + "intField=3\nintField=4\nintField=5\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -152,15 +164,16 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException @Test void testArrowProjectTwoFields() { String sql = "select \"intField\", \"stringField\" from arrowdata\n"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=0; stringField=0\n" + "intField=1; stringField=1\n" + "intField=2; stringField=2\n" + "intField=3; stringField=3\n" + "intField=4; stringField=4\n" + "intField=5; stringField=5\n"; - String plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -173,39 +186,134 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\" < 4"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[<($0, 4)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=0; stringField=0\n" + "intField=1; stringField=1\n" + "intField=2; stringField=2\n" + "intField=3; stringField=3\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + @Test void testArrowProjectFieldsWithMultipleFilterSameField() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"intField\" > 1 and \"intField\" < 4"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[SEARCH($0, Sarg[(1..4)])])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=2; stringField=2\n" + + "intField=3; stringField=3\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + @Test void testArrowProjectFieldsWithConjunctiveFilters() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"intField\"=12 and \"stringField\"='12'"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[<($0, 4)])\n" + + " ArrowFilter(condition=[AND(=($0, 12), =($1, '12'))])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=12; stringField=12\n"; + CalciteAssert.that() .with(arrow) .query(sql) - .limit(4) .returns(result) .explainContains(plan); } - @Test void testArrowProjectFieldsWithMultipleFilters() { + @Disabled("OR is not supported yet") + @Test void testArrowProjectFieldsWithDisjunctiveFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" - + "where \"intField\"=12 and \"stringField\"='12'"; + + "where \"intField\"=12 or \"stringField\"='12'"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[AND(=($0, 12), =($1, '12'))])\n" + + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=12; stringField=12\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + @Disabled("IN is not supported as OR is not supported yet") + @Test void testArrowProjectFieldsWithInFilter() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"intField\" in (0, 1, 2)"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=0; stringField=0\n" + + "intField=1; stringField=1\n" + + "intField=2; stringField=2\n"; + CalciteAssert.that() .with(arrow) .query(sql) - .limit(3) .returns(result) .explainContains(plan); } + @Disabled("IS NOT NULL is not supported yet") + @Test void testArrowProjectFieldsWithIsNotNullFilter() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"intField\" is not null\n" + + "order by \"intField\"\n" + + "limit 1"; + String plan = "PLAN=EnumerableLimit(fetch=[1])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[IS NOT NULL($0)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=0; stringField=0\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + @Disabled("IS NULL is not supported yet") + @Test void testArrowProjectFieldsWithIsNullFilter() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"intField\" is null"; + String plan = "ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[IS NOT NULL($0)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returnsCount(0) + .explainContains(plan); + } + @Test void testArrowProjectFieldsWithFloatFilter() { String sql = "select * from arrowdata\n" + " where \"floatField\"=15.0"; @@ -213,6 +321,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowFilter(condition=[=(CAST($2):DOUBLE, 15.0)])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=15; stringField=15; floatField=15.0; longField=15\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -229,6 +338,49 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowFilter(condition=[=($0, 25)])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=25\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + @Disabled("Sub-queries are not yet supported") + @Test void testArrowSubquery() { + String sql = "select \"intField\"\n" + + "from (select \"intField\", \"stringField\" from arrowdata where \"stringField\" = 2)\n" + + "where \"intField\" = 2"; + String plan = "PLAN=\n\n"; + String result = "intField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + @Disabled("UNION does not work") + @Test void testArrowUnion() { + String sql = "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 2)\n" + + " union \n" + + "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 1)\n"; + String plan = "PLAN=EnumerableUnion(all=[false])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0])\n" + + " ArrowFilter(condition=[=($0, 2)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0])\n" + + " ArrowFilter(condition=[=($0, 1)])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=1\nintField=2\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -253,6 +405,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowProject(DEPTNO=[$0])\n" + " ArrowTableScan(table=[[ARROW, DEPT]], fields=[[0, 1, 2]])\n\n"; String result = "DEPTNO=10\nDEPTNO=20\nDEPTNO=30\nDEPTNO=40\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -282,6 +435,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, SALGRADE]], fields=[[0, 1, 2]])\n\n"; String result = "trunc=700\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -298,6 +452,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, SALGRADE]], fields=[[0, 1, 2]])\n\n"; String result = "extra=700.0\nextra=1201.0\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -314,6 +469,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, SALGRADE]], fields=[[0, 1, 2]])\n\n"; String result = "extra=700.0\nextra=1201.0\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -325,11 +481,11 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException @Test void testCastIntToDouble() { String sql = "select CAST(\"intField\" AS DOUBLE) as \"dbl\" from arrowdata"; - String result = "dbl=0.0\ndbl=1.0\n"; String plan = "PLAN=EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t0):DOUBLE], dbl=[$t4])\n" + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "dbl=0.0\ndbl=1.0\n"; CalciteAssert.that() .with(arrow) @@ -340,8 +496,6 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException .explainContains(plan); } - // TODO: test IS NULL, IS NOT NULL - // TODO: test 3-valued boolean logic, e.g. 'WHERE (x > 5) IS NOT FALSE', // 'SELECT (x > 5) ...' @@ -354,6 +508,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "field1=0_suffix\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -375,17 +530,12 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException // projectors. Also, in some cases the optimal plan would combine selections // and projectors.) - // TODO: test an OR condition - - // TODO: test an IN condition, e.g. x IN (1, 7, 8, 9) - @Test void testAggWithoutAggFunctions() { String sql = "select DISTINCT(\"intField\") as \"dep\" from arrowdata"; - String result = "dep=0\ndep=1\n"; - String plan = "PLAN=EnumerableAggregate(group=[{0}])\n" + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "dep=0\ndep=1\n"; CalciteAssert.that() .with(arrow) @@ -397,11 +547,10 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException @Test void testAggWithAggFunctions() { String sql = "select JOB, SUM(SAL) as TOTAL from EMP GROUP BY JOB"; - String result = "JOB=SALESMAN; TOTAL=5600.00\nJOB=ANALYST; TOTAL=6000.00\n"; - String plan = "PLAN=EnumerableAggregate(group=[{2}], TOTAL=[SUM($5)])\n" + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, EMP]], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])\n\n"; + String result = "JOB=SALESMAN; TOTAL=5600.00\nJOB=ANALYST; TOTAL=6000.00\n"; CalciteAssert.that() .with(arrow) @@ -412,15 +561,31 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException } @Test void testFilteredAgg() { - // TODO add group by String sql = "select SUM(SAL) FILTER (WHERE COMM > 400) as SALESSUM from EMP"; - String result = "SALESSUM=2500.00\n"; - String plan = "PLAN=EnumerableAggregate(group=[{}], SALESSUM=[SUM($0) FILTER $1])\n" + " EnumerableCalc(expr#0..7=[{inputs}], expr#8=[400], expr#9=[>($t6, $t8)], " + "expr#10=[IS TRUE($t9)], SAL=[$t5], $f1=[$t10])\n" + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, EMP]], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])\n\n"; + String result = "SALESSUM=2500.00\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } + + @Test void testFilteredAggGroupBy() { + String sql = "select SUM(SAL) FILTER (WHERE COMM > 400) as SALESSUM from EMP group by EMPNO"; + String plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], SALESSUM=[$t1])\n" + + " EnumerableAggregate(group=[{0}], SALESSUM=[SUM($1) FILTER $2])\n" + + " EnumerableCalc(expr#0..7=[{inputs}], expr#8=[400], expr#9=[>($t6, $t8)], " + + "expr#10=[IS TRUE($t9)], EMPNO=[$t0], SAL=[$t5], $f2=[$t10])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowTableScan(table=[[ARROW, EMP]], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])\n\n"; + String result = "SALESSUM=1250.00\nSALESSUM=null\n"; CalciteAssert.that() .with(arrow) @@ -432,12 +597,6 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException @Test void testAggGroupedByNullable() { String sql = "select COMM, SUM(SAL) as SALESSUM from EMP GROUP BY COMM"; - String result = "COMM=0.00; SALESSUM=1500.00\n" - + "COMM=1400.00; SALESSUM=1250.00\n" - + "COMM=300.00; SALESSUM=1600.00\n" - + "COMM=500.00; SALESSUM=1250.00\n" - + "COMM=null; SALESSUM=23425.00"; - String plan = "PLAN=EnumerableAggregate(group=[{6}], SALESSUM=[SUM($5)])\n" + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, EMP]], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])\n\n"; @@ -462,6 +621,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=0\nintField=1\n"; + CalciteAssert.that() .with(arrow) .query(sql) @@ -478,6 +638,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) throws IOException + " ArrowToEnumerableConverter\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=2\nintField=3\n"; + CalciteAssert.that() .with(arrow) .query(sql) diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java new file mode 100644 index 00000000000..4065d4b5bce --- /dev/null +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.arrow; + +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.vector.types.pojo.Schema; + +import org.apache.calcite.config.CalciteSystemProperty; + +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.util.ArrayList; +import java.util.List; + +/** + * JUnit5 extension to handle Arrow tests. + * + *

Tests will be skipped if the Gandiva library cannot be loaded on the given platform. + */ +class ArrowExtension implements ExecutionCondition { + + /** + * Whether to run this test. + * + *

Enabled by default, unless explicitly disabled from command line + * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to implement arrow + * filtering/projection, cannot be loaded. + * + * @return {@code true} if the test is enabled and can run in the current environment, + * {@code false} otherwise + */ + @Override public ConditionEvaluationResult evaluateExecutionCondition( + final ExtensionContext context) { + + boolean enabled = CalciteSystemProperty.TEST_ARROW.value(); + try { + Schema emptySchema = new Schema(new ArrayList<>(), null); + List expressions = new ArrayList<>(); + Projector.make(emptySchema, expressions); + } catch (GandivaException e) { + // this exception comes from using an empty expression, but the JNI library was loaded properly + } catch (UnsatisfiedLinkError e) { + enabled = false; + } + + if (enabled) { + return ConditionEvaluationResult.enabled("Arrow tests enabled"); + } else { + return ConditionEvaluationResult.disabled("Cassandra tests disabled"); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 4db1b306a05..4d454c69fac 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -210,6 +210,12 @@ public final class CalciteSystemProperty { return "."; }); + /** + * Whether to run Arrow tests. + */ + public static final CalciteSystemProperty TEST_ARROW = + booleanProperty("calcite.test.arrow", true); + /** * Whether to run MongoDB tests. */