-
Notifications
You must be signed in to change notification settings - Fork 137
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Chen Dai <[email protected]>
- Loading branch information
Showing
6 changed files
with
164 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
55 changes: 55 additions & 0 deletions
55
...c/test/scala/org/opensearch/flint/spark/skipping/minmax/MinMaxSkippingStrategySuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.flint.spark.skipping.minmax | ||
|
||
import org.scalatest.matchers.should.Matchers | ||
|
||
import org.apache.spark.SparkFunSuite | ||
import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, EqualTo, GreaterThan, GreaterThanOrEqual, In, LessThan, LessThanOrEqual, Literal, Or} | ||
import org.apache.spark.sql.functions.col | ||
import org.apache.spark.sql.types.IntegerType | ||
|
||
class MinMaxSkippingStrategySuite extends SparkFunSuite with Matchers { | ||
|
||
private val strategy = MinMaxSkippingStrategy(columnName = "age", columnType = "integer") | ||
|
||
private val indexCol = AttributeReference("age", IntegerType, nullable = false)() | ||
|
||
private val minCol = col("MinMax_age_0").expr | ||
private val maxCol = col("MinMax_age_1").expr | ||
|
||
test("should rewrite EqualTo(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(EqualTo(indexCol, Literal(30))) shouldBe Some( | ||
And(LessThanOrEqual(minCol, Literal(30)), GreaterThanOrEqual(maxCol, Literal(30)))) | ||
} | ||
|
||
test("should rewrite LessThan(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(LessThan(indexCol, Literal(30))) shouldBe Some( | ||
LessThan(minCol, Literal(30))) | ||
} | ||
|
||
test("should rewrite LessThanOrEqual(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(LessThanOrEqual(indexCol, Literal(30))) shouldBe Some( | ||
LessThanOrEqual(minCol, Literal(30))) | ||
} | ||
|
||
test("should rewrite GreaterThan(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(GreaterThan(indexCol, Literal(30))) shouldBe Some( | ||
GreaterThan(maxCol, Literal(30))) | ||
} | ||
|
||
test("should rewrite GreaterThanOrEqual(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(GreaterThanOrEqual(indexCol, Literal(30))) shouldBe Some( | ||
GreaterThanOrEqual(maxCol, Literal(30))) | ||
} | ||
|
||
test("should rewrite In(<indexCol>, <value1, value2 ...>") { | ||
strategy.rewritePredicate(In(indexCol, Seq(Literal(25), Literal(30)))) shouldBe Some( | ||
Or( | ||
And(LessThanOrEqual(minCol, Literal(25)), GreaterThanOrEqual(maxCol, Literal(25))), | ||
And(LessThanOrEqual(minCol, Literal(30)), GreaterThanOrEqual(maxCol, Literal(30))))) | ||
} | ||
} |
44 changes: 44 additions & 0 deletions
44
.../scala/org/opensearch/flint/spark/skipping/partition/PartitionSkippingStrategySuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.flint.spark.skipping.partition | ||
|
||
import org.scalatest.matchers.should.Matchers | ||
|
||
import org.apache.spark.SparkFunSuite | ||
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute | ||
import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, EqualTo, GreaterThan, Literal} | ||
import org.apache.spark.sql.types.IntegerType | ||
|
||
class PartitionSkippingStrategySuite extends SparkFunSuite with Matchers { | ||
|
||
private val strategy = PartitionSkippingStrategy(columnName = "year", columnType = "int") | ||
|
||
private val indexCol = AttributeReference("year", IntegerType, nullable = false)() | ||
|
||
test("should rewrite EqualTo(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(EqualTo(indexCol, Literal(2023))) shouldBe Some( | ||
EqualTo(UnresolvedAttribute("year"), Literal(2023))) | ||
} | ||
|
||
test("should not rewrite predicate with other column)") { | ||
val predicate = | ||
EqualTo(AttributeReference("month", IntegerType, nullable = false)(), Literal(4)) | ||
|
||
strategy.rewritePredicate(predicate) shouldBe empty | ||
} | ||
|
||
test("should not rewrite GreaterThan(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(GreaterThan(indexCol, Literal(2023))) shouldBe empty | ||
} | ||
|
||
test("should only rewrite EqualTo(<indexCol>, <value>) in conjunction") { | ||
val predicate = | ||
And(EqualTo(indexCol, Literal(2023)), GreaterThan(indexCol, Literal(2023))) | ||
|
||
strategy.rewritePredicate(predicate) shouldBe Some( | ||
EqualTo(UnresolvedAttribute("year"), Literal(2023))) | ||
} | ||
} |
44 changes: 44 additions & 0 deletions
44
...st/scala/org/opensearch/flint/spark/skipping/valueset/ValueSetSkippingStrategySuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.flint.spark.skipping.valueset | ||
|
||
import org.scalatest.matchers.should.Matchers | ||
|
||
import org.apache.spark.SparkFunSuite | ||
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute | ||
import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, EqualTo, GreaterThan, Literal} | ||
import org.apache.spark.sql.types.StringType | ||
|
||
class ValueSetSkippingStrategySuite extends SparkFunSuite with Matchers { | ||
|
||
private val strategy = ValueSetSkippingStrategy(columnName = "name", columnType = "string") | ||
|
||
private val indexCol = AttributeReference("name", StringType, nullable = false)() | ||
|
||
test("should rewrite EqualTo(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(EqualTo(indexCol, Literal("hello"))) shouldBe Some( | ||
EqualTo(UnresolvedAttribute("name"), Literal("hello"))) | ||
} | ||
|
||
test("should not rewrite predicate with other column") { | ||
val predicate = | ||
EqualTo(AttributeReference("address", StringType, nullable = false)(), Literal("hello")) | ||
|
||
strategy.rewritePredicate(predicate) shouldBe empty | ||
} | ||
|
||
test("should not rewrite GreaterThan(<indexCol>, <value>)") { | ||
strategy.rewritePredicate(GreaterThan(indexCol, Literal("hello"))) shouldBe empty | ||
} | ||
|
||
test("should only rewrite EqualTo(<indexCol>, <value>) in conjunction") { | ||
val predicate = | ||
And(EqualTo(indexCol, Literal("hello")), GreaterThan(indexCol, Literal(2023))) | ||
|
||
strategy.rewritePredicate(predicate) shouldBe Some( | ||
EqualTo(UnresolvedAttribute("name"), Literal("hello"))) | ||
} | ||
} |