From f75f048e580a1c5186b8f1b4b90bb95e6321d6c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Oct 2024 01:42:31 +0000 Subject: [PATCH] Lateral eval expressions supported after Spark upgrading (#544) Signed-off-by: Lantao Jin (cherry picked from commit c733d3cb403e5e57c2c2c4625be7ec5ce8c8ff10) Signed-off-by: github-actions[bot] --- .../flint/spark/ppl/FlintSparkPPLEvalITSuite.scala | 7 +------ ppl-spark-integration/README.md | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala index 407c2cb3b..19295fbe8 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLEvalITSuite.scala @@ -480,12 +480,7 @@ class FlintSparkPPLEvalITSuite comparePlans(logicalPlan, expectedPlan, checkAnalysis = false) } - // +--------------------------------+ - // | Below tests are not supported | - // +--------------------------------+ - // Todo: Upgrading spark version to 3.4.0 and above could fix this test. - // https://issues.apache.org/jira/browse/SPARK-27561 - ignore("test lateral eval expressions references - SPARK-27561 required") { + test("test lateral eval expressions references") { val frame = sql(s""" | source = $testTable | eval col1 = 1, col2 = col1 | fields name, age, col2 | """.stripMargin) diff --git a/ppl-spark-integration/README.md b/ppl-spark-integration/README.md index fa668041d..a10ce60e5 100644 --- a/ppl-spark-integration/README.md +++ b/ppl-spark-integration/README.md @@ -258,7 +258,7 @@ Assumptions: `a`, `b`, `c` are existing fields in `table` - `source = table | eval n = now() | eval t = unix_timestamp(a) | fields n,t` - `source = table | eval f = a | where f > 1 | sort f | fields a,b,c | head 5` - `source = table | eval f = a * 2 | eval h = f * 2 | fields a,f,h` - - `source = table | eval f = a * 2, h = f * 2 | fields a,f,h` (Spark 3.4.0+ required) + - `source = table | eval f = a * 2, h = f * 2 | fields a,f,h` - `source = table | eval f = a * 2, h = b | stats avg(f) by h` - `source = table | eval f = ispresent(a)`