Nested fields query fix (opensearch-project#564)

* fix nested fields query and filter issues as specified in - opensearch-project#508 - opensearch-project#503 Signed-off-by: YANGDB <[email protected]> * fix nested fields query and filter issues as specified in - opensearch-project#508 - opensearch-project#503 Signed-off-by: YANGDB <[email protected]> * fix scalafmtAll formatting Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> * update README doc with desc command Signed-off-by: YANGDB <[email protected]> * update failed test Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> * update new nested test class Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> * add correlation tests to identity issue with nested query fix Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> * add correlation tests to identity issue with nested query fix Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> * add correlation tests to identity issue with nested query fix Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> * add nested fields query sample to the README.md Signed-off-by: YANGDB <[email protected]> Signed-off-by: YANGDB <[email protected]> --------- Signed-off-by: YANGDB <[email protected]>
YANG-DB · Aug 20, 2024 · aa55968 · aa55968
1 parent 4af03c2
commit aa55968
Show file tree

Hide file tree

Showing 10 changed files with 1,243 additions and 22 deletions.
diff --git a/...spark-integration/src/main/scala/org/opensearch/flint/spark/sql/FlintSparkSqlParser.scala b/...spark-integration/src/main/scala/org/opensearch/flint/spark/sql/FlintSparkSqlParser.scala
@@ -31,6 +31,7 @@ import org.antlr.v4.runtime._
 import org.antlr.v4.runtime.atn.PredictionMode
 import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException}
 import org.antlr.v4.runtime.tree.TerminalNodeImpl
+import org.opensearch.flint.core.logging.CustomLogging.{logError, logInfo}
 import org.opensearch.flint.spark.sql.FlintSparkSqlExtensionsParser._
 
 import org.apache.spark.sql.AnalysisException
@@ -61,7 +62,12 @@ class FlintSparkSqlParser(sparkParser: ParserInterface) extends ParserInterface
       }
     } catch {
       // Fall back to Spark parse plan logic if flint cannot parse
-      case _: ParseException => sparkParser.parsePlan(sqlText)
+      case e: ParseException =>
+        logInfo(
+          s"Failed to parse PPL with PPL parser. Falling back to Spark parser. PPL: $sqlText",
+          e)
+        sparkParser.parsePlan(sqlText)
+
     }
   }
 

diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala
@@ -391,6 +391,32 @@ trait FlintSparkSuite extends QueryTest with FlintSuite with OpenSearchSuite wit
          |""".stripMargin)
   }
 
+  protected def createStructNestedTable(testTable: String): Unit = {
+    sql(s"""
+         | CREATE TABLE $testTable
+         | (
+         |   int_col INT,
+         |   struct_col  STRUCT<field1: STRUCT<subfield:STRING>, field2: INT>,
+         |   struct_col2 STRUCT<field1: STRUCT<subfield:STRING>, field2: INT>
+         | )
+         | USING JSON
+         |""".stripMargin)
+
+    sql(s"""
+         | INSERT INTO $testTable
+         | SELECT /*+ COALESCE(1) */ *
+         | FROM VALUES
+         | ( 30, STRUCT(STRUCT("value1"),123), STRUCT(STRUCT("valueA"),23) ),
+         | ( 40, STRUCT(STRUCT("value5"),123), STRUCT(STRUCT("valueB"),33) ),
+         | ( 30, STRUCT(STRUCT("value4"),823), STRUCT(STRUCT("valueC"),83) ),
+         | ( 40, STRUCT(STRUCT("value2"),456), STRUCT(STRUCT("valueD"),46) )
+         |""".stripMargin)
+    sql(s"""
+         | INSERT INTO $testTable
+         | VALUES ( 50, STRUCT(STRUCT("value3"),789), STRUCT(STRUCT("valueE"),89) )
+         |""".stripMargin)
+  }
+
   protected def createTableIssue112(testTable: String): Unit = {
     sql(s"""
            | CREATE TABLE $testTable (