Fix extractFromSemiStructured in duckdbExtension

legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-function/src/test/java/org/finos/legend/engine/testable/function/TestFunctionTestSuite.java

@@ -262,6 +262,18 @@ public void testModelRelationFunctionTestWithRelationAccessor()
         testFailingTest(findTestById(testResults, "testFail"), expected, actual);
     }
 
+    @Test
+    public void testRelationFunctionWithSemiStructuredExtract()
+    {
+        List<TestResult> varcharResults = executeFunctionTest("legend-testable-function-test-relation-semistructured.pure", "model::FirmLegalNameQuery__Relation_1_");
+        Assert.assertEquals(1, varcharResults.size());
+        Assert.assertTrue(hasTestPassed(findTestById(varcharResults, "test_extractVarcharIsUnquoted")));
+
+        List<TestResult> intResults = executeFunctionTest("legend-testable-function-test-relation-semistructured.pure", "model::FirmEmployeeCountQuery__Relation_1_");
+        Assert.assertEquals(1, intResults.size());
+        Assert.assertTrue(hasTestPassed(findTestById(intResults, "test_extractIntegerCast")));
+    }
+
     @Test
     public void testRelationFunctionWithDateFilterParameter()
     {

legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-function/src/test/resources/testable/legend-testable-function-test-relation-semistructured.pure

@@ -0,0 +1,125 @@
+###Relational
+Database store::SemiStructuredTestDB
+(
+    Table FIRM_TABLE
+    (
+      ID INTEGER PRIMARY KEY,
+      FIRM_DETAILS SEMISTRUCTURED
+    )
+)
+
+
+###Pure
+import meta::pure::precisePrimitives::*;
+
+Class model::SemiStructuredFirm
+{
+  id: Integer[1];
+  legalName: String[1];
+  employeeCount: Integer[1];
+}
+
+function model::FirmLegalNameQuery(): meta::pure::metamodel::relation::Relation<('LegalName':String)>[1]
+{
+  model::SemiStructuredFirm.all()
+    ->project(~['LegalName': f | $f.legalName])
+    ->from(
+       execution::SemiStructuredMapping,
+       execution::SemiStructuredRuntime
+    )
+}
+{
+  testSuite_1
+  (
+    store::SemiStructuredTestDB:
+        Relation
+        #{
+          default.FIRM_TABLE:
+            ID,FIRM_DETAILS
+            1,"{""legalName"":""Firm X"",""employeeCount"":4}"
+            2,"{""legalName"":""Firm A"",""employeeCount"":1}";
+        }#;
+    test_extractVarcharIsUnquoted | FirmLegalNameQuery() => Relation
+                                                                                                                          #{
+                                                                                                                              LegalName
+                                                                                                                              Firm A
+                                                                                                                              Firm X;
+                                                                                                                          }#;
+  )
+}
+
+function model::FirmEmployeeCountQuery(): meta::pure::metamodel::relation::Relation<('Employee Count':Integer)>[1]
+{
+  model::SemiStructuredFirm.all()
+    ->project(~['Employee Count': f | $f.employeeCount])
+    ->from(
+       execution::SemiStructuredMapping,
+       execution::SemiStructuredRuntime
+    )
+}
+{
+  testSuite_1
+  (
+    store::SemiStructuredTestDB:
+        Relation
+        #{
+          default.FIRM_TABLE:
+            ID,FIRM_DETAILS
+            1,"{""legalName"":""Firm X"",""employeeCount"":4}"
+            2,"{""legalName"":""Firm A"",""employeeCount"":1}";
+        }#;
+    test_extractIntegerCast | FirmEmployeeCountQuery() => Relation
+                                                              #{
+                                                                  Employee Count
+                                                                  1
+                                                                  4;
+                                                              }#;
+  )
+}
+
+
+###Mapping
+Mapping execution::SemiStructuredMapping
+(
+  *model::SemiStructuredFirm: Relational
+  {
+    ~primaryKey
+    (
+      [store::SemiStructuredTestDB]FIRM_TABLE.ID
+    )
+    ~mainTable [store::SemiStructuredTestDB]FIRM_TABLE
+    id: [store::SemiStructuredTestDB]FIRM_TABLE.ID,
+    legalName: extractFromSemiStructured([store::SemiStructuredTestDB]FIRM_TABLE.FIRM_DETAILS, 'legalName', 'VARCHAR'),
+    employeeCount: extractFromSemiStructured([store::SemiStructuredTestDB]FIRM_TABLE.FIRM_DETAILS, 'employeeCount', 'INTEGER')
+  }
+)
+
+
+###Connection
+RelationalDatabaseConnection model::SemiStructuredConnection
+{
+  store: store::SemiStructuredTestDB;
+  type: H2;
+  specification: LocalH2
+  {
+  };
+  auth: DefaultH2;
+}
+
+
+###Runtime
+Runtime execution::SemiStructuredRuntime
+{
+  mappings:
+  [
+    execution::SemiStructuredMapping
+  ];
+  connections:
+  [
+    store::SemiStructuredTestDB:
+    [
+      connection_1: model::SemiStructuredConnection
+    ]
+  ];
+}
+

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure

@@ -351,7 +351,7 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::duckD
 
   let elementAccess = format('json_extract(%s, \'$.%s\')', [$baseRelationalOp, $pathNavigation]);
 
-  if ($returnType->in(['CHAR', 'VARCHAR', 'STRING']), | 'cast(' + $elementAccess + ' as VARCHAR)', |
+  if ($returnType->in(['CHAR', 'VARCHAR', 'STRING']), | format('json_extract_string(%s, \'$.%s\')', [$baseRelationalOp, $pathNavigation]), |
   if ($returnType->in(['DATE','DATETIME', 'TIMESTAMP']), | 'cast(' + $elementAccess + ' as TIMESTAMP)', |
   if ($returnType == 'BOOLEAN', | 'cast(' + $elementAccess + ' as BOOLEAN)', |
   if ($returnType == 'FLOAT', | 'cast(' + $elementAccess + ' as DOUBLE)', |

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/tests/testDuckDBSQLGeneration.pure

@@ -74,4 +74,12 @@ function <<test.Test>> meta::relational::tests::sqlQueryToString::duckDB::testTo
 {
    let s = toSQLString(|SqlFunctionDemo.all()->project([s | $s.convertToDateUserDefinedFormat2], ['convertToDate']), testMapping, meta::relational::runtime::DatabaseType.DuckDB, meta::relational::extension::relationalExtensions());
    assertEquals('select strptime("root".stringUserDefinedDateFormat, \'%Y/%m/%d\')::DATE as "convertToDate" from dataTable as "root"', $s);
-}
+}
+
+function <<test.Test>> meta::relational::tests::sqlQueryToString::duckDB::testToSQLStringExtractFromSemiStructuredVarcharInDuckDB(): Boolean[1]
+{
+   let s = toSQLString(|SqlFunctionDemo.all()->project([s | $s.extractedLegalName], ['Legal Name']), testMapping, meta::relational::runtime::DatabaseType.DuckDB, meta::relational::extension::relationalExtensions());
+   print('Succeeded');
+   assertEquals('select json_extract_string("root".semiStructuredCol, \'$.legalName\') as "Legal Name" from dataTable as "root"', $s);
+}
+

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/sqlFunction/testSqlFunctionsInMapping.pure

@@ -718,6 +718,9 @@ Class meta::relational::tests::mapping::sqlFunction::model::domain::SqlFunctionD
    adjustDate: DateTime[1];
    toBoolean: Boolean[1];
    convertTimeZone: String[1];
+
+   extractedLegalName: String[1];
+   extractedEmployeeCount: Integer[1];
 }
 
 Class meta::relational::tests::mapping::sqlFunction::model::domain::SqlFunctionDemoForPresto
@@ -750,7 +753,8 @@ Database meta::relational::tests::mapping::sqlFunction::model::store::myDB
         float1 Float,
         stringUserDefinedDateFormat VARCHAR(7),
         stringToInt VARCHAR(5),
-        alphaNumericString VARCHAR(15)
+        alphaNumericString VARCHAR(15),
+        semiStructuredCol SEMISTRUCTURED
     )
 )
 
@@ -834,7 +838,10 @@ Mapping meta::relational::tests::mapping::sqlFunction::model::mapping::testMappi
           dateTime: dateTime,
           adjustDate: adjust(dateTime, -7, 'DAYS'),
           toBoolean: parseBoolean('true'),
-          convertTimeZone: convertTimeZone( dateTime,'EST','yyyy-MM-dd HH:mm:ss')
+          convertTimeZone: convertTimeZone( dateTime,'EST','yyyy-MM-dd HH:mm:ss'),
+
+          extractedLegalName: extractFromSemiStructured(semiStructuredCol, 'legalName', 'VARCHAR'),
+          extractedEmployeeCount: extractFromSemiStructured(semiStructuredCol, 'employeeCount', 'INTEGER')
        )
 
     }