Add null check when pure multiplicity is nullable

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/transformation/join.pure

@@ -146,7 +146,7 @@ function <<PCT.test>> meta::pure::functions::relation::tests::join::testJoin_for
 function <<PCT.test>> meta::pure::functions::relation::tests::join::testFullJoin<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
 {
   let tds = #TDS
-              id, name
+              id:Integer[1], name
               1, George
               2, Pierre
               3, Sachin

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/tests/composition.pure

@@ -721,6 +721,42 @@ function <<PCT.test>> meta::pure::functions::relation::tests::composition::testP
                 '#', $res->sort([~id->ascending(),~name->ascending()])->toString());
 }
 
+function <<PCT.test, PCTRelationQualifier.relation>> meta::pure::functions::relation::tests::composition::testJoinOnNullKey<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+  let tds = #TDS
+              id, name
+              1, George
+              null, Null Left 1
+              2, Pierre
+              null, Null Left 2
+            #;
+
+  let tds2 = #TDS
+               id2, col
+               1, More George
+               null, Null Right 1
+               null, Null Right 2
+               3, More Sachin
+             #;
+
+  let expr = {|
+    $tds
+      ->join($tds2, JoinKind.INNER, {x,y| $x.id == $y.id2})
+      ->project(~[id : x | $x.id, name : x | $x.name, col : x | $x.col])
+  };
+
+  let res = $f->eval($expr);
+
+  assertEquals( '#TDS\n'+
+                '   id,name,col\n'+
+                '   1,George,More George\n'+
+                '   null,Null Left 1,Null Right 1\n'+
+                '   null,Null Left 1,Null Right 2\n'+
+                '   null,Null Left 2,Null Right 1\n'+
+                '   null,Null Left 2,Null Right 2\n'+
+                '#', $res->sort([~name->ascending(), ~col->ascending()])->toString());
+}
+
 function <<PCT.test, PCTRelationQualifier.relation, PCTCoreQualifier.variant>> meta::pure::functions::relation::tests::composition::testVariantArrayColumn_sort<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
 {
    let tds = #TDS

legend-engine-xts-deephaven/legend-engine-xt-deephaven-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/test/deephaven/pct/Test_Deephaven_RelationFunctions_PCT.java

@@ -112,6 +112,7 @@ public class Test_Deephaven_RelationFunctions_PCT extends PCTReportConfiguration
             one("meta::pure::functions::relation::tests::composition::testProjectExtendNestedIfLeadAdjust_Function_1__Boolean_1_", "\"Match failure: PlatformClusteredValueSpecificationObject instanceOf PlatformClusteredValueSpecification\""),
             one("meta::pure::functions::relation::tests::composition::testVariantArrayColumn_joinStrings_Function_1__Boolean_1_", "\"Cannot cast a collection of size 0 to multiplicity [1]\""),
             one("meta::pure::functions::relation::tests::composition::testConcatenateWithJoinOnOneSide_Function_1__Boolean_1_", "\"Match failure: LambdaFunctionObject instanceOf LambdaFunction\""),
+            one("meta::pure::functions::relation::tests::composition::testJoinOnNullKey_Function_1__Boolean_1_", "\"function not supported yet: meta::pure::functions::relation::project_Relation_1__FuncColSpecArray_1__Relation_1_\""),
 
             one("meta::pure::functions::relation::tests::concatenate::testSimpleConcatenateShared_Function_1__Boolean_1_", "\"function not supported yet: meta::pure::functions::relation::concatenate_Relation_1__Relation_1__Relation_1_\""),
             one("meta::pure::functions::relation::tests::concatenate::testSimpleConcatenate_MultipleExpressions_Function_1__Boolean_1_", "\"Match failure: LambdaFunctionObject instanceOf LambdaFunction\""),

legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java

@@ -138,6 +138,7 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration
             one("meta::pure::functions::relation::tests::composition::testNestedJoinArithmeticComparisonExpression_Function_1__Boolean_1_", "\"Instance of type 'meta::pure::metamodel::relation::TDS' can't be translated\"", AdapterQualifier.unsupportedFeature),
             one("meta::pure::functions::relation::tests::composition::testExtendLeadAdjustDerivedOffset_Function_1__Boolean_1_", "\"Instance of type 'meta::pure::metamodel::relation::TDS' can't be translated\"", AdapterQualifier.unsupportedFeature),
             one("meta::pure::functions::relation::tests::composition::testProjectExtendNestedIfLeadAdjust_Function_1__Boolean_1_", "\"Instance of type 'meta::pure::metamodel::relation::TDS' can't be translated\"", AdapterQualifier.unsupportedFeature),
+            one("meta::pure::functions::relation::tests::composition::testJoinOnNullKey_Function_1__Boolean_1_", "\"Instance of type 'meta::pure::metamodel::relation::TDS' can't be translated\"", AdapterQualifier.unsupportedFeature),
             one("meta::pure::functions::relation::tests::composition::testVariantArrayColumn_joinStrings_Function_1__Boolean_1_", "\"Instance of type 'meta::pure::metamodel::relation::TDS' can't be translated\"", AdapterQualifier.unsupportedFeature),
             one("meta::pure::functions::relation::tests::composition::testConcatenateWithJoinOnOneSide_Function_1__Boolean_1_", "\"Instance of type 'meta::pure::metamodel::relation::TDS' can't be translated\"", AdapterQualifier.unsupportedFeature)
     );

legend-engine-xts-python/legend-engine-xt-python-reversePCT-legendQL/src/main/resources/core_external_python_reverse_pct_legend_ql/pythonReversePCTLegendQLApi.pure

@@ -26,7 +26,7 @@ function <<PCT.adapter>> {PCT.adapterName = 'Python - LegendQL'} meta::external:
     | let pythonString = $rev.reverse->toOne();
       let func = getPureLambdaForPythonCodeBlock($pythonString);
       // println($func->meta::pure::router::printer::asString());
-      $func->cast(@Function<{->X[o]}>)->eval();,
+      $func->evaluate([])->toMultiplicity(@[o]);,
     | fail('Should not be supported'); @X->toMultiplicity(@[o]);
       // ('{' + $rev.function + '}')->compileValueSpecification().result->toOne()->cast(@InstanceValue).values->toOne()->cast(@LambdaFunction<{->X[o]}>)->eval()
   );

legend-engine-xts-python/legend-engine-xt-python-reversePCT-legendQL/src/main/resources/core_external_python_reverse_pct_legend_ql/relation/relation.pure

@@ -1910,7 +1910,7 @@ function meta::external::python::reversePCT::legendQL::pythonLegendQLReversesEss
 
       meta::pure::functions::relation::tests::join::testFullJoin_Function_1__Boolean_1_->testRev(
         [
-          rev('|$tds->meta::pure::functions::relation::join($tds2, meta::pure::functions::relation::JoinKind.FULL, {x: (id:Integer[0..1], name:String[0..1])[1], y: (id2:Integer[0..1], col:String[0..1], other:Integer[0..1])[1]|$x.id == $y.id2})', 
+          rev('|$tds->meta::pure::functions::relation::join($tds2, meta::pure::functions::relation::JoinKind.FULL, {x: (id:Integer[1], name:String[0..1])[1], y: (id2:Integer[0..1], col:String[0..1], other:Integer[0..1])[1]|$x.id == $y.id2})', 
                '(lambda frame:' +
                '  frame.join(csv_frame("id2,col,other\\n1,More George 1,1\\n1,More George 2,2\\n4,More David,1\\n6,John,3"), (lambda x , y : x.id == y.id2), "FULL")' +
                ')(csv_frame("id,name\\n1,George\\n2,Pierre\\n3,Sachin\\n4,David\\n5,James"))')

legend-engine-xts-python/legend-engine-xt-python-reversePCT-pandasAPI/src/main/resources/core_external_python_reverse_pct_pandas_api/pythonReversePCTPandasAPIApi.pure

@@ -25,7 +25,7 @@ function <<PCT.adapter>> {PCT.adapterName = 'Python - PandasAPI'} meta::external
   if ($rev.shouldBeSupported,
     | let pythonString = $rev.reverse->toOne();
       let func = getPureLambdaForPandasAPIPythonCodeBlock($pythonString);
-      $func->cast(@Function<{->X[o]}>)->eval();,
+      $func->evaluate([])->toMultiplicity(@[o]);,
     | fail('Should not be supported'); @X->toMultiplicity(@[o]);
   );
 }

legend-engine-xts-python/legend-engine-xt-python-reversePCT-pandasAPI/src/main/resources/core_external_python_reverse_pct_pandas_api/relation/relation.pure

@@ -1630,7 +1630,7 @@ function meta::external::python::reversePCT::pandasAPI::pythonPandasAPIReversesE
 
       meta::pure::functions::relation::tests::join::testFullJoin_Function_1__Boolean_1_->testRev(
         [
-          rev('|$tds->meta::pure::functions::relation::join($tds2, meta::pure::functions::relation::JoinKind.FULL, {x: (id:Integer[0..1], name:String[0..1])[1], y: (id2:Integer[0..1], col:String[0..1], other:Integer[0..1])[1]|$x.id == $y.id2})',
+          rev('|$tds->meta::pure::functions::relation::join($tds2, meta::pure::functions::relation::JoinKind.FULL, {x: (id:Integer[1], name:String[0..1])[1], y: (id2:Integer[0..1], col:String[0..1], other:Integer[0..1])[1]|$x.id == $y.id2})',
                '(lambda frame:' +
                '  frame.merge(csv_frame("id2,col,other\\n1,More George 1,1\\n1,More George 2,2\\n4,More David,1\\n6,John,3"), how="outer", left_on=["id"], right_on=["id2"])' +
                ')(csv_frame("id,name\\n1,George\\n2,Pierre\\n3,Sachin\\n4,David\\n5,James"))')

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-clickhouse/legend-engine-xt-relationalStore-clickhouse-pure/src/main/resources/core_relational_clickhouse/relational/sqlQueryToString/clickHouseExtension.pure

@@ -1,4 +1,5 @@
 import meta::relational::metamodel::*;
+import meta::relational::metamodel::join::*;
 import meta::pure::extension::*;
 import meta::relational::metamodel::relation::*;
 import meta::relational::functions::sqlQueryToString::default::*;
@@ -171,6 +172,23 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::click
    $s->processSelectSQLQueryForClickHouse($sgc.dbConfig, $sgc.format, $sgc.config, $isSubSelect, $sgc.extensions);
 }
 
+function <<access.private>> meta::relational::functions::sqlQueryToString::clickhouse::hasNullableJoinInTree(data:RelationalTreeNode[0..1]):Boolean[1]
+{
+   if($data->isEmpty(),
+      | false,
+      | $data->toOne()->match([
+           r:RootJoinTreeNode[1] | $r->children()->cast(@JoinTreeNode)->exists(c | $c->hasNullableJoinInTree()),
+           j:JoinTreeNode[1]     | $j.joinType == JoinType.FULL_OUTER || $j.joinType == JoinType.RIGHT_OUTER || $j->children()->cast(@JoinTreeNode)->exists(c | $c->hasNullableJoinInTree()),
+           a:Any[1]              | false
+        ])
+   );
+}
+
+function <<access.private>> meta::relational::functions::sqlQueryToString::clickhouse::hasNullableJoinInTree(j:JoinTreeNode[1]):Boolean[1]
+{
+   $j.joinType == JoinType.FULL_OUTER || $j.joinType == JoinType.RIGHT_OUTER || $j->children()->cast(@JoinTreeNode)->exists(c | $c->hasNullableJoinInTree());
+}
+
 function <<access.private>> meta::relational::functions::sqlQueryToString::clickhouse::processSelectSQLQueryForClickHouse(s:SelectSQLQuery[1], dbConfig:DbConfig[1], format:Format[1], config:Config[1], isSubSelect : Boolean[1], extensions:Extension[*]):String[1]
 {
   assertEmpty($s.pivot, 'pivot is not supported for ClickHouse');
@@ -179,14 +197,17 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::click
   let havingStr = if($s.havingOperation->isEmpty(), |'', |$s.havingOperation->map(s|$s->processOperation($dbConfig, $format->indent(), $config, $extensions))->filter(s|$s != '')->joinStrings(' <||> '));
   let qualifyStr = if($s.qualifyOperation->isEmpty(), |'', |$s.qualifyOperation->map(s|$s->processOperation($dbConfig, $format->indent(), $config, $extensions))->filter(s|$s != '')->joinStrings(' <||> '));
 
+  let needsJoinUseNulls = hasNullableJoinInTree($s.data);
+
   $format.separator + 'select ' + if($s.distinct == true,|'distinct ',|'') +
   processSelectColumns($s.columns, $dbConfig, $format->indent(), true, $extensions) +
   if($s.data == [],|'',| ' ' + $format.separator + 'from ' + $s.data->toOne()->processJoinTreeNode([], $dbConfig, $format->indent(), [], $extensions)) +
   if (eq($opStr, ''), |'', | ' ' + $format.separator + 'where ' + $opStr) +
   if ($s.groupBy->isEmpty(),|'',| ' ' + $format.separator + 'group by '+$s.groupBy->processGroupByColumns($dbConfig, $format->indent(), true, $extensions)->makeString(','))+
   if (eq($havingStr, ''), |'', | ' ' + $format.separator + 'having ' + $havingStr) +
   if ($s.orderBy->isEmpty(),|'',| ' ' + $format.separator + 'order by '+ $s.orderBy->processOrderBy($dbConfig, $format->indent(), $config, $extensions)->makeString(','))+
-  processLimit($s, $dbConfig, $format, $extensions, processTakeDefault_SelectSQLQuery_1__Format_1__DbConfig_1__Extension_MANY__String_1_, processSliceOrDropForClickHouse_SelectSQLQuery_1__Format_1__DbConfig_1__Extension_MANY__Any_1__String_1_);
+  processLimit($s, $dbConfig, $format, $extensions, processTakeDefault_SelectSQLQuery_1__Format_1__DbConfig_1__Extension_MANY__String_1_, processSliceOrDropForClickHouse_SelectSQLQuery_1__Format_1__DbConfig_1__Extension_MANY__Any_1__String_1_) +
+  if($needsJoinUseNulls, | ' SETTINGS join_use_nulls = 1', | '');
 }
 
 function <<access.private>> meta::relational::functions::sqlQueryToString::clickhouse::processSliceOrDropForClickHouse(s:SelectSQLQuery[1], format:Format[1],dbConfig : DbConfig[1], extensions:Extension[*], size:Any[1]):String[1]

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure

@@ -8715,7 +8715,13 @@ function meta::relational::functions::pureToSqlQuery::processEquals(
                                               |^FreeMarkerOperationHolder(name = 'optionalVarPlaceHolderOpSelector', parameters = [$rightVal, ^DynaFunction(name = 'equal', parameters = [$leftVal, $rightVal]), ^DynaFunction(name = 'isNull', parameters = [$leftVal])]),
                                               |if($leftVal->hasOptionalVarPlaceHolderValue(),
                                                   |^FreeMarkerOperationHolder(name = 'optionalVarPlaceHolderOpSelector', parameters = [$leftVal, ^DynaFunction(name = 'equal', parameters = [$leftVal, $rightVal]), ^DynaFunction(name = 'isNull', parameters = [$rightVal])]),
-                                                  |^DynaFunction(name = 'equal', parameters = [$leftVal, $rightVal]));
+                                                  |if(!$state.inFilter && $leftParam.multiplicity->getLowerBound() == 0 && $rightParam.multiplicity->getLowerBound() == 0,
+                                                      |^DynaFunction(name = 'group', parameters = [^DynaFunction(name = 'or', parameters = [
+                                                          ^DynaFunction(name = 'equal', parameters = [$leftVal, $rightVal]),
+                                                          ^DynaFunction(name = 'group', parameters = [^DynaFunction(name = 'and', parameters = [^DynaFunction(name = 'isNull', parameters = [$leftVal]), ^DynaFunction(name = 'isNull', parameters = [$rightVal])])])
+                                                      ])]),
+                                                      |^DynaFunction(name = 'equal', parameters = [$leftVal, $rightVal])
+                                                  ));
                                               );
                                           );
                                      );

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/modelJoin/modelJoinAdvancedSetup.pure

@@ -438,14 +438,14 @@ Mapping meta::relational::tests::mapping::modelJoin::advanced::ComplexRelationFu
     lastName: LASTNAME,
     age: AGE,
     +firmId: Integer[1]: FIRMID,
-    +city: String[0..1]: CITY
+    +city: String[1]: CITY
   }
   *meta::relational::tests::mapping::modelJoin::domain::Firm[firm]: Relation
   {
     ~func meta::relational::tests::mapping::modelJoin::function::firmTable():Relation<Any>[1]
     legalName: LEGALNAME,
     +id: Integer[1]: ID,
-    +city: String[0..1]: CITY
+    +city: String[1]: CITY
   }
 
   Person_Firm: ModelJoin

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/modelJoin/modelJoinSimpleSetup.pure

@@ -322,14 +322,14 @@ Mapping meta::relational::tests::mapping::modelJoin::simple::FunctionInCondition
     lastName: LASTNAME,
     age: AGE,
     +firmId: Integer[1]: FIRMID,
-    +code: String[0..1]: LASTNAME
+    +code: String[1]: LASTNAME
   }
   *meta::relational::tests::mapping::modelJoin::domain::Firm[firm]: Relation
   {
     ~func meta::relational::tests::mapping::modelJoin::function::firmTable():Relation<Any>[1]
     legalName: LEGALNAME,
     +id: Integer[1]: ID,
-    +code: String[0..1]: CODE
+    +code: String[1]: CODE
   }
 
   meta::relational::tests::mapping::modelJoin::domain::Person_Firm: ModelJoin

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure

@@ -767,3 +767,19 @@ function <<test.Test, test.AlloyOnly>> meta::relational::tests::functions::sqlst
           + 'rawtohex(hash(\'MD5\', concat("root".FIRSTNAME, "root".LASTNAME, \'|\'))) as "tds_digest" from personTable as "root"', $result);
 
 }
+
+function <<test.Test>> meta::relational::tests::functions::sqlstring::testNullSafeEqualityForOptionalProperties():Boolean[1]
+{
+   let s = toSQLString(
+      |Person.all()->project([
+         col(p | $p.firstName, 'name'),
+         col(p | $p.age == $p.manager.age, 'match')
+      ]),
+      simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()
+   );
+   assertEquals('select "root".FIRSTNAME as "name", "root".AGE = "personTable_d#5_d#2_m1_d_m2".AGE '
+      + 'or ("root".AGE is null and "personTable_d#5_d#2_m1_d_m2".AGE is null) as "match" '
+      + 'from personTable as "root" left outer join personTable as "personTable_d#5_d#2_m1_d_m2" '
+      + 'on ("root".MANAGERID = "personTable_d#5_d#2_m1_d_m2".ID)', $s);
+}
+