Skip to content

Commit 9c35e39

Browse files
yuli-hanyuli_han
andauthored
[Backfill]Remove exception when the keys are all null in unfilledLeftDf (#853)
* remove exception when the keys are all null * update warning message * use array * fix bug * fix bug --------- Co-authored-by: yuli_han <[email protected]>
1 parent ce70de5 commit 9c35e39

File tree

1 file changed

+7
-5
lines changed

1 file changed

+7
-5
lines changed

spark/src/main/scala/ai/chronon/spark/JoinUtils.scala

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -304,11 +304,10 @@ object JoinUtils {
304304
s"KeyName: $keyName, leftSide KeyName: $leftSideKeyName , Join right to left: ${joinPart.rightToLeft
305305
.mkString(", ")}")
306306
val values = collectedLeft.map(row => row.getAs[Any](leftSideKeyName))
307-
// Check for null keys, warn if found, err if all null
307+
// Check for null keys, warn if found
308308
val (notNullValues, nullValues) = values.partition(_ != null)
309309
if (notNullValues.isEmpty) {
310-
throw new RuntimeException(
311-
s"No not-null keys found for key: $keyName. Check source table or where clauses.")
310+
logger.warn(s"No not-null keys found for key: $keyName.")
312311
} else if (!nullValues.isEmpty) {
313312
logger.warn(s"Found ${nullValues.length} null keys for key: $keyName.")
314313
}
@@ -319,8 +318,11 @@ object JoinUtils {
319318
case other => other.toString // Keep other types (like Int) as they are
320319
}.toSet
321320

322-
// Form the final WHERE clause for injection
323-
s"$groupByKeyExpression in (${valueSet.mkString(sep = ",")})"
321+
if (valueSet.isEmpty) {
322+
"false"
323+
} else {
324+
s"$groupByKeyExpression in (${valueSet.mkString(sep = ",")})"
325+
}
324326
}
325327
.foreach { whereClause =>
326328
// Skip adding the filter if it is a join source

0 commit comments

Comments
 (0)