Skip to content

Commit fd3c363

Browse files
committed
fix(adapters): tighten hallucination_detection contract and DecodeSentences bounds (#1719)
- Replace the generic list[dict] contract for hallucination_detection with _HallucinationDetectionContract in mellea/backends/adapters/io_contracts.py. - Validate per-record faithfulness against {"faithful", "partial", "unfaithful", "NA"}, integer response_begin/response_end (rejecting bool) with response_begin <= response_end, and str explanation. - Deduplicate identical (response_begin, response_end) spans when faithfulness matches (keeping first record) and raise AdapterSchemaMismatchError on conflicting duplicates or overlapping [begin, end) spans. - Update DecodeSentences._transform to reject bool indices with TypeError and out-of-range sentence indices with ValueError naming the index and sentence count. - Update flag_hallucinated_content, find_citations, and find_context_attributions docstrings and add unit tests in test_io_contracts.py. Assisted-by: Google Antigravity Signed-off-by: CG-5228 <g15222152017@gmail.com>
1 parent 36505ad commit fd3c363

5 files changed

Lines changed: 606 additions & 23 deletions

File tree

‎mellea/backends/adapters/io_contracts.py‎

Lines changed: 152 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,157 @@ def parse(self, raw: str) -> dict[str, object]:
186186
return data
187187

188188

189+
_HALLUCINATION_REQUIRED_KEYS: frozenset[str] = frozenset(
190+
{"response_begin", "response_end", "response_text", "faithfulness", "explanation"}
191+
)
192+
_HALLUCINATION_FAITHFULNESS_LABELS: tuple[str, ...] = (
193+
"faithful",
194+
"partial",
195+
"unfaithful",
196+
"NA",
197+
)
198+
199+
200+
class _HallucinationDetectionContract(IOContract):
201+
"""Validate and deduplicate `hallucination_detection` adapter output.
202+
203+
Unlike other contracts in this module (which return parsed data unchanged),
204+
`parse()` drops duplicate records that share `(response_begin, response_end)`
205+
and agree on `faithfulness` (keeping the first record). Duplicate spans that
206+
disagree on `faithfulness`, or distinct spans whose half-open intervals
207+
`[response_begin, response_end)` overlap, raise `AdapterSchemaMismatchError`.
208+
209+
The `faithfulness` value set (`"faithful"`, `"partial"`, `"unfaithful"`,
210+
`"NA"`) is closed on purpose (fail-closed): on Granite Switch, `io.yaml`
211+
comes from the served model's `io_configs/` while this contract comes from
212+
Mellea's registry, so a Switch release adding a new label raises
213+
`AdapterSchemaMismatchError` until Mellea is upgraded.
214+
"""
215+
216+
def build_prompt(self, **_kwargs: object) -> Component:
217+
raise NotImplementedError(_BUILD_PROMPT_NOT_IMPLEMENTED)
218+
219+
def parse(self, raw: str) -> dict[str, object]:
220+
"""Parse, validate, and deduplicate `hallucination_detection` output.
221+
222+
Args:
223+
raw (str): Raw JSON string from the model.
224+
225+
Returns:
226+
dict[str, object]: `{"items": <list of validated, deduplicated dicts>}`.
227+
228+
Raises:
229+
ValueError: When `raw` is not valid JSON, is not a JSON array, or
230+
contains a non-object element.
231+
AdapterSchemaMismatchError: When any record is missing a required key,
232+
has an invalid `faithfulness` value, has non-`int` (or `bool`) or
233+
inverted span offsets (`response_begin > response_end`), has a
234+
non-`str` `explanation`, conflicts on `faithfulness` for a repeated
235+
`(response_begin, response_end)` span, or overlaps another span's
236+
half-open interval `[response_begin, response_end)`.
237+
"""
238+
data = json.loads(raw)
239+
if not isinstance(data, list):
240+
raise ValueError(
241+
f"Adapter 'hallucination_detection' output must be a JSON array, "
242+
f"got {type(data).__name__}."
243+
)
244+
245+
seen_spans: dict[tuple[int, int], str] = {}
246+
accepted_spans: list[tuple[int, int, frozenset[str]]] = []
247+
deduplicated: list[dict[str, object]] = []
248+
249+
for item in data:
250+
if not isinstance(item, dict):
251+
raise ValueError(
252+
f"Adapter 'hallucination_detection' output array must contain "
253+
f"only JSON objects, got a {type(item).__name__} element."
254+
)
255+
observed = frozenset(item.keys())
256+
missing = _HALLUCINATION_REQUIRED_KEYS - observed
257+
if missing:
258+
raise AdapterSchemaMismatchError(
259+
"hallucination_detection", observed, _HALLUCINATION_REQUIRED_KEYS
260+
)
261+
262+
faithfulness = item["faithfulness"]
263+
if (
264+
not isinstance(faithfulness, str)
265+
or faithfulness not in _HALLUCINATION_FAITHFULNESS_LABELS
266+
):
267+
valid_labels = ", ".join(
268+
repr(lbl) for lbl in _HALLUCINATION_FAITHFULNESS_LABELS
269+
)
270+
raise AdapterSchemaMismatchError(
271+
"hallucination_detection",
272+
observed,
273+
_HALLUCINATION_REQUIRED_KEYS,
274+
f"invalid `faithfulness` value {faithfulness!r}; expected one of "
275+
f"{valid_labels} (upgrade `mellea` if the upstream adapter added "
276+
f"a new label)",
277+
)
278+
279+
begin = item["response_begin"]
280+
end = item["response_end"]
281+
if (
282+
isinstance(begin, bool)
283+
or not isinstance(begin, int)
284+
or isinstance(end, bool)
285+
or not isinstance(end, int)
286+
):
287+
raise AdapterSchemaMismatchError(
288+
"hallucination_detection",
289+
observed,
290+
_HALLUCINATION_REQUIRED_KEYS,
291+
"`response_begin` and `response_end` must be `int` (not `bool`)",
292+
)
293+
if begin > end:
294+
raise AdapterSchemaMismatchError(
295+
"hallucination_detection",
296+
observed,
297+
_HALLUCINATION_REQUIRED_KEYS,
298+
f"`response_begin` ({begin}) must be <= `response_end` ({end})",
299+
)
300+
301+
explanation = item["explanation"]
302+
if not isinstance(explanation, str):
303+
raise AdapterSchemaMismatchError(
304+
"hallucination_detection",
305+
observed,
306+
_HALLUCINATION_REQUIRED_KEYS,
307+
f"`explanation` must be a `str`, got {type(explanation).__name__}",
308+
)
309+
310+
span = (begin, end)
311+
if span in seen_spans:
312+
if seen_spans[span] != faithfulness:
313+
raise AdapterSchemaMismatchError(
314+
"hallucination_detection",
315+
observed,
316+
_HALLUCINATION_REQUIRED_KEYS,
317+
f"conflicting `faithfulness` values ({seen_spans[span]!r} vs "
318+
f"{faithfulness!r}) for duplicate span {span}",
319+
)
320+
continue
321+
322+
seen_spans[span] = faithfulness
323+
accepted_spans.append((begin, end, observed))
324+
deduplicated.append(item)
325+
326+
for i, (a_begin, a_end, _) in enumerate(accepted_spans):
327+
for b_begin, b_end, b_observed in accepted_spans[i + 1 :]:
328+
if max(a_begin, b_begin) < min(a_end, b_end):
329+
raise AdapterSchemaMismatchError(
330+
"hallucination_detection",
331+
b_observed,
332+
_HALLUCINATION_REQUIRED_KEYS,
333+
f"overlapping response spans [{a_begin}, {a_end}) and "
334+
f"[{b_begin}, {b_end})",
335+
)
336+
337+
return {"items": deduplicated}
338+
339+
189340
_INTRINSIC_IO_CONTRACTS: dict[str, IOContract] = {
190341
"answerability": _DictContract("answerability", frozenset({"answerability"})),
191342
"query_rewrite": _DictContract("query_rewrite", frozenset({"rewritten_question"})),
@@ -206,18 +357,7 @@ def parse(self, raw: str) -> dict[str, object]:
206357
}
207358
),
208359
),
209-
"hallucination_detection": _ListContract(
210-
"hallucination_detection",
211-
frozenset(
212-
{
213-
"response_begin",
214-
"response_end",
215-
"response_text",
216-
"faithfulness",
217-
"explanation",
218-
}
219-
),
220-
),
360+
"hallucination_detection": _HallucinationDetectionContract(),
221361
"policy-guardrails": _PolicyGuardrailsContract(),
222362
"guardian-core": _GuardianCheckContract(),
223363
"factuality-detection": _DictContract("factuality-detection", frozenset({"score"})),

‎mellea/formatters/granite/intrinsics/output.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -715,11 +715,17 @@ def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict:
715715
document_ids = prepare_output.get("document_ids")
716716
message_indices = prepare_output.get("message_indices")
717717

718-
if not isinstance(value, int):
718+
if isinstance(value, bool) or not isinstance(value, int):
719719
raise TypeError(
720720
f"Expected integer sentence number at path {path}, but "
721721
f"found non-integer value {value} of type {type(value)}"
722722
)
723+
num_sentences = len(begins)
724+
if value < 0 or value >= num_sentences:
725+
raise ValueError(
726+
f"Sentence number {value} at path {path} is out of range for "
727+
f"{num_sentences} sentence(s)"
728+
)
723729
sentence_num = value
724730

725731
result = {}

‎mellea/stdlib/components/intrinsic/core.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,11 @@ def find_context_attributions(
142142
143143
Raises:
144144
ValueError: When the model output is not valid JSON, is not a
145-
JSON array, or contains a non-object element.
145+
JSON array, contains a non-object element, or contains a
146+
sentence index that is negative or out of range for the decoded
147+
source sentences.
148+
TypeError: When sentence decoding encounters a non-integer (or
149+
`bool`) sentence index in the model output.
146150
AdapterSchemaMismatchError: When any record in the output is missing a
147151
required field.
148152
AloraActivationError: When the resolved adapter is an aLoRA whose

‎mellea/stdlib/components/intrinsic/rag.py‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,11 @@ def find_citations(
218218
219219
Raises:
220220
ValueError: When the model output is not valid JSON, is not a
221-
JSON array, or contains a non-object element.
221+
JSON array, contains a non-object element, or contains a
222+
sentence index that is negative or out of range for the decoded
223+
source sentences.
224+
TypeError: When sentence decoding encounters a non-integer (or
225+
`bool`) sentence index in the model output.
222226
AdapterSchemaMismatchError: When any record in the output is missing a
223227
required field.
224228
AloraActivationError: When the resolved adapter is an aLoRA whose
@@ -254,15 +258,30 @@ def flag_hallucinated_content(
254258
user question are faithful to the retrieved document snippets. Sentences that do
255259
not align with the retrieved snippets are flagged as potential hallucinations.
256260
257-
The `faithfulness` field in each record is a string label (e.g.
258-
`"faithful"`, `"hallucinated"`); coercion to a boolean is the caller's
259-
responsibility.
261+
The `faithfulness` field in each record is one of the four string labels
262+
defined by the adapter's `io.yaml` schema:
263+
- `"faithful"` — the sentence is fully supported by the retrieved documents.
264+
- `"partial"` — the sentence is only partially supported by the retrieved
265+
documents.
266+
- `"unfaithful"` — the sentence is not supported by (or contradicts) the
267+
retrieved documents.
268+
- `"NA"` — faithfulness does not apply (for example, the sentence makes no
269+
factual claim that needs grounding in the documents).
270+
271+
Coercion to a boolean is the caller's responsibility.
260272
261273
Output contract — each record must contain: `response_begin`,
262274
`response_end`, `response_text`, `faithfulness`, `explanation`. A
263-
record missing any of these keys raises
264-
:class:`~mellea.backends.adapters.AdapterSchemaMismatchError`; extra optional
265-
keys do not raise (forward-compatible).
275+
record missing any of these keys raises `AdapterSchemaMismatchError`;
276+
extra optional keys do not raise (forward-compatible). In addition, the
277+
contract validates that `faithfulness` is one of `"faithful"`, `"partial"`,
278+
`"unfaithful"`, or `"NA"`; `response_begin` and `response_end` are `int`
279+
(not `bool`) with `response_begin <= response_end`; and `explanation` is a
280+
`str`. Duplicate records for the same `(response_begin, response_end)`
281+
span that agree on `faithfulness` are deduplicated (keeping the first
282+
record), whereas duplicate spans with conflicting `faithfulness` values or
283+
distinct spans whose half-open intervals `[response_begin, response_end)`
284+
overlap raise `AdapterSchemaMismatchError`.
266285
267286
Args:
268287
response: The assistant's response to the user's question in the last turn
@@ -284,9 +303,17 @@ def flag_hallucinated_content(
284303
285304
Raises:
286305
ValueError: When the model output is not valid JSON, is not a
287-
JSON array, or contains a non-object element.
306+
JSON array, contains a non-object element, or contains a
307+
sentence index that is negative or out of range for the decoded
308+
response sentences.
309+
TypeError: When sentence decoding encounters a non-integer (or
310+
`bool`) sentence index in the model output.
288311
AdapterSchemaMismatchError: When any record in the output is missing a
289-
required field.
312+
required field, has an invalid `faithfulness` label, has non-`int`
313+
(or `bool`) or inverted span offsets (`response_begin > response_end`),
314+
has a non-`str` `explanation`, has conflicting `faithfulness`
315+
labels for the same `(response_begin, response_end)` span, or has
316+
overlapping `[response_begin, response_end)` spans.
290317
AloraActivationError: When the resolved adapter is an aLoRA whose
291318
declared activation sequence is absent from the assembled prompt.
292319
"""

0 commit comments

Comments
 (0)