From 59730fe7dae07ad23810603d609d2486e0f2203a Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 23 Jul 2026 21:10:38 +0200 Subject: [PATCH] corpus(auditor): enforce max_documents cumulatively across .jsonl files (#224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In `_load_documents`' directory branch the cumulative cap check (`len(documents) >= self.max_documents`) sat only inside the non-jsonl `else` branch, so it never ran after a `.jsonl` file. `_load_jsonl_documents` caps within a single file only (its local list resets per call), so a directory of `.jsonl` files silently blew past `max_documents` — a memory / safety bound — while a directory of `.txt` files capped correctly. Move the cap check out of the `else` so it runs after both branches, and trim the extended list to `max_documents` before breaking. Behavior for single-file loads and `.txt`-only directories is unchanged (a per-file `.txt` append never overshoots, so the trim is a no-op there). Regression: a directory of 2 jsonl files x 3 lines with max_documents=2 now loads exactly 2 (was 4), and a mixed jsonl+txt directory caps at 2 (was 3). Co-Authored-By: Claude Opus 4.8 --- openagent_eval/corpus/auditor.py | 9 ++++-- tests/unit/test_corpus/test_auditor.py | 38 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/openagent_eval/corpus/auditor.py b/openagent_eval/corpus/auditor.py index f19920d..cbcd058 100644 --- a/openagent_eval/corpus/auditor.py +++ b/openagent_eval/corpus/auditor.py @@ -198,8 +198,13 @@ def _load_documents(self, path: Path) -> list[CorpusDocument]: doc = self._load_single_file(file_path) if doc: documents.append(doc) - if len(documents) >= self.max_documents: - break + # Enforce the cumulative max_documents cap across every + # parsed file. _load_jsonl_documents only caps within a + # single file, so without trimming here a directory of + # .jsonl files would blow past the limit (#224). + if len(documents) >= self.max_documents: + documents = documents[: self.max_documents] + break return documents diff --git a/tests/unit/test_corpus/test_auditor.py b/tests/unit/test_corpus/test_auditor.py index 4a4bc6b..521b2f6 100644 --- a/tests/unit/test_corpus/test_auditor.py +++ b/tests/unit/test_corpus/test_auditor.py @@ -2,6 +2,7 @@ from __future__ import annotations +from pathlib import Path import pytest @@ -109,6 +110,43 @@ async def test_max_documents_limit(self, tmp_path): assert report.total_documents <= 3 + def test_max_documents_cap_across_jsonl_files(self, tmp_path): + """#224: the cap must apply cumulatively across .jsonl files. + + Two .jsonl files of 3 valid lines each (6 documents available) with + max_documents=2 must load exactly 2 — the old code only enforced the + cap in the non-jsonl branch, so it counted per-file and returned 4. + """ + corpus_dir = tmp_path / "corpus" + corpus_dir.mkdir() + for name in ("a", "b"): + lines = [f'{{"text": "{name}{i}"}}' for i in range(3)] + (corpus_dir / f"{name}.jsonl").write_text( + "\n".join(lines), encoding="utf-8" + ) + + auditor = CorpusAuditor(checks=["staleness"], max_documents=2) + documents = auditor._load_documents(Path(corpus_dir)) + + assert len(documents) == 2 + + def test_max_documents_cap_across_mixed_directory(self, tmp_path): + """#224: the cap applies across a mix of .jsonl and .txt files.""" + corpus_dir = tmp_path / "corpus" + corpus_dir.mkdir() + # Sorted rglob visits a.jsonl (3 lines) before b.txt / c.txt. + (corpus_dir / "a.jsonl").write_text( + "\n".join(f'{{"text": "a{i}"}}' for i in range(3)), + encoding="utf-8", + ) + (corpus_dir / "b.txt").write_text("plain text b", encoding="utf-8") + (corpus_dir / "c.txt").write_text("plain text c", encoding="utf-8") + + auditor = CorpusAuditor(checks=["staleness"], max_documents=2) + documents = auditor._load_documents(Path(corpus_dir)) + + assert len(documents) == 2 + @pytest.mark.asyncio async def test_load_documents_skips_empty_files(self, tmp_path): """Test that empty files are skipped."""