diff --git a/CHANGELOG.md b/CHANGELOG.md index 94a48e1..0156faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed +- **Mock Provider Regression Test** — add regression test verifying that `--llm-provider mock` never falls back to `OpenAIProvider` (#40) - **Chunking Metadata Usage** — `ChunkingQualityAnalyzer.analyze()` now uses the `metadata` parameter to perform more informed analysis: checks chunk size deviation from expected, detects excessive character overlap, and adjusts empty chunk thresholds (#65) --- diff --git a/tests/unit/test_cli/test_synth.py b/tests/unit/test_cli/test_synth.py index 57eba11..a829709 100644 --- a/tests/unit/test_cli/test_synth.py +++ b/tests/unit/test_cli/test_synth.py @@ -41,6 +41,25 @@ def test_create_provider_uses_offline_mock_provider() -> None: assert response.model == "synthetic-test-model" +def test_mock_provider_never_instantiates_openai() -> None: + """Regression: --llm-provider mock must NOT create an OpenAI provider. + + Prior to the fix, the mock case fell back to OpenAIProvider, which + required API keys and could incur real API costs. This test verifies + that the OpenAI constructor is never called when provider='mock'. + """ + from unittest.mock import patch + + openai_init = patch( + "openagent_eval.providers.llm.openai.AsyncOpenAI", autospec=True + ) + with openai_init as mock_openai_cls: + provider = _create_provider("mock", "test-model") + + assert isinstance(provider, MockLLMProvider) + mock_openai_cls.assert_not_called() + + @pytest.fixture def fake_generator(monkeypatch: pytest.MonkeyPatch) -> dict: """Replace SyntheticDataGenerator so no LLM/network call is ever made.