diff --git a/src/xrouter_llm/resources/config/models/claude-opus-5.yaml b/src/xrouter_llm/resources/config/models/claude-opus-5.yaml new file mode 100644 index 0000000..457ddd4 --- /dev/null +++ b/src/xrouter_llm/resources/config/models/claude-opus-5.yaml @@ -0,0 +1,22 @@ +# One model per file. Load the registry with: --benchmark-profiles config/models +# Benchmarks are PUBLISHED PERCENTAGES (0-100); featurizer divides >1 by 100. +# Only sourced/official values are active; unverified numbers stay in comments. +# Researched 2026-07-26. +model_id: anthropic/claude-opus-5 +provider: anthropic +aliases: ["claude-opus-5"] +release_date: "2026-07-24" +source_quality: third_party # specs/pricing are official; gpqa/lcb are AA/Vals +source_urls: + - https://www.anthropic.com/news/claude-opus-5 + - https://platform.claude.com/docs/en/about-claude/models/overview + - https://openrouter.ai/anthropic/claude-opus-5 + - https://artificialanalysis.ai/models/claude-opus-5 + - https://www.vals.ai/models/anthropic_claude-opus-5 +context_length: 1000000 +max_output_tokens: 128000 +input_cost_per_1k: 0.005 +output_cost_per_1k: 0.025 +benchmarks: + gpqa_diamond: 93.2 # Artificial Analysis (93.232%, adaptive, max effort); Vals 93.434% + livecodebench: 89.0 # Vals LCB v6 (89.033%, max effort); AA has not run LCB diff --git a/src/xrouter_llm/resources/config/models/moonshotai-kimi-k3.yaml b/src/xrouter_llm/resources/config/models/moonshotai-kimi-k3.yaml index ac52f8a..688cda2 100644 --- a/src/xrouter_llm/resources/config/models/moonshotai-kimi-k3.yaml +++ b/src/xrouter_llm/resources/config/models/moonshotai-kimi-k3.yaml @@ -1,7 +1,7 @@ # One model per file. Load the registry with: --benchmark-profiles config/models -# Benchmarks are PUBLISHED PERCENTAGES (0-100). Served via OpenRouter. Researched 2026-07-18. +# Benchmarks are PUBLISHED PERCENTAGES (0-100). Served via OpenRouter. Researched 2026-07-26. # Moonshot's launch table does not report GPQA Diamond or LiveCodeBench; the -# capability score below is from Artificial Analysis' independent API run. +# capability scores below are from independent AA/Vals API runs. model_id: moonshotai/kimi-k3 provider: moonshotai aliases: ["moonshotai/kimi-k3", "kimi-k3"] @@ -12,11 +12,11 @@ source_urls: - https://platform.kimi.ai/ - https://openrouter.ai/moonshotai/kimi-k3 - https://artificialanalysis.ai/models/kimi-k3 + - https://www.vals.ai/models/kimi_kimi-k3 context_length: 1048576 parameters_b: 2800 input_cost_per_1k: 0.003 # $3.00 / 1M cache miss; $0.30 / 1M cache hit output_cost_per_1k: 0.015 # $15.00 / 1M benchmarks: gpqa_diamond: 93.5 # Artificial Analysis (93.535%, max reasoning) -# livecodebench: pending (AA reports no result as of 2026-07-18). Do not -# substitute DeepSWE, FrontierSWE, Program Bench, or other agentic coding scores. + livecodebench: 87.2 # Vals LCB v6 (87.185%); AA has not run LCB diff --git a/tests/test_yaml_profiles.py b/tests/test_yaml_profiles.py index b279dc8..e0dd254 100644 --- a/tests/test_yaml_profiles.py +++ b/tests/test_yaml_profiles.py @@ -49,7 +49,7 @@ def test_shipped_models_registry_loads() -> None: from xrouter_llm.paths import default_models_dir catalog = load_benchmark_profiles(default_models_dir()) - assert len(catalog) == 14 + assert len(catalog) == 15 # model_id is the canonical OpenRouter slug; the bare id stays as an alias. opus = catalog.get("anthropic/claude-opus-4.8") assert opus.provider == "anthropic" @@ -60,8 +60,13 @@ def test_shipped_models_registry_loads() -> None: assert catalog.get("z-ai/glm-5.2").benchmarks["livecodebench"] == 69.5 assert "livecodebench" not in catalog.get("deepseek/deepseek-v4-flash").benchmarks assert catalog.get("openai/gpt-5.5").provider == "openai" - # 2026-07 additions: latest gemini flash/pro/flash-lite, sonnet 5, Kimi models + # 2026-07 additions: latest Gemini, Claude, and Kimi models assert catalog.get("claude-sonnet-5").model_id == "anthropic/claude-sonnet-5" + opus_5 = catalog.get("claude-opus-5") + assert opus_5.model_id == "anthropic/claude-opus-5" + assert opus_5.source_quality == "third_party" + assert opus_5.benchmarks["gpqa_diamond"] == 93.2 + assert opus_5.benchmarks["livecodebench"] == 89.0 assert catalog.get("google/gemini-3.5-flash").benchmarks["gpqa_diamond"] == 92.2 assert catalog.get("google/gemini-3.5-flash").benchmarks["livecodebench"] == 87.6 assert catalog.get("google/gemini-3.1-pro-preview").provider == "google" @@ -72,7 +77,7 @@ def test_shipped_models_registry_loads() -> None: assert kimi_k3.model_id == "moonshotai/kimi-k3" assert kimi_k3.parameters_b == 2800 assert kimi_k3.benchmarks["gpqa_diamond"] == 93.5 - assert "livecodebench" not in kimi_k3.benchmarks + assert kimi_k3.benchmarks["livecodebench"] == 87.2 # superseded models are removed from the registry removed = {"google/gemini-2.5-flash-lite", "anthropic/claude-sonnet-4.6"} assert removed.isdisjoint({p.model_id for p in catalog.profiles()})