diff --git a/src/whichllm/models/benchmark_sources/aa_index.py b/src/whichllm/models/benchmark_sources/aa_index.py index 76b5878..77fcdea 100644 --- a/src/whichllm/models/benchmark_sources/aa_index.py +++ b/src/whichllm/models/benchmark_sources/aa_index.py @@ -87,106 +87,121 @@ ], } -# AA Intelligence Index typically ranges 12..56 for open-weights models -# (2026-Q2 frontier tops out at 54; 8B-class lands ~30). The 12..56 window -# is anchored by a two-point fit: top open frontier (Kimi K2.6 = 54) → 95 -# normalized, and 8B-class (Qwen3-8B = 30) → 40 normalized. This keeps a -# strong 8B model competitive with frozen-OLLB 7B scores while still leaving -# clear headroom for frontier-tier models. -_AA_INDEX_MIN = 12.5 -_AA_INDEX_MAX = 56.2 +# Bounds used to normalize raw AA index values onto the 0-100 scale the rest +# of the ranking system uses. AA reworked their Intelligence Index in +# 2026-Q2 and the open-weights distribution compressed sharply (top open model +# ~44, 8B-class ~7, weakest mapped repos ~3). The window is anchored by the +# same two-point fit as before: top open frontier (DeepSeek-V4-Pro = 44.3) → 95 +# normalized, and 8B-class (Qwen3-8B = 7.4) → 40 normalized. This keeps a strong +# 8B model competitive with frozen-OLLB 7B scores while leaving headroom for +# frontier-tier models. On the reworked scale that fit puts the floor below +# zero; that is fine, live AA values are always positive and clamp at 0. +_AA_INDEX_MIN = -19.4 +_AA_INDEX_MAX = 47.6 AA_LEADERBOARD_URL = "https://artificialanalysis.ai/leaderboards/models" -# Snapshot of the AA Intelligence Index (open-weights only) verified on -# 2026-05-14 from artificialanalysis.ai. This is used as a fallback when the -# live HTML scrape returns no results (e.g. because Next.js __NEXT_DATA__ -# format changes again). All entries map directly to HuggingFace model IDs -# and are normalized through _normalize_aa_index(). -AA_INDEX_FALLBACK_2026_05_14: dict[str, float] = { +# Snapshot of the AA Intelligence Index (open-weights only), refreshed on +# 2026-06-29 from artificialanalysis.ai against their reworked index. Used as a +# fallback when the live HTML scrape returns no results (e.g. because the +# Next.js payload format changes again). Entries are raw AA index values, +# normalized through _normalize_aa_index() in get_aa_curated_fallback(). +# +# Entries marked "live" are real values from the reworked index. Entries marked +# "peer" are models AA does not track; their raw value is set so that, under the +# current bounds, they reproduce their previous normalized score (hand-estimated +# LB-equivalents). On the reworked scale several peers fall below the index +# floor and read as negative raw; that is an artifact of reusing the AA-index +# field for non-AA estimates. _normalize_aa_index() clamps them at 0, so the +# normalized output stays correct. (We could instead store this snapshot as +# already-normalized 0-100 values, which would drop the negatives and decouple +# the fallback from future bound retunes, but that is a larger change than this +# issue calls for and is left for a follow-up if you want it.) +AA_INDEX_FALLBACK_2026_06_29: dict[str, float] = { # Frontier MoE / very large - "moonshotai/Kimi-K2-Thinking": 50.0, - "moonshotai/Kimi-K2-Instruct": 47.0, - "XiaomiMiMo/MiMo-V2.5-Pro": 54.0, - "XiaomiMiMo/MiMo-V2.5": 49.0, - "deepseek-ai/DeepSeek-V4-Pro": 52.0, - "deepseek-ai/DeepSeek-V4-Flash": 47.0, - "deepseek-ai/DeepSeek-V3.2": 45.0, - "deepseek-ai/DeepSeek-V3.2-Exp": 44.0, - "deepseek-ai/DeepSeek-V3.1": 42.0, - "deepseek-ai/DeepSeek-V3-0324": 40.0, - "deepseek-ai/DeepSeek-V3": 38.0, - "deepseek-ai/DeepSeek-R1-0528": 48.0, - "deepseek-ai/DeepSeek-R1": 43.0, - "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": 32.0, - "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": 26.0, - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B": 20.0, - "Qwen/QwQ-32B": 36.0, - "Qwen/Qwen3-4B-Thinking-2507": 22.0, - "zai-org/GLM-5.1": 51.0, - "zai-org/GLM-5": 50.0, - "zai-org/GLM-5-FP8": 50.0, - "zai-org/GLM-5.1-FP8": 51.0, - "zai-org/GLM-4.7-Flash": 42.0, - "zai-org/GLM-4.6": 40.0, - "zai-org/GLM-4.5": 38.0, - "zai-org/GLM-4.5-Air": 36.0, + "moonshotai/Kimi-K2-Thinking": 32.7, # live + "moonshotai/Kimi-K2-Instruct": 19.4, # live + "XiaomiMiMo/MiMo-V2.5-Pro": 42.2, # live + "XiaomiMiMo/MiMo-V2.5": 40.1, # live + "deepseek-ai/DeepSeek-V4-Pro": 44.3, # live + "deepseek-ai/DeepSeek-V4-Flash": 40.3, # live + "deepseek-ai/DeepSeek-V3.2": 33.4, # live + "deepseek-ai/DeepSeek-V3.2-Exp": 25.4, # live + "deepseek-ai/DeepSeek-V3.1": 21.0, # live + "deepseek-ai/DeepSeek-V3-0324": 10.4, # live + "deepseek-ai/DeepSeek-V3": 10.4, # live + "deepseek-ai/DeepSeek-R1-0528": 20.1, # live + "deepseek-ai/DeepSeek-R1": 12.6, # live + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": 10.5, # peer + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": 1.3, # peer + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B": -7.9, # peer + "Qwen/QwQ-32B": 13.4, # live + "Qwen/Qwen3-4B-Thinking-2507": -4.8, # peer + "zai-org/GLM-5.1": 40.2, # live + "zai-org/GLM-5": 39.5, # live + "zai-org/GLM-5-FP8": 39.5, # live + "zai-org/GLM-5.1-FP8": 40.2, # live + "zai-org/GLM-4.7-Flash": 22.9, # live + "zai-org/GLM-4.6": 25.1, # live + "zai-org/GLM-4.5": 19.5, # live + "zai-org/GLM-4.5-Air": 19.5, # live # Qwen family - "Qwen/Qwen3.6-27B": 46.0, - "Qwen/Qwen3.5-397B-A17B": 45.0, - "Qwen/Qwen3-Next-80B-A3B-Instruct": 42.0, - "Qwen/Qwen3-235B-A22B": 41.0, - "Qwen/Qwen3-Coder-30B-A3B-Instruct": 38.0, - "Qwen/Qwen3-32B": 37.0, - "Qwen/Qwen3-14B": 33.0, - "Qwen/Qwen3-8B": 30.0, - "Qwen/Qwen3-4B-Instruct-2507": 28.0, - "Qwen/Qwen3-4B": 26.0, - "Qwen/Qwen3-1.7B": 20.0, - "Qwen/Qwen3-0.6B": 16.0, + "Qwen/Qwen3.6-27B": 32.0, # peer + "Qwen/Qwen3.5-397B-A17B": 33.7, # live + "Qwen/Qwen3-Next-80B-A3B-Instruct": 19.8, # live + "Qwen/Qwen3-235B-A22B": 13.4, # live + "Qwen/Qwen3-Coder-30B-A3B-Instruct": 19.7, # peer + "Qwen/Qwen3-32B": 11.5, # live + "Qwen/Qwen3-14B": 10.1, # live + "Qwen/Qwen3-8B": 7.4, # live + "Qwen/Qwen3-4B-Instruct-2507": 4.4, # peer + "Qwen/Qwen3-4B": 1.3, # peer + "Qwen/Qwen3-1.7B": -7.9, # peer + "Qwen/Qwen3-0.6B": -14.0, # peer # 8B-class peers (no AA tracking but realistic LB-equivalents) - "meta-llama/Llama-3.1-8B-Instruct": 22.0, - "meta-llama/Meta-Llama-3-8B-Instruct": 20.0, - "google/gemma-2-9b-it": 23.0, - "microsoft/Phi-4-mini-instruct": 24.0, - "mistralai/Mistral-7B-Instruct-v0.3": 20.0, - "Qwen/Qwen2.5-7B-Instruct": 22.0, - "Qwen/Qwen2.5-14B-Instruct": 26.0, - "Qwen/Qwen2.5-32B-Instruct": 30.0, - "Qwen/Qwen3-30B-A3B": 32.0, + "meta-llama/Llama-3.1-8B-Instruct": -4.8, # peer + "meta-llama/Meta-Llama-3-8B-Instruct": -7.9, # peer + "google/gemma-2-9b-it": -3.3, # peer + "microsoft/Phi-4-mini-instruct": -1.8, # peer + "mistralai/Mistral-7B-Instruct-v0.3": -7.9, # peer + "Qwen/Qwen2.5-7B-Instruct": -4.8, # peer + "Qwen/Qwen2.5-14B-Instruct": 1.3, # peer + "Qwen/Qwen2.5-32B-Instruct": 7.4, # peer + "Qwen/Qwen3-30B-A3B": 10.5, # peer # Other major open releases - "openai/gpt-oss-120b": 41.0, - "openai/gpt-oss-20b": 34.0, - "meta-llama/Llama-4-Maverick-17B-128E-Instruct": 38.0, - "meta-llama/Llama-4-Scout-17B-16E-Instruct": 34.0, - "meta-llama/Llama-3.3-70B-Instruct": 33.0, - "google/gemma-4-31b-it": 38.0, - "google/gemma-4-26b-a4b-it": 36.0, - "google/gemma-3-27b-it": 33.0, - "google/gemma-3-12b-it": 30.0, - "microsoft/phi-4": 33.0, - "mistralai/Mistral-Large-Instruct-2411": 35.0, - "mistralai/Mistral-Small-3.2-24B-Instruct-2506": 32.0, - "mistralai/Mistral-Small-3.1-24B-Instruct-2503": 30.0, - "mistralai/Devstral-Small-2505": 33.0, - "MiniMaxAI/MiniMax-M2.5": 40.0, - "stepfun-ai/Step-3.5-Flash": 38.0, - "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": 36.0, - "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": 33.0, + "openai/gpt-oss-120b": 23.8, # live + "openai/gpt-oss-20b": 14.9, # live + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": 14.3, # live + "meta-llama/Llama-4-Scout-17B-16E-Instruct": 10.0, # live + "meta-llama/Llama-3.3-70B-Instruct": 12.0, # peer + "google/gemma-4-31b-it": 29.4, # live + "google/gemma-4-26b-a4b-it": 25.7, # live + "google/gemma-3-27b-it": 12.0, # peer + "google/gemma-3-12b-it": 7.4, # peer + "microsoft/phi-4": 4.9, # live + "mistralai/Mistral-Large-Instruct-2411": 9.1, # live + "mistralai/Mistral-Small-3.2-24B-Instruct-2506": 10.5, # peer + "mistralai/Mistral-Small-3.1-24B-Instruct-2503": 7.4, # peer + "mistralai/Devstral-Small-2505": 11.8, # live + "MiniMaxAI/MiniMax-M2.5": 33.7, # live + "stepfun-ai/Step-3.5-Flash": 19.7, # peer + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": 16.6, # peer + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": 12.0, # peer # Correct IDs for OLMo / Granite / Codestral families (the earlier # forecast IDs like "OLMo-3-32B-Instruct" or "granite-4.1-30b-instruct" # never shipped publicly under those names). - "allenai/Olmo-3-7B-Instruct": 22.0, - "allenai/Olmo-3-1025-7B": 22.0, - "ibm-granite/granite-4.0-h-small": 30.0, - "ibm-granite/granite-4.0-h-tiny": 22.0, - "ibm-granite/granite-3.3-8b-instruct": 23.0, - "ibm-granite/granite-3.3-2b-instruct": 17.0, - "mistralai/Codestral-22B-v0.1": 28.0, + "allenai/Olmo-3-7B-Instruct": -4.8, # peer + "allenai/Olmo-3-1025-7B": -4.8, # peer + "ibm-granite/granite-4.0-h-small": 7.4, # peer + "ibm-granite/granite-4.0-h-tiny": -4.8, # peer + "ibm-granite/granite-3.3-8b-instruct": -3.3, # peer + "ibm-granite/granite-3.3-2b-instruct": -12.5, # peer + "mistralai/Codestral-22B-v0.1": 4.4, # peer } def _normalize_aa_index(index: float) -> float: + """Normalize a raw AA index value onto the 0-100 scale.""" if not isinstance(index, (int, float)): return 0.0 span = _AA_INDEX_MAX - _AA_INDEX_MIN @@ -358,13 +373,13 @@ async def fetch_aa_index_scores(client: httpx.AsyncClient) -> dict[str, float]: def get_aa_curated_fallback() -> dict[str, float]: - """Return the 2026-05-14 curated snapshot, normalized to the 0-100 scale. + """Return the 2026-06-29 curated snapshot, normalized to the 0-100 scale. Used whenever the live HTML scrape cannot extract data — for example when artificialanalysis.ai changes its Next.js payload shape. """ result: dict[str, float] = {} - for hf_id, raw in AA_INDEX_FALLBACK_2026_05_14.items(): + for hf_id, raw in AA_INDEX_FALLBACK_2026_06_29.items(): normalized = _normalize_aa_index(raw) if normalized > 0: result[hf_id] = normalized diff --git a/tests/test_aa_index.py b/tests/test_aa_index.py index 39abaf7..5a62ebf 100644 --- a/tests/test_aa_index.py +++ b/tests/test_aa_index.py @@ -19,6 +19,7 @@ _canonical_name, _decode_rsc_blob, _extract_aa_pairs_from_html, + _normalize_aa_index, fetch_aa_index_scores, get_aa_curated_fallback, ) @@ -112,3 +113,25 @@ def test_fetch_maps_canonical_names_and_merges_over_fallback(): def test_fetch_raises_when_no_records_found(): with pytest.raises(ExtractionFailed): _run_fetch("nothing to see") + + +def test_live_normalization_anchors_on_reworked_scale(): + # Retuned bounds keep the calibration: the top mapped open model lands ~95 + # and an 8B-class model lands ~40, on AA's reworked (compressed) raw scale. + assert _normalize_aa_index(44.3) == pytest.approx(95, abs=0.5) # top open model + assert _normalize_aa_index(7.4) == pytest.approx(40, abs=0.5) # 8B-class + # Values below the floor clamp at 0 (raw is always positive in practice). + assert _normalize_aa_index(-100.0) == 0.0 + assert _normalize_aa_index(60.0) == 100.0 + + +def test_curated_fallback_normalizes_refreshed_snapshot(): + # The snapshot holds refreshed raw AA values; get_aa_curated_fallback maps + # them onto the 0-100 scale with the retuned bounds. + fb = get_aa_curated_fallback() + assert fb["deepseek-ai/DeepSeek-V4-Pro"] == pytest.approx(95, abs=0.5) + assert fb["Qwen/Qwen3-8B"] == 40.0 + assert fb["XiaomiMiMo/MiMo-V2.5-Pro"] == pytest.approx(92, abs=0.5) + # Reworked scale ranks the strong 8B above the small/old peers. + assert fb["Qwen/Qwen3-8B"] > fb["Qwen/Qwen3-0.6B"] + assert all(0.0 < v <= 100.0 for v in fb.values()) diff --git a/tests/test_r3_regressions.py b/tests/test_r3_regressions.py index ccccd7a..472ddf0 100644 --- a/tests/test_r3_regressions.py +++ b/tests/test_r3_regressions.py @@ -21,7 +21,7 @@ from whichllm.hardware.gpu_simulator import create_synthetic_gpu from whichllm.hardware.types import GPUInfo, HardwareInfo from whichllm.models.benchmark import _params_compatible -from whichllm.models.benchmark_sources.aa_index import AA_INDEX_FALLBACK_2026_05_14 +from whichllm.models.benchmark_sources.aa_index import AA_INDEX_FALLBACK_2026_06_29 from whichllm.models.benchmark_sources.livebench import LIVEBENCH_RAW_DATA from whichllm.models.benchmark_sources.vision import VISION_FALLBACK_2026_05 from whichllm.models.grouper import group_models @@ -269,7 +269,7 @@ class TestReasoningSurface: def test_qwq32b_has_curated_benchmarks(self): assert "Qwen/QwQ-32B" in LIVEBENCH_RAW_DATA - assert "Qwen/QwQ-32B" in AA_INDEX_FALLBACK_2026_05_14 + assert "Qwen/QwQ-32B" in AA_INDEX_FALLBACK_2026_06_29 def test_r1_distill_family_has_curated_benchmarks(self): for mid in ( @@ -278,7 +278,7 @@ def test_r1_distill_family_has_curated_benchmarks(self): "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", ): assert mid in LIVEBENCH_RAW_DATA, f"{mid} missing in LB" - assert mid in AA_INDEX_FALLBACK_2026_05_14, f"{mid} missing in AA" + assert mid in AA_INDEX_FALLBACK_2026_06_29, f"{mid} missing in AA" def test_qwq32b_surfaces_with_curated_score(self): qwq = ModelInfo(