diff --git a/.github/workflows/evaluation.yml b/.github/workflows/evaluation.yml index c3b135ff57..9de4fe2547 100644 --- a/.github/workflows/evaluation.yml +++ b/.github/workflows/evaluation.yml @@ -1829,21 +1829,40 @@ jobs: continue } - # Parse once, then group by the executor model recorded inside each - # results.json. For a one-model profile (e.g. opus48) all files share - # one model, so this yields exactly one group. + # Parse once, then group by the (executor model, judge model) pair + # recorded inside each results.json. Grouping by model ALONE collapsed + # every judge of one executor into a single entry stamped with the first + # file's judgeModel β€” so a dual-judge run mislabelled the second judge and + # concatenated both judges' verdicts under one label. Keying on model+judge + # keeps each judge's verdicts and its correct judgeModel, which is exactly + # the dimension the Skill Value view keys on. For a single-judge profile + # all files share one judge, so this still yields one group per model. + # + # Scope note: only the PRIMARY judge's results.json (vally-results-*) feed + # this benchmark data. The scheduled dual-judge cadence re-scores the same + # executor trajectories and uploads under vally-crossjudge-* (see the + # "Download second-judge" step), which only feeds judge-comparison.json β€” + # never this loop. So Skill Value and Quality/Efficiency both reflect the + # primary judge, and each executor model here carries a single judge. + # (The model+judge key and -SkillValueOnly dedup below stay correct even + # if a future change ever routes two primary judges into this set.) $parsed = @($resultsFiles | Sort-Object | ForEach-Object { [pscustomobject]@{ File = $_; Json = (Get-Content $_ -Raw | ConvertFrom-Json) } }) - $modelGroups = @($parsed | Group-Object -Property { "$($_.Json.model)" }) + $modelGroups = @($parsed | Group-Object -Property { "$($_.Json.model)|$($_.Json.judgeModel)" }) $existingFile = "/tmp/eval-data/data/$plugin.json" + # Quality/Efficiency views key on executor model alone, so emit them + # once per model. Track which executor models already emitted them; the + # second+ judge of one model runs SkillValue-only to avoid duplicate + # same-model Quality/Efficiency points that would inflate those windows. + $qeEmittedModels = @{} foreach ($mg in $modelGroups) { $group = @($mg.Group) - # Merge this model's per-skill/per-shard verdicts into a single - # synthetic results.json. Schema: { model, verdicts[], ... }. Concat - # the verdicts arrays and keep top-level scalars from the first file - # of the group (all share the same model/judgeModel). + # Merge this (model, judge) pair's per-skill/per-shard verdicts into a + # single synthetic results.json. Schema: { model, judgeModel, verdicts[], ... }. + # Concat the verdicts arrays and keep top-level scalars from the first + # file of the group (all now share the same model AND judgeModel). $merged = $null $allVerdicts = [System.Collections.Generic.List[object]]::new() foreach ($item in $group) { @@ -1854,12 +1873,16 @@ jobs: $merged.verdicts = $allVerdicts.ToArray() $safeModel = ("$($merged.model)" -replace '[^A-Za-z0-9._-]', '_') if (-not $safeModel) { $safeModel = 'default' } + # Include the judge in the filename so two judges of one executor write + # to distinct synthetic files instead of overwriting each other. + $safeJudge = ("$($merged.judgeModel)" -replace '[^A-Za-z0-9._-]', '_') + if (-not $safeJudge) { $safeJudge = 'nojudge' } $mergedDir = "all-results/_merged" New-Item -ItemType Directory -Force -Path $mergedDir | Out-Null - $resultsFile = Join-Path $mergedDir "$plugin--$safeModel.results.json" + $resultsFile = Join-Path $mergedDir "$plugin--$safeModel--$safeJudge.results.json" $merged | ConvertTo-Json -Depth 100 | Out-File -FilePath $resultsFile -Encoding utf8 - Write-Host "`n=== Generating benchmark data for: $plugin (model '$($merged.model)', $($allVerdicts.Count) verdicts from $($group.Count) results.json) ===" + Write-Host "`n=== Generating benchmark data for: $plugin (model '$($merged.model)', judge '$($merged.judgeModel)', $($allVerdicts.Count) verdicts from $($group.Count) results.json) ===" $params = @{ ResultsFile = $resultsFile PluginName = $plugin @@ -1869,6 +1892,15 @@ jobs: Source = 'scheduled' SkipTokenUsage = $true } + # Emit Quality/Efficiency once per executor model; SkillValue every + # (model, judge). The first judge of a model writes all three; later + # judges of the same model write SkillValue only. + $execModel = "$($merged.model)" + if ($qeEmittedModels.ContainsKey($execModel)) { + $params.SkillValueOnly = $true + } else { + $qeEmittedModels[$execModel] = $true + } # Accumulate: each model's entry appends to $plugin.json. The first # iteration reads the fetched history; later iterations read the file # the previous iteration just wrote (same path as OutputDir/$plugin.json). @@ -2156,6 +2188,7 @@ jobs: cp ${{ github.workspace }}/eng/dashboard/dashboard.html index.html cp ${{ github.workspace }}/eng/dashboard/dashboard.js dashboard.js cp ${{ github.workspace }}/eng/dashboard/token-usage.js token-usage.js + cp ${{ github.workspace }}/eng/dashboard/skill-value.js skill-value.js # Deploy AGENTVIZ SPA (skip if already present and unchanged) if [ "${{ steps.check-replay.outputs.skip }}" != "true" ]; then diff --git a/eng/dashboard/dashboard.html b/eng/dashboard/dashboard.html index ca4a64a9b4..aef8e457ce 100644 --- a/eng/dashboard/dashboard.html +++ b/eng/dashboard/dashboard.html @@ -222,6 +222,28 @@ @media (max-width: 600px) { .charts-grid { grid-template-columns: 1fr; } } + /* Skill Value view */ + .sv-controls { display: flex; flex-wrap: wrap; gap: 16px; align-items: center; margin-bottom: 16px; } + .sv-controls label { font-size: 13px; color: var(--text-muted); } + .sv-controls select { + background: var(--surface); color: var(--text); border: 1px solid var(--border); + border-radius: 6px; padding: 6px 8px; font-size: 13px; margin-left: 4px; + } + .sv-sub { display: block; color: var(--text-muted); font-size: 11px; font-weight: 400; margin-top: 2px; } + .sv-controls .sv-sub { display: inline; margin: 0; } + .sv-table td { vertical-align: top; } + .sv-value { font-size: 13px; max-width: 360px; } + .sv-insufficient { color: var(--text-muted); font-style: italic; } + .sv-drill { padding: 4px 0; } + .sv-arm { font-size: 13px; padding: 3px 0; font-family: 'SF Mono', SFMono-Regular, Consolas, monospace; } + .sv-arm-label { display: inline-block; min-width: 110px; color: var(--text-muted); font-family: inherit; } + .sv-diluted { opacity: 0.6; } + .sv-dilute-mark { color: var(--text-muted); margin-right: 3px; cursor: help; font-weight: 600; } + .sv-btn { background: var(--surface); color: var(--text); border: 1px solid var(--border); border-radius: 6px; padding: 6px 10px; font-size: 12px; cursor: pointer; } + .sv-btn:hover { border-color: var(--skilled); color: var(--skilled); } + /* Model leaf rows carry the real numbers, so undo the token-table level-2 muting. */ + .sv-table tr.level-2 td { color: var(--text); } + .sv-table tr.sv-detail td { padding-left: 88px; } @@ -235,7 +257,8 @@

πŸ“Š Skills Evaluation Dashboard

Generated by the skills evaluation pipeline Β· Verdict evidence shows the latest retained run per model; 0–10 score averages are triage only, not the pass gate. - + + diff --git a/eng/dashboard/dashboard.js b/eng/dashboard/dashboard.js index 89db44137f..53ba312489 100644 --- a/eng/dashboard/dashboard.js +++ b/eng/dashboard/dashboard.js @@ -27,6 +27,9 @@ plugins = []; } + // skill-value.json is a compact derived index, not a dashboard plugin. + // Components manifests may include it when generated from all JSON data files. + plugins = plugins.filter(plugin => plugin !== 'skill-value'); plugins.sort(); const tabBar = document.getElementById('tab-bar'); @@ -34,16 +37,16 @@ const loadedPlugins = new Map(); // track loaded plugin data // Build tabs and placeholder panels - plugins.forEach((plugin, idx) => { + plugins.forEach((plugin) => { const tab = document.createElement('div'); - tab.className = 'tab' + (idx === 0 ? ' active' : ''); + tab.className = 'tab'; tab.textContent = plugin; tab.dataset.plugin = plugin; tab.addEventListener('click', () => switchTab(plugin)); tabBar.appendChild(tab); const panel = document.createElement('div'); - panel.className = 'tab-content' + (idx === 0 ? ' active' : ''); + panel.className = 'tab-content'; panel.id = `panel-${plugin}`; panel.innerHTML = '

Loading...

'; tabContentContainer.appendChild(panel); @@ -51,20 +54,35 @@ // Add Token Usage tab at the end const tokenTabId = '__token-usage__'; - const noPlugins = plugins.length === 0; const tokenTab = document.createElement('div'); - tokenTab.className = 'tab' + (noPlugins ? ' active' : ''); + tokenTab.className = 'tab'; tokenTab.textContent = 'πŸ”’ Token Usage'; tokenTab.dataset.plugin = tokenTabId; tokenTab.addEventListener('click', () => switchTab(tokenTabId)); tabBar.appendChild(tokenTab); const tokenPanel = document.createElement('div'); - tokenPanel.className = 'tab-content' + (noPlugins ? ' active' : ''); + tokenPanel.className = 'tab-content'; tokenPanel.id = `panel-${tokenTabId}`; tokenPanel.innerHTML = '

Loading…

'; tabContentContainer.appendChild(tokenPanel); + // Skill Value is the default landing tab, placed FIRST in the tab bar so the + // per-skill value story is the first thing a viewer sees. + const skillValueTabId = '__skill-value__'; + const skillValueTab = document.createElement('div'); + skillValueTab.className = 'tab active'; + skillValueTab.textContent = 'πŸ’‘ Skill Value'; + skillValueTab.dataset.plugin = skillValueTabId; + skillValueTab.addEventListener('click', () => switchTab(skillValueTabId)); + tabBar.insertBefore(skillValueTab, tabBar.firstChild); + + const skillValuePanel = document.createElement('div'); + skillValuePanel.className = 'tab-content active'; + skillValuePanel.id = `panel-${skillValueTabId}`; + skillValuePanel.innerHTML = '

Loading…

'; + tabContentContainer.appendChild(skillValuePanel); + async function switchTab(plugin) { tabBar.querySelectorAll('.tab').forEach(t => t.classList.toggle('active', t.dataset.plugin === plugin)); tabContentContainer.querySelectorAll('.tab-content').forEach(p => p.classList.toggle('active', p.id === `panel-${plugin}`)); @@ -72,6 +90,10 @@ if (window.initTokenUsage) window.initTokenUsage(); return; } + if (plugin === skillValueTabId) { + if (window.initSkillValue) window.initSkillValue(); + return; + } if (!loadedPlugins.has(plugin)) { await loadPlugin(plugin); } @@ -1126,8 +1148,16 @@ ]); } - // Load first plugin immediately (skip if no evaluation plugins) - if (plugins.length > 0) { - await loadPlugin(plugins[0]); + // Skill Value is the default active tab, so render it immediately. Plugin tabs + // load lazily on first click; Token Usage self-inits when its tab is shown. + if (window.initSkillValue) { + window.initSkillValue(); + } else if (plugins.length > 0) { + // Defensive fallback: if skill-value.js failed to load, activate the first plugin. + await switchTab(plugins[0]); + } else { + // No plugins either β€” fall back to Token Usage so the page is not stuck on + // the Skill Value panel's permanent "Loading…". + await switchTab(tokenTabId); } })(); diff --git a/eng/dashboard/generate-benchmark-data.ps1 b/eng/dashboard/generate-benchmark-data.ps1 index 507f130bff..f0f293b08b 100644 --- a/eng/dashboard/generate-benchmark-data.ps1 +++ b/eng/dashboard/generate-benchmark-data.ps1 @@ -44,6 +44,12 @@ When set, skips generation of token-usage.json entries. Use this when only benchmark data (.json) is needed, such as in the publish-eval-data job. +.PARAMETER SkillValueOnly + When set, appends only the SkillValue entry and skips the Quality/Efficiency + entries. Use this on the second and later judges of one executor model so a + dual-judge run adds one SkillValue point per (model, judge) but still only one + Quality/Efficiency point per executor model (those views key on model alone). + .PARAMETER RetentionDays Number of days of data to retain. Entries older than this are purged. Required for the Purge parameter set; optional for the Generate parameter set (no default value). @@ -81,6 +87,9 @@ param( [Parameter(ParameterSetName = 'Generate')] [switch]$SkipTokenUsage, + [Parameter(ParameterSetName = 'Generate')] + [switch]$SkillValueOnly, + [Parameter(Mandatory, ParameterSetName = 'Purge')] [switch]$PurgeStaleFiles, @@ -143,11 +152,62 @@ function Get-PluginActivityStatus { return $(if ($Activation.activated -eq $true) { "plugin-activity-observed" } else { "plugin-no-activity-observed" }) } +# Mean of a running sum over a count, or $null when nothing was counted. Keeping +# a real $null (rather than 0) lets the dashboard show "n=0" honestly instead of +# a fake zero average. +function Get-MeanOrNull([double]$sum, [int]$count) { + if ($count -gt 0) { return $sum / $count } + return $null +} + +function Update-SkillValueIndex([string]$Directory) { + if (-not (Test-Path $Directory)) { return } + + $entries = [System.Collections.Generic.List[object]]::new() + $reservedFiles = @( + "components.json", + "token-usage.json", + "judge-comparison.json", + "skill-value.json" + ) + $pluginFiles = Get-ChildItem -Path $Directory -Filter "*.json" -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notin $reservedFiles } | + Sort-Object Name + + foreach ($file in $pluginFiles) { + try { + $data = Get-Content $file.FullName -Raw | ConvertFrom-Json + foreach ($entry in @($data.entries.SkillValue)) { + if ($null -eq $entry) { continue } + $entries.Add([ordered]@{ + plugin = $file.BaseName + commit = $entry.commit + date = $entry.date + model = $entry.model + judgeModel = $entry.judgeModel + skills = $entry.skills + }) + } + } catch { + Write-Warning "Failed to add $($file.Name) to skill-value.json: $_" + } + } + + $index = [ordered]@{ + schemaVersion = 1 + lastUpdate = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + entries = $entries.ToArray() + } + $index | ConvertTo-Json -Depth 10 | + Out-File -FilePath (Join-Path $Directory "skill-value.json") -Encoding utf8 + Write-Host "[OK] Skill Value index generated: $($entries.Count) entries" +} + # --- Purge mode: scan a data directory and remove stale files --- if ($PurgeStaleFiles) { $cutoffMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - ([long]$RetentionDays * 24 * 60 * 60 * 1000) $dataFiles = Get-ChildItem -Path $DataDir -Filter "*.json" -File -ErrorAction SilentlyContinue | - Where-Object { $_.Name -ne "components.json" } + Where-Object { $_.Name -notin @("components.json", "skill-value.json") } foreach ($file in $dataFiles) { try { $data = Get-Content $file.FullName -Raw | ConvertFrom-Json -AsHashtable @@ -159,7 +219,10 @@ if ($PurgeStaleFiles) { $data['entries'] = @($data['entries'] | Where-Object { $_.date -ge $cutoffMs }) if ($data['entries'].Count -gt 0) { $hasRecentEntries = $true } } else { - foreach ($category in $data['entries'].Keys) { + # Snapshot the keys before replacing category arrays. Enumerating + # the live hashtable key collection while assigning values can + # throw "Collection was modified" in PowerShell. + foreach ($category in @($data['entries'].Keys)) { $data['entries'][$category] = @($data['entries'][$category] | Where-Object { $_.date -ge $cutoffMs }) if ($data['entries'][$category].Count -gt 0) { $hasRecentEntries = $true } } @@ -174,6 +237,9 @@ if ($PurgeStaleFiles) { Write-Warning "Failed to process $($file.Name) for purge: $_" } } + # Rebuild after purge so the compact index cannot retain entries removed from + # the source plugin histories. + Update-SkillValueIndex $DataDir exit 0 } @@ -550,6 +616,136 @@ $efficiencyEntry = @{ $qualityKey = "Quality" $efficiencyKey = "Efficiency" +$skillValueKey = "SkillValue" + +# --- Build Skill Value per-skill aggregates for this run --- +# One record per skill: baseline (without-skill) vs treatment (with-skill) arm +# metric means, the treatment activation count, and the aggregate pass matrix. +# Everything here comes straight from the verdict the adapter already emits, so +# no adapter change is needed. The dashboard's Skill Value view keys the trailing +# average by (skill, model, judgeModel) and gates on the sample sizes carried here. +$skillValueSkills = [System.Collections.Generic.List[object]]::new() +foreach ($verdict in $results.verdicts) { + $skillName = $verdict.skillName + + $activationExpected = 0 # scenarios where the skill is expected to fire + $activationFired = 0 # of those, how many actually fired in the treatment arm + $anyTimedOut = $false + + $baseN = 0; $baseTime = 0.0; $baseTok = 0.0; $baseIn = 0.0; $baseOut = 0.0; $baseCR = 0.0; $baseCW = 0.0 + $treatN = 0; $treatTime = 0.0; $treatTok = 0.0; $treatIn = 0.0; $treatOut = 0.0; $treatCR = 0.0; $treatCW = 0.0 + # Transparency counts: how many scenarios carried metrics in each arm on its + # own (before pairing). A large gap vs the paired count means one arm dropped + # scenarios (e.g. treatment timeouts), which would bias an unpaired delta. + $baseAvail = 0; $treatAvail = 0 + + # Aggregate pass matrix (baselinePassed/treatmentPassed per counted trial). + # NOTE: per adapt.mjs these per-arm pass booleans may include LLM-grader + # results, so they are pass TELEMETRY, not an objective/deterministic gate. + $passTotal = 0; $baselineFail = 0; $treatmentFail = 0 + + foreach ($scenario in $verdict.scenarios) { + # Activation is only meaningful where the scenario expects the skill to + # fire; expect_activation:false scenarios are excluded from the ratio. + $expectActivation = $true + if ($scenario.PSObject.Properties['expectActivation'] -and $scenario.expectActivation -eq $false) { + $expectActivation = $false + } + $sa = if ($scenario.PSObject.Properties['skillActivationIsolated']) { $scenario.skillActivationIsolated } else { $scenario.skillActivation } + if ($expectActivation) { + $activationExpected++ + if ($sa -and $sa.activated) { $activationFired++ } + } + + if ($scenario.timedOut -eq $true) { $anyTimedOut = $true } + + # Probe both arms' metrics. Adapter already fills absent token fields with + # 0, so a non-null metrics block with a numeric wallTimeMs is fully numeric. + $skilled = if ($scenario.PSObject.Properties['skilledIsolated']) { $scenario.skilledIsolated } else { $scenario.withSkill } + $tm = $skilled.metrics + $bm = $scenario.baseline.metrics + $treatHas = ($null -ne $tm -and $null -ne $tm.wallTimeMs) + $baseHas = ($null -ne $bm -and $null -ne $bm.wallTimeMs) + if ($treatHas) { $treatAvail++ } + if ($baseHas) { $baseAvail++ } + + # Accumulate a scenario into the delta ONLY when BOTH arms have metrics, so + # baseline and treatment means always describe the same scenario population + # (a true paired delta). baseN == treatN == pairedN by construction. + if ($treatHas -and $baseHas) { + $treatN++ + $treatTime += [double]$tm.wallTimeMs + $treatTok += [double]$tm.tokenEstimate + $treatIn += [double]$tm.inputTokens + $treatOut += [double]$tm.outputTokens + $treatCR += [double]$tm.cacheReadTokens + $treatCW += [double]$tm.cacheWriteTokens + + $baseN++ + $baseTime += [double]$bm.wallTimeMs + $baseTok += [double]$bm.tokenEstimate + $baseIn += [double]$bm.inputTokens + $baseOut += [double]$bm.outputTokens + $baseCR += [double]$bm.cacheReadTokens + $baseCW += [double]$bm.cacheWriteTokens + } + + # Aggregate pass matrix: count only trials that carry BOTH arms' boolean + # pass result and did not error. A skill whose scenarios expose no such + # booleans ends with passTotal=0 β†’ the view shows the not-passed rate as N/A. + foreach ($trial in @($scenario.trials)) { + if ($null -eq $trial -or $trial.errored -eq $true) { continue } + $bp = $trial.baselinePassed + $tp = $trial.treatmentPassed + if ($bp -is [bool] -and $tp -is [bool]) { + $passTotal++ + if (-not $bp) { $baselineFail++ } + if (-not $tp) { $treatmentFail++ } + } + } + } + + $skillValueSkills.Add(@{ + skill = $skillName + activationExpected = $activationExpected + activationFired = $activationFired + timedOut = $anyTimedOut + pairedN = $baseN + baseAvailable = $baseAvail + treatAvailable = $treatAvail + baseline = @{ + n = $baseN + timeMs = (Get-MeanOrNull $baseTime $baseN) + tokens = (Get-MeanOrNull $baseTok $baseN) + tokensIn = (Get-MeanOrNull $baseIn $baseN) + tokensOut = (Get-MeanOrNull $baseOut $baseN) + cacheRead = (Get-MeanOrNull $baseCR $baseN) + cacheWrite = (Get-MeanOrNull $baseCW $baseN) + } + treatment = @{ + n = $treatN + timeMs = (Get-MeanOrNull $treatTime $treatN) + tokens = (Get-MeanOrNull $treatTok $treatN) + tokensIn = (Get-MeanOrNull $treatIn $treatN) + tokensOut = (Get-MeanOrNull $treatOut $treatN) + cacheRead = (Get-MeanOrNull $treatCR $treatN) + cacheWrite = (Get-MeanOrNull $treatCW $treatN) + } + passTotal = $passTotal + baselineFail = $baselineFail + treatmentFail = $treatmentFail + hasPassData = ($passTotal -gt 0) + }) +} + +$skillValueEntry = @{ + commit = $commit + date = $now + model = $model + judgeModel = $results.judgeModel + skills = $skillValueSkills.ToArray() +} + # PR evaluations only collect token-usage data β€” skip benchmark history so PR # runs don't contaminate the nightly benchmark results in .json. @@ -563,6 +759,7 @@ if ($Source -ne 'pr' -and -not $SkipBenchmarkData) { entries = @{ $qualityKey = @() $efficiencyKey = @() + $skillValueKey = @() } } @@ -587,15 +784,24 @@ if ($Source -ne 'pr' -and -not $SkipBenchmarkData) { if (-not $benchmarkData['entries'][$efficiencyKey]) { $benchmarkData['entries'][$efficiencyKey] = @() } + if (-not $benchmarkData['entries'][$skillValueKey]) { + $benchmarkData['entries'][$skillValueKey] = @() + } - $benchmarkData['entries'][$qualityKey] += @($qualityEntry) - $benchmarkData['entries'][$efficiencyKey] += @($efficiencyEntry) + # Quality/Efficiency key on executor model only, so emit them once per model. + # On dual-judge runs the caller passes -SkillValueOnly for the second+ judge to + # avoid duplicate same-model points that would inflate those trailing windows. + if (-not $SkillValueOnly) { + $benchmarkData['entries'][$qualityKey] += @($qualityEntry) + $benchmarkData['entries'][$efficiencyKey] += @($efficiencyEntry) + } + $benchmarkData['entries'][$skillValueKey] += @($skillValueEntry) # Purge entries older than the retention window if ($RetentionDays -gt 0) { $cutoffMs = $now - ([long]$RetentionDays * 24 * 60 * 60 * 1000) - foreach ($key in @($qualityKey, $efficiencyKey)) { + foreach ($key in @($qualityKey, $efficiencyKey, $skillValueKey)) { $before = $benchmarkData['entries'][$key].Count $benchmarkData['entries'][$key] = @($benchmarkData['entries'][$key] | Where-Object { $_.date -ge $cutoffMs diff --git a/eng/dashboard/skill-value.js b/eng/dashboard/skill-value.js new file mode 100644 index 0000000000..83a844a72f --- /dev/null +++ b/eng/dashboard/skill-value.js @@ -0,0 +1,465 @@ +// Skill Value dashboard view +// Loaded by dashboard.html, exposes window.initSkillValue(). +// +// Answers, per skill: "Add this skill and model X uses ~M% fewer tokens and +// ~Y% less time for typical scenarios; without it ~p% of those scenarios do not +// pass their checks." +// Data source: compact `data/skill-value.json`, derived from the +// `entries.SkillValue` arrays in per-plugin dashboard data by +// eng/dashboard/generate-benchmark-data.ps1. Both arms β€” baseline (without the +// skill) and treatment (with the skill) β€” are carried per run. The compact index +// avoids downloading every plugin's large Quality/Efficiency history on startup. +(function () { + let initialized = false; + let initialization = null; + + // ── Tunable thresholds (documented in eng/dashboard skill-value design) ── + // Minimum PAIRED OBSERVATIONS (scenario-runs measured in both arms, summed + // across the trailing window) before a stat earns a number instead of + // "insufficient signal". This is a minimum-observations floor for a stable + // trailing mean, NOT a distinct-stimulus breadth guarantee: e.g. one scenario + // measured across 5 runs also reaches n=5. The distinct scenario breadth per + // run and the run count are shown in the drill-down so the two are not + // conflated. 5 mirrors the harness's MIN_CREDIBLE_STIMULI as a familiar floor. + const MIN_SAMPLES = 5; + // The plain-English value sentence is suppressed below this activation fraction: + // if the skill rarely fires, the treatment arm behaves like baseline and the + // delta is diluted rather than real. + const ACTIVATION_MIN = 0.5; + // Trailing window: most-recent N runs per (skill, executor model, judge model). + const TRAILING_RUNS = 20; + // Headline "tokens" = input + output (the adapter's tokenEstimate / totalTokens). + // Cache read/write are billed/served differently and swing with infra caching, + // so they are shown only in the drill-down, never in the headline delta. + + function escapeHtml(str) { + if (str == null) return ''; + const div = document.createElement('div'); + div.textContent = String(str); + return div.innerHTML; + } + + function fmtInt(n) { return Math.round(n).toLocaleString(); } + function fmtK(n) { + if (n == null) return '–'; + if (Math.abs(n) >= 1_000_000) return (n / 1_000_000).toFixed(1) + 'M'; + if (Math.abs(n) >= 1_000) return (n / 1_000).toFixed(1) + 'k'; + return Math.round(n).toString(); + } + function fmtPct(frac) { return (frac * 100).toFixed(0) + '%'; } + function fmtSecs(ms) { return (ms / 1000).toFixed(1) + 's'; } + + // n-weighted running accumulator for one arm's metrics across runs. + function newArm() { + return { n: 0, timeMs: 0, tokens: 0, tokensIn: 0, tokensOut: 0, cacheRead: 0, cacheWrite: 0 }; + } + function addArm(acc, arm) { + const w = arm && arm.n ? arm.n : 0; + if (w <= 0) return; + acc.n += w; + acc.timeMs += (arm.timeMs || 0) * w; + acc.tokens += (arm.tokens || 0) * w; + acc.tokensIn += (arm.tokensIn || 0) * w; + acc.tokensOut += (arm.tokensOut || 0) * w; + acc.cacheRead += (arm.cacheRead || 0) * w; + acc.cacheWrite += (arm.cacheWrite || 0) * w; + } + function meanArm(acc) { + if (acc.n <= 0) return null; + return { + n: acc.n, + timeMs: acc.timeMs / acc.n, + tokens: acc.tokens / acc.n, + tokensIn: acc.tokensIn / acc.n, + tokensOut: acc.tokensOut / acc.n, + cacheRead: acc.cacheRead / acc.n, + cacheWrite: acc.cacheWrite / acc.n, + }; + } + + // ── Public entry point ────────────────────────────────────────────── + window.initSkillValue = async function () { + if (initialized) return; + if (initialization) return initialization; + + const container = document.getElementById('skill-value-content'); + initialization = (async () => { + try { + const res = await fetch('data/skill-value.json'); + if (!res.ok) throw new Error(res.statusText); + const data = await res.json(); + const entries = Array.isArray(data.entries) ? data.entries : []; + + if (entries.length === 0) { + container.innerHTML = '

No skill-value data available yet. It is produced by scheduled evaluation runs.

'; + } else { + render(container, entries); + } + initialized = true; + } catch { + container.innerHTML = '

Unable to load skill-value data. Select the Skill Value tab to retry.

'; + } finally { + initialization = null; + } + })(); + + return initialization; + }; + + // Aggregate entries into per-(skill, model, judgeModel) rows over the trailing + // window. Never blends different model strings β€” model version is part of the key. + function aggregate(entries) { + // Group runs by key. + const groups = new Map(); + for (const e of entries) { + const model = e.model || 'unknown'; + const judge = e.judgeModel || 'unknown'; + for (const s of (e.skills || [])) { + // Key includes plugin: two plugins may share a skill name, and the view is + // grouped Plugin -> Skill -> Model, so their histories must never blend. + const key = `${e.plugin}\u0000${s.skill}\u0000${model}\u0000${judge}`; + if (!groups.has(key)) groups.set(key, { skill: s.skill, plugin: e.plugin, model, judge, runs: [] }); + groups.get(key).runs.push({ date: e.date || 0, s }); + } + } + + const rows = []; + for (const g of groups.values()) { + // Trailing window: most-recent runs only. + const runs = g.runs.sort((a, b) => b.date - a.date).slice(0, TRAILING_RUNS); + const base = newArm(); + const treat = newArm(); + let actExpected = 0, actFired = 0; + let passTotal = 0, baseFail = 0, treatFail = 0; + let hasPass = false; + let timedOutRuns = 0, baseAvail = 0, treatAvail = 0; + for (const { s } of runs) { + addArm(base, s.baseline); + addArm(treat, s.treatment); + actExpected += s.activationExpected || 0; + actFired += s.activationFired || 0; + passTotal += s.passTotal || 0; + baseFail += s.baselineFail || 0; + treatFail += s.treatmentFail || 0; + if (s.hasPassData) hasPass = true; + if (s.timedOut) timedOutRuns += 1; + baseAvail += s.baseAvailable || 0; + treatAvail += s.treatAvailable || 0; + } + rows.push({ + skill: g.skill, plugin: g.plugin, model: g.model, judge: g.judge, + runCount: runs.length, + baseline: meanArm(base), treatment: meanArm(treat), + activation: actExpected > 0 ? actFired / actExpected : null, + activationExpected: actExpected, activationFired: actFired, + passTotal, baseFail, treatFail, hasPass, + timedOutRuns, baseAvail, treatAvail, + }); + } + rows.sort((a, b) => a.skill.localeCompare(b.skill) || a.model.localeCompare(b.model) || a.judge.localeCompare(b.judge)); + return rows; + } + + // Relative reduction of treatment vs baseline (positive = treatment is smaller). + function reduction(baseVal, treatVal) { + if (baseVal == null || treatVal == null || baseVal === 0) return null; + return (baseVal - treatVal) / baseVal; + } + + // Render a reduction as a signed percent: 'βˆ’N%' = N% less (good), '+N%' = N% + // more, '0%' = no measurable change. Zero β€” exact or rounded β€” never gets a + // sign, so "no change" is not misread as a +0% regression. + function signedPct(r, nullText) { + if (r == null) return nullText; + const pct = Math.round(r * 100); + if (pct === 0) return '0%'; + return `${pct > 0 ? 'βˆ’' : '+'}${Math.abs(pct)}%`; + } + + function deltaCell(base, treat, unitFmt, diluted) { + if (base == null || treat == null) return '–'; + const r = reduction(base, treat); + const abs = treat - base; + const cls = r == null ? 'neutral' : (r > 0 ? 'positive' : (r < 0 ? 'negative' : 'neutral')); + const sign = abs > 0 ? '+' : ''; + // Null reduction (baseline metric is 0) shows an explicit "n/a", matching the + // rollup cells, so the percent is never a blank/ambiguous label. + const pctTxt = signedPct(r, 'n/a'); + // When the skill barely fires, the treatment arm is mostly baseline, so this + // delta understates the on-activation effect. Mark it so it is not misread. + const dil = diluted ? ' sv-diluted' : ''; + const mark = diluted ? 'β‰ˆ' : ''; + return `${mark}${pctTxt}` + + `${escapeHtml(unitFmt(base))} β†’ ${escapeHtml(unitFmt(treat))} (${sign}${escapeHtml(unitFmt(abs))})`; + } + + function gated(n) { return n != null && n >= MIN_SAMPLES; } + + function valueSentence(row) { + const pairedN = Math.min(row.baseline ? row.baseline.n : 0, row.treatment ? row.treatment.n : 0); + if (!gated(pairedN)) return { text: `Insufficient signal (n=${pairedN} paired, need β‰₯${MIN_SAMPLES})`, cls: 'sv-insufficient' }; + // Activation contamination guard. If the skill is not confirmed to fire in + // most expected scenarios, the treatment arm behaves like baseline and any + // delta is diluted β€” so suppress the confident value claim. A null activation + // means NO scenario declared expect_activation, so we cannot confirm the skill + // fired at all; treat that as unverified rather than asserting value. + if (row.activation == null) { + return { text: 'Insufficient signal β€” no expected-activation scenarios, so the skill firing is unverified', cls: 'sv-insufficient' }; + } + if (row.activation < ACTIVATION_MIN) { + return { text: `Insufficient signal β€” skill fired in only ${fmtPct(row.activation)} of expected scenarios (delta is diluted)`, cls: 'sv-insufficient' }; + } + const tokR = reduction(row.baseline.tokens, row.treatment.tokens); + const timeR = reduction(row.baseline.timeMs, row.treatment.timeMs); + // A null reduction means the baseline value was zero/absent β€” do not coerce it + // into a "~0% fewer" claim. Fall back to reporting the metric as unavailable. + if (tokR == null || timeR == null) { + return { text: `Add ${escapeHtml(row.skill)}: token/time delta unavailable for ${escapeHtml(row.model)} (a baseline metric was zero or missing).`, cls: 'sv-insufficient' }; + } + const tokWord = tokR >= 0 ? 'fewer' : 'more'; + const timeWord = timeR >= 0 ? 'less' : 'more'; + let s = `Add ${escapeHtml(row.skill)} and ${escapeHtml(row.model)} uses ~${Math.abs(tokR * 100).toFixed(0)}% ${tokWord} tokens and ~${Math.abs(timeR * 100).toFixed(0)}% ${timeWord} time for typical scenarios`; + if (row.hasPass && gated(row.passTotal)) { + const baseFailFrac = row.passTotal > 0 ? row.baseFail / row.passTotal : 0; + s += `; without it ~${fmtPct(baseFailFrac)} of its counted trials do not pass their checks.`; + } else if (row.hasPass) { + s += ` (not-passed rate still gathering data: n=${row.passTotal}, need β‰₯${MIN_SAMPLES}).`; + } else { + s += '. (No pass/fail data for a not-passed estimate.)'; + } + return { text: s, cls: 'sv-value' }; + } + + function failureCell(row) { + if (!row.hasPass || row.passTotal === 0) return 'N/A'; + if (!gated(row.passTotal)) return `n=${row.passTotal}`; + const b = row.baseFail / row.passTotal; + const t = row.treatFail / row.passTotal; + const cls = t < b ? 'positive' : (t > b ? 'negative' : 'neutral'); + return `${fmtPct(b)} β†’ ${fmtPct(t)}` + + `n=${row.passTotal}`; + } + + function activationCell(row) { + if (row.activation == null) return '–'; + const cls = row.activation >= ACTIVATION_MIN ? 'positive' : 'negative'; + return `${fmtPct(row.activation)}` + + `${row.activationFired}/${row.activationExpected}`; + } + + function metricCell(base, treat, fmt) { + // Delta cell already shows both arms; this is the paired-n column. + const n = Math.min(base ? base.n : 0, treat ? treat.n : 0); + const cls = gated(n) ? 'neutral' : 'sv-insufficient'; + return `n=${n}`; + } + + function drilldown(row) { + const arm = (a, label) => { + if (!a) return `
${label} no metrics
`; + return `
${label} ` + + `time ${fmtSecs(a.timeMs)} Β· tokens ${fmtK(a.tokens)} ` + + `(in ${fmtK(a.tokensIn)} / out ${fmtK(a.tokensOut)}) Β· ` + + `cache read ${fmtK(a.cacheRead)} / write ${fmtK(a.cacheWrite)} Β· n=${a.n}
`; + }; + const pairedN = Math.min(row.baseline ? row.baseline.n : 0, row.treatment ? row.treatment.n : 0); + // Surface how many scenarios each arm measured on its own vs. the paired set, + // and any timed-out runs β€” both indicate selective missingness that could bias + // an unpaired reading (e.g. treatment timeouts dropping the slowest cases). + let notes = `
Paired observations n=${pairedN}` + + ` · measured alone: baseline ${row.baseAvail}, treatment ${row.treatAvail}`; + if (row.timedOutRuns > 0) notes += ` · ⚠ ${row.timedOutRuns} run(s) had a timed-out scenario`; + notes += '
'; + return `
` + + `
${escapeHtml(row.plugin)} Β· executor ${escapeHtml(row.model)} Β· judge ${escapeHtml(row.judge)} Β· ${row.runCount} run(s) in window
` + + arm(row.baseline, 'Without skill') + arm(row.treatment, 'With skill') + + notes + + `
`; + } + + // A (skill, executor, judge) leaf reached a confident value claim. + function isConfirmed(row) { return valueSentence(row).cls === 'sv-value'; } + + function pctSigned(r) { return signedPct(r, 'n/a'); } + + // Rollup for a group that resolves to a SINGLE model leaf (e.g. a skill with one + // model, or any group once the viewer filters to one executor/judge). With one + // model there is nothing to blend, so show that model's ACTUAL deltas instead of + // a bare count β€” this is the header working under a single-model filter. + function singleModelRollup(row) { + const tokR = reduction(row.baseline ? row.baseline.tokens : null, row.treatment ? row.treatment.tokens : null); + const timeR = reduction(row.baseline ? row.baseline.timeMs : null, row.treatment ? row.treatment.timeMs : null); + const diluted = row.activation != null && row.activation < ACTIVATION_MIN; + const mark = diluted ? 'β‰ˆ' : ''; + const tag = isConfirmed(row) + ? 'value confirmed' + : 'gathering data'; + return `${escapeHtml(row.model)}: ${mark}${pctSigned(tokR)} tokens Β· ${pctSigned(timeR)} time Β· ${tag}`; + } + + // Rollup for a multi-model group. Avoids the ambiguous "0/m" (which reads like a + // failure): it states how many models show a measured value AND how many are + // still gathering data, so an all-insufficient group reads as "awaiting data", + // never as "the skill failed". + function countRollup(rows) { + const conf = rows.filter(isConfirmed).length; + const insuff = rows.length - conf; + let s = `${conf} of ${rows.length} model/judge result(s) show measured value`; + if (insuff > 0) s += ` Β· ${insuff} still gathering data (need β‰₯${MIN_SAMPLES} paired obs)`; + return s; + } + + function render(container, entries) { + const allRows = aggregate(entries); + const models = [...new Set(allRows.map(r => r.model))].sort(); + const judges = [...new Set(allRows.map(r => r.judge))].sort(); + + container.innerHTML = + `
` + + `` + + `` + + `` + + `` + + `Grouped Plugin β†’ Skill β†’ Model. Trailing average over the last ${TRAILING_RUNS} runs per (skill, executor, judge). Gate β‰₯${MIN_SAMPLES} paired observations. Tokens = input+output. β‰ˆ marks a diluted (low-activation) delta. Deltas are never blended across models.` + + `
` + + `
`; + + const modelSel = container.querySelector('#sv-model'); + const judgeSel = container.querySelector('#sv-judge'); + const wrap = container.querySelector('#sv-table-wrap'); + + // A Plugin/Skill group-header row. Metric columns are intentionally blank: + // token/time deltas are RELATIVE to each model's own baseline, so they cannot + // be averaged across different executor models without misleading β€” the same + // "never blend model versions" rule the leaf rows follow. The header instead + // carries a non-numeric rollup (child counts + how many model rows reached a + // confirmed value). + function groupRow(level, toggleId, parentId, label, rollup) { + const childCls = parentId ? ` child-of-${parentId}` : ''; + const style = parentId ? ' style="display:none"' : ''; + return `` + + `β–Ά${label}` + + (rollup ? ` ${rollup}` : '') + + ``; + } + + function collapseChildren(parentId) { + wrap.querySelectorAll('.child-of-' + parentId).forEach(c => { + c.style.display = 'none'; + if (c.dataset.toggle) { + const icon = wrap.querySelector('#icon-' + c.dataset.toggle); + if (icon) icon.classList.remove('expanded'); + collapseChildren(c.dataset.toggle); + } + }); + } + + function openRow(toggleId) { + const icon = wrap.querySelector('#icon-' + toggleId); + if (icon) icon.classList.add('expanded'); + wrap.querySelectorAll('.child-of-' + toggleId).forEach(c => { c.style.display = ''; }); + } + + function setAll(open) { + // Reveal/hide plugin, skill and model rows. Model drill-down rows (sv-detail) + // stay closed on "Expand all" β€” opening every drill-down at once buries the + // table; the viewer opens a specific model to see its arms. + wrap.querySelectorAll('.expandable').forEach(tr => { + const icon = wrap.querySelector('#icon-' + tr.dataset.toggle); + // Model-leaf (level-2) icons keep their collapsed β–Ά because their detail row + // is intentionally left closed. + if (icon) icon.classList.toggle('expanded', open && !tr.classList.contains('level-2')); + }); + wrap.querySelectorAll('[class*="child-of-"]').forEach(c => { + if (open && c.classList.contains('sv-detail')) { c.style.display = 'none'; return; } + c.style.display = open ? '' : 'none'; + }); + } + + function wireTree() { + wrap.querySelectorAll('.expandable').forEach(tr => { + tr.addEventListener('click', () => { + const tid = tr.dataset.toggle; + const icon = wrap.querySelector('#icon-' + tid); + if (icon && icon.classList.contains('expanded')) { + icon.classList.remove('expanded'); + collapseChildren(tid); + } else { + openRow(tid); + } + }); + }); + // Default: reveal plugins and skills (levels 0-1); keep per-model detail closed. + wrap.querySelectorAll('tr.level-0.expandable, tr.level-1.expandable').forEach(tr => openRow(tr.dataset.toggle)); + } + + function draw() { + const fm = modelSel.value, fj = judgeSel.value; + const rows = allRows.filter(r => (!fm || r.model === fm) && (!fj || r.judge === fj)); + if (rows.length === 0) { wrap.innerHTML = '

No skills match the selected filters.

'; return; } + + // Group into Plugin β†’ Skill β†’ model rows. + const byPlugin = new Map(); + for (const r of rows) { + if (!byPlugin.has(r.plugin)) byPlugin.set(r.plugin, new Map()); + const sk = byPlugin.get(r.plugin); + if (!sk.has(r.skill)) sk.set(r.skill, []); + sk.get(r.skill).push(r); + } + + let html = `` + + `` + + ``; + + let uid = 0; + for (const plugin of [...byPlugin.keys()].sort()) { + const skills = byPlugin.get(plugin); + const pid = `svp${uid++}`; + const pRows = []; + for (const arr of skills.values()) { for (const r of arr) pRows.push(r); } + // One model in the whole plugin (e.g. filtered to a single executor+judge): + // show its real delta; otherwise a skill count + measured/gathering breakdown. + const pRollup = pRows.length === 1 + ? singleModelRollup(pRows[0]) + : `${skills.size} skill(s) Β· ${countRollup(pRows)}`; + html += groupRow(0, pid, null, escapeHtml(plugin), pRollup); + + for (const skill of [...skills.keys()].sort()) { + const modelRows = skills.get(skill).slice() + .sort((a, b) => a.model.localeCompare(b.model) || a.judge.localeCompare(b.judge)); + const sid = `svs${uid++}`; + const sRollup = modelRows.length === 1 + ? singleModelRollup(modelRows[0]) + : countRollup(modelRows); + html += groupRow(1, sid, pid, escapeHtml(skill), sRollup); + + for (const row of modelRows) { + const mid = `svm${uid++}`; + const v = valueSentence(row); + const diluted = row.activation != null && row.activation < ACTIVATION_MIN; + const label = `${escapeHtml(row.model)} judge ${escapeHtml(row.judge)}`; + html += `` + + `` + + activationCell(row) + + deltaCell(row.baseline ? row.baseline.tokens : null, row.treatment ? row.treatment.tokens : null, fmtK, diluted) + + metricCell(row.baseline, row.treatment, fmtK) + + deltaCell(row.baseline ? row.baseline.timeMs : null, row.treatment ? row.treatment.timeMs : null, fmtSecs, diluted) + + metricCell(row.baseline, row.treatment, fmtSecs) + + failureCell(row) + + `` + + ``; + } + } + } + html += '
Plugin / Skill / ModelActivationTokens ΔnTime ΔnNot-passed (base→treat)Value
'; + wrap.innerHTML = html; + wireTree(); + } + + container.querySelector('#sv-expand').addEventListener('click', () => setAll(true)); + container.querySelector('#sv-collapse').addEventListener('click', () => setAll(false)); + modelSel.addEventListener('change', draw); + judgeSel.addEventListener('change', draw); + draw(); + } +})(); diff --git a/plugins/dotnet-test/skills/grade-tests/SKILL.md b/plugins/dotnet-test/skills/grade-tests/SKILL.md index 843eeef74d..0f713509b6 100644 --- a/plugins/dotnet-test/skills/grade-tests/SKILL.md +++ b/plugins/dotnet-test/skills/grade-tests/SKILL.md @@ -1,15 +1,12 @@ --- name: grade-tests description: > - Grades a specified set of test methods individually and produces a concise - table mapping each test (fully-qualified name) to a letter grade (A–F), a - score band, and a one-line note β€” designed to be posted as a PR comment. - Use when the caller wants per-test feedback on a curated list of methods - (for example, the new or modified tests in a pull request), not a - suite-wide audit. Polyglot: .NET, Python, TS/JS, Java, Go, Ruby, Rust, - Swift, Kotlin, PowerShell, C++. Input is a list of test methods (or method - bodies / file+line spans); output is a compact markdown table plus a short - summary. DO NOT USE FOR: + Grade specified test methods individually and produce a concise PR-ready + table with each fully qualified test name, an A-F grade, score band, and + one-line note. USE FOR per-test feedback on a curated list such as new or + modified tests in a pull request, not a suite-wide audit. Polyglot: .NET, + Python, TS/JS, Java, Go, Ruby, Rust, Swift, Kotlin, PowerShell, C++. Inputs + may be test methods, method bodies, or file-and-line spans. DO NOT USE FOR: full suite audits (use test-quality-auditor agent or test-anti-patterns), writing new tests (use code-testing-generator agent or writing-mstest-tests), fixing failures, or measuring code coverage.