From af334b367300e0067f11d2887a79b3f1f9f0ad7b Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 26 Feb 2026 19:00:36 -0600 Subject: [PATCH 01/10] Add --keep-sessions flag and rejudge command - Add --keep-sessions flag to preserve agent session data in SQLite DB - Create SessionDatabase service with WAL mode and thread-safe writes - Update AgentRunner to separate config/work dir lifecycle - Track session metadata, RunMetrics, judge and pairwise results - Add rejudge command to re-run judges on saved sessions - Add 8 tests for SessionDatabase (CRUD, concurrency, isolation) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Commands/RejudgeCommand.cs | 226 ++++++++++++++++++ .../src/Commands/ValidateCommand.cs | 82 ++++++- eng/skill-validator/src/Models/Models.cs | 1 + eng/skill-validator/src/Program.cs | 1 + .../src/Services/AgentRunner.cs | 38 ++- .../src/Services/SessionDatabase.cs | 200 ++++++++++++++++ eng/skill-validator/src/SkillValidator.csproj | 1 + .../tests/SessionDatabaseTests.cs | 173 ++++++++++++++ 8 files changed, 702 insertions(+), 20 deletions(-) create mode 100644 eng/skill-validator/src/Commands/RejudgeCommand.cs create mode 100644 eng/skill-validator/src/Services/SessionDatabase.cs create mode 100644 eng/skill-validator/tests/SessionDatabaseTests.cs diff --git a/eng/skill-validator/src/Commands/RejudgeCommand.cs b/eng/skill-validator/src/Commands/RejudgeCommand.cs new file mode 100644 index 0000000000..241c3ce7f4 --- /dev/null +++ b/eng/skill-validator/src/Commands/RejudgeCommand.cs @@ -0,0 +1,226 @@ +using System.CommandLine; +using System.Text.Json; +using SkillValidator.Models; +using SkillValidator.Services; +using SkillValidator.Utilities; + +namespace SkillValidator.Commands; + +public static class RejudgeCommand +{ + public static Command Create() + { + var resultsDirArg = new Argument("results-dir") { Description = "Path to a timestamped results directory containing sessions.db" }; + var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to original model)" }; + var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }; + var judgeTimeoutOpt = new Option("--judge-timeout") { Description = "Judge timeout in seconds", DefaultValueFactory = _ => 300 }; + var verboseOpt = new Option("--verbose") { Description = "Show detailed output" }; + var minImprovementOpt = new Option("--min-improvement") { Description = "Minimum improvement score to pass (0-1)", DefaultValueFactory = _ => 0.1 }; + var requireCompletionOpt = new Option("--require-completion") { Description = "Fail if skill regresses task completion", DefaultValueFactory = _ => true }; + var confidenceLevelOpt = new Option("--confidence-level") { Description = "Confidence level for statistical intervals (0-1)", DefaultValueFactory = _ => 0.95 }; + + var command = new Command("rejudge", "Re-run judges on saved sessions without re-running agents") + { + resultsDirArg, + judgeModelOpt, + judgeModeOpt, + judgeTimeoutOpt, + verboseOpt, + minImprovementOpt, + requireCompletionOpt, + confidenceLevelOpt, + }; + + command.SetAction(async (parseResult, _) => + { + var resultsDir = parseResult.GetValue(resultsDirArg)!; + var judgeModel = parseResult.GetValue(judgeModelOpt); + var verbose = parseResult.GetValue(verboseOpt); + var judgeTimeout = parseResult.GetValue(judgeTimeoutOpt) * 1000; + var minImprovement = parseResult.GetValue(minImprovementOpt); + var requireCompletion = parseResult.GetValue(requireCompletionOpt); + var confidenceLevel = parseResult.GetValue(confidenceLevelOpt); + + var judgeMode = parseResult.GetValue(judgeModeOpt) switch + { + "independent" => JudgeMode.Independent, + "both" => JudgeMode.Both, + _ => JudgeMode.Pairwise, + }; + + return await Run(resultsDir, judgeModel, judgeMode, judgeTimeout, verbose, + minImprovement, requireCompletion, confidenceLevel); + }); + + return command; + } + + public static async Task Run( + string resultsDir, + string? judgeModel, + JudgeMode judgeMode, + int judgeTimeout, + bool verbose, + double minImprovement, + bool requireCompletion, + double confidenceLevel) + { + var dbPath = Path.Combine(resultsDir, "sessions.db"); + if (!File.Exists(dbPath)) + { + Console.Error.WriteLine($"No sessions.db found at {dbPath}"); + Console.Error.WriteLine("Use --keep-sessions during evaluation to enable rejudging."); + return 1; + } + + using var sessionDb = new SessionDatabase(dbPath); + var sessions = sessionDb.GetCompletedSessions(); + + if (sessions.Count == 0) + { + Console.Error.WriteLine("No completed sessions found in the database."); + return 1; + } + + // Determine judge model from sessions if not specified + var effectiveJudgeModel = judgeModel ?? sessions[0].Model; + + // Validate model + try + { + var client = await AgentRunner.GetSharedClient(verbose); + var models = await client.ListModelsAsync(); + if (!models.Any(m => m.Id == effectiveJudgeModel)) + { + Console.Error.WriteLine($"Invalid model: \"{effectiveJudgeModel}\"\nAvailable models: {string.Join(", ", models.Select(m => m.Id))}"); + return 1; + } + } + catch (Exception error) + { + Console.Error.WriteLine($"Failed to validate model: {error}"); + return 1; + } + + Console.WriteLine($"Rejudging {sessions.Count} sessions with model: {effectiveJudgeModel}, mode: {judgeMode}"); + + bool usePairwise = judgeMode is JudgeMode.Pairwise or JudgeMode.Both; + var jsonOpts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; + + // Group sessions into run pairs: (skill, scenario, run_index) → (baseline, with-skill) + var runPairs = sessions + .GroupBy(s => (s.SkillName, s.ScenarioName, s.RunIndex)) + .Where(g => g.Any(s => s.Role == "baseline") && g.Any(s => s.Role == "with-skill")) + .ToList(); + + if (runPairs.Count == 0) + { + Console.Error.WriteLine("No complete run pairs (baseline + with-skill) found."); + return 1; + } + + Console.WriteLine($"Found {runPairs.Count} run pair(s) across {runPairs.Select(g => g.Key.SkillName).Distinct().Count()} skill(s)\n"); + + // Group by skill → scenario for verdict computation + var skillGroups = runPairs.GroupBy(g => g.Key.SkillName); + var verdicts = new List(); + + foreach (var skillGroup in skillGroups) + { + var skillName = skillGroup.Key; + var firstSession = skillGroup.First().First(); + Console.WriteLine($"[{skillName}] Rejudging..."); + + var scenarioGroups = skillGroup.GroupBy(g => g.Key.ScenarioName); + var comparisons = new List(); + + foreach (var scenarioGroup in scenarioGroups) + { + var scenarioName = scenarioGroup.Key; + var perRunScores = new List(); + RunResult? lastBaseline = null; + RunResult? lastWithSkill = null; + PairwiseJudgeResult? lastPairwise = null; + + foreach (var runGroup in scenarioGroup) + { + var baselineSess = runGroup.First(s => s.Role == "baseline"); + var skillSess = runGroup.First(s => s.Role == "with-skill"); + + var baselineMetrics = JsonSerializer.Deserialize(baselineSess.MetricsJson!, jsonOpts)!; + var withSkillMetrics = JsonSerializer.Deserialize(skillSess.MetricsJson!, jsonOpts)!; + + // Reconstruct scenario for judge (we need rubric) + // For now, create a minimal scenario from the saved data + var scenario = new EvalScenario(scenarioName, ""); + + // Re-judge + var judgeOpts = new JudgeOptions(effectiveJudgeModel, verbose, judgeTimeout, baselineMetrics.WorkDir, firstSession.SkillPath); + var judgeTasks = await Task.WhenAll( + Judge.JudgeRun(scenario, baselineMetrics, judgeOpts), + Judge.JudgeRun(scenario, withSkillMetrics, judgeOpts with { WorkDir = withSkillMetrics.WorkDir })); + + var baselineResult = new RunResult(baselineMetrics, judgeTasks[0]); + var withSkillResult = new RunResult(withSkillMetrics, judgeTasks[1]); + + // Update judge results in DB + sessionDb.SaveJudgeResult(baselineSess.Id, JsonSerializer.Serialize(judgeTasks[0])); + sessionDb.SaveJudgeResult(skillSess.Id, JsonSerializer.Serialize(judgeTasks[1])); + + // Pairwise + PairwiseJudgeResult? pairwise = null; + if (usePairwise) + { + try + { + pairwise = await PairwiseJudge.Judge( + scenario, baselineMetrics, withSkillMetrics, + new PairwiseJudgeOptions(effectiveJudgeModel, verbose, judgeTimeout, baselineMetrics.WorkDir, firstSession.SkillPath)); + sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise)); + } + catch (Exception error) + { + Console.Error.WriteLine($" ⚠️ Pairwise judge failed: {error.Message}"); + } + } + + var runComparison = Comparator.CompareScenario(scenarioName, baselineResult, withSkillResult, pairwise); + perRunScores.Add(runComparison.ImprovementScore); + + lastBaseline = baselineResult; + lastWithSkill = withSkillResult; + lastPairwise = pairwise; + } + + if (lastBaseline is not null && lastWithSkill is not null) + { + var comparison = Comparator.CompareScenario(scenarioName, lastBaseline, lastWithSkill, lastPairwise); + comparison.PerRunScores = perRunScores; + comparisons.Add(comparison); + } + } + + if (comparisons.Count > 0) + { + var skill = new SkillInfo(skillName, "", firstSession.SkillPath, firstSession.SkillPath, "", null, null); + var verdict = Comparator.ComputeVerdict(skill, comparisons, minImprovement, requireCompletion, confidenceLevel); + Console.WriteLine($"[{skillName}] {(verdict.Passed ? "✅" : "❌")} Score: {verdict.OverallImprovementScore * 100:F1}%"); + verdicts.Add(verdict); + } + } + + // Write new results + var reporters = new List + { + new(ReporterType.Console), + new(ReporterType.Json), + new(ReporterType.Markdown), + }; + await Reporter.ReportResults(verdicts, reporters, verbose, + effectiveJudgeModel, effectiveJudgeModel, resultsDir); + + await AgentRunner.StopSharedClient(); + + return verdicts.All(v => v.Passed) ? 0 : 1; + } +} diff --git a/eng/skill-validator/src/Commands/ValidateCommand.cs b/eng/skill-validator/src/Commands/ValidateCommand.cs index dcd4b87f14..19e814e381 100644 --- a/eng/skill-validator/src/Commands/ValidateCommand.cs +++ b/eng/skill-validator/src/Commands/ValidateCommand.cs @@ -31,6 +31,7 @@ public static RootCommand Create() var reporterOpt = new Option("--reporter") { Description = "Reporter (console, json, junit, markdown). Can be repeated.", AllowMultipleArgumentsPerToken = true }; var noOverfittingCheckOpt = new Option("--no-overfitting-check") { Description = "Disable LLM-based overfitting analysis (on by default)" }; var overfittingFixOpt = new Option("--overfitting-fix") { Description = "Generate a fixed eval.yaml with improved rubric items/assertions" }; + var keepSessionsOpt = new Option("--keep-sessions") { Description = "Preserve agent session data in the results directory for later rejudging" }; var noiseSkillsDirOpt = new Option("--noise-skills-dir") { Description = "Directory containing skills to load as noise. Enables the noise test: re-runs scenarios with all noise skills loaded and measures degradation." }; var noiseMaxDegradationOpt = new Option("--noise-max-degradation") { Description = "Maximum acceptable average quality degradation (0-1) in noise test (only positive degradations count)", DefaultValueFactory = _ => 0.2 }; var noiseMaxScenarioDegradationOpt = new Option("--noise-max-scenario-degradation") { Description = "Maximum acceptable quality degradation (0-1) for any single noise-test scenario", DefaultValueFactory = _ => 0.4 }; @@ -57,6 +58,7 @@ public static RootCommand Create() reporterOpt, noOverfittingCheckOpt, overfittingFixOpt, + keepSessionsOpt, noiseSkillsDirOpt, noiseMaxDegradationOpt, noiseMaxScenarioDegradationOpt, @@ -105,6 +107,7 @@ public static RootCommand Create() TestsDir = parseResult.GetValue(testsDirOpt), OverfittingCheck = !parseResult.GetValue(noOverfittingCheckOpt), OverfittingFix = parseResult.GetValue(overfittingFixOpt), + KeepSessions = parseResult.GetValue(keepSessionsOpt), NoiseSkillsDir = parseResult.GetValue(noiseSkillsDirOpt), NoiseDegradationLimit = parseResult.GetValue(noiseMaxDegradationOpt), NoiseMaxScenarioDegradation = parseResult.GetValue(noiseMaxScenarioDegradationOpt), @@ -321,13 +324,26 @@ public static async Task Run(ValidatorConfig config) bool usePairwise = config.JudgeMode is JudgeMode.Pairwise or JudgeMode.Both; + string? sessionsDir = null; + SessionDatabase? sessionDb = null; + string? timestampedResultsDir = null; + if (config.KeepSessions && config.ResultsDir is not null) + { + timestampedResultsDir = Path.Combine(config.ResultsDir, Reporter.FormatTimestamp(DateTime.Now)); + Directory.CreateDirectory(timestampedResultsDir); + sessionsDir = Path.Combine(timestampedResultsDir, "sessions"); + Directory.CreateDirectory(sessionsDir); + sessionDb = new SessionDatabase(Path.Combine(timestampedResultsDir, "sessions.db")); + Console.WriteLine($"Session persistence enabled: {timestampedResultsDir}"); + } + using var spinner = new Spinner(); using var skillLimit = new ConcurrencyLimiter(config.ParallelSkills); // Evaluate skills spinner.Start($"Evaluating {allSkills.Count} skill(s)..."); var skillTasks = allSkills.Select(skill => - skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills))); + skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills, sessionsDir, sessionDb))); var settled = await Task.WhenAll(skillTasks.Select(async t => { try { return (Result: await t, Error: (Exception?)null); } @@ -350,7 +366,7 @@ public static async Task Run(ValidatorConfig config) } await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose, - config.Model, config.JudgeModel, config.ResultsDir, + config.Model, config.JudgeModel, config.ResultsDir, timestampedResultsDir, rejectedCount: rejectionMessages.Count); if (rejectionMessages.Count > 0) @@ -362,7 +378,8 @@ await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose, } await AgentRunner.StopAllClients(); - await AgentRunner.CleanupWorkDirs(); + await AgentRunner.CleanupWorkDirs(config.KeepSessions); + sessionDb?.Dispose(); // Always fail on execution errors, even in --verdict-warn-only mode if (rejectionMessages.Count > 0) return 1; @@ -432,7 +449,9 @@ internal static List CheckAggregateDescriptionLimits(IReadOnlyList noiseSkills) + IReadOnlyList noiseSkills, + string? sessionsDir, + SessionDatabase? sessionDb) { var prefix = $"[{skill.Name}]"; var log = (string msg) => spinner.Log($"{prefix} {msg}"); @@ -505,7 +524,7 @@ internal static List CheckAggregateDescriptionLimits(IReadOnlyList - scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner))); + scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb))); var comparisons = (await Task.WhenAll(scenarioTasks)).ToList(); // Await overfitting result (non-fatal — never blocks an otherwise-successful evaluation) @@ -587,7 +606,9 @@ private static async Task ExecuteScenario( ValidatorConfig config, bool usePairwise, bool singleScenario, - Spinner spinner) + Spinner spinner, + string? sessionsDir, + SessionDatabase? sessionDb) { var tag = singleScenario ? $"[{skill.Name}]" : $"[{skill.Name}/{scenario.Name}]"; var scenarioLog = (string msg) => spinner.Log($"{tag} {msg}"); @@ -597,7 +618,7 @@ private static async Task ExecuteScenario( scenarioLog("📋 Starting scenario"); var runTasks = Enumerable.Range(0, config.Runs).Select(i => - runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner))); + runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb))); var runResults = await Task.WhenAll(runTasks); scenarioLog($"✓ All {config.Runs} run(s) complete"); @@ -717,7 +738,9 @@ private static async Task ExecuteRun( ValidatorConfig config, bool usePairwise, bool singleScenario, - Spinner spinner) + Spinner spinner, + string? sessionsDir, + SessionDatabase? sessionDb) { var runTag = config.Runs > 1 ? (singleScenario ? $"[{skill.Name}/{runIndex + 1}]" : $"[{skill.Name}/{scenario.Name}/{runIndex + 1}]") @@ -728,21 +751,47 @@ private static async Task ExecuteRun( runLog("running agents..."); var pluginRoot = SkillDiscovery.FindPluginRoot(skill.Path); + var baselineSessionId = Guid.NewGuid().ToString("N"); + var isolatedSessionId = Guid.NewGuid().ToString("N"); + var pluginSessionId = Guid.NewGuid().ToString("N"); + + var skillDir = Path.GetDirectoryName(skill.Path); + var skillSha = skillDir is not null ? SessionDatabase.ComputeDirectorySha(skillDir) : null; + var baselineConfigDir = sessionsDir is not null ? Path.Combine("sessions", baselineSessionId) : null; + var isolatedConfigDir = sessionsDir is not null ? Path.Combine("sessions", isolatedSessionId) : null; + var pluginConfigDir = sessionsDir is not null ? Path.Combine("sessions", pluginSessionId) : null; + + sessionDb?.RegisterSession(baselineSessionId, skill.Name, skill.Path, scenario.Name, runIndex, + "baseline", config.Model, baselineConfigDir, null, scenario.Prompt, skillSha); + sessionDb?.RegisterSession(isolatedSessionId, skill.Name, skill.Path, scenario.Name, runIndex, + "with-skill-isolated", config.Model, isolatedConfigDir, null, scenario.Prompt, skillSha); + sessionDb?.RegisterSession(pluginSessionId, skill.Name, skill.Path, scenario.Name, runIndex, + "with-skill-plugin", config.Model, pluginConfigDir, null, scenario.Prompt, skillSha); var agentTasks = await Task.WhenAll( // 1. Baseline: no plugin, no skills — vanilla agent AgentRunner.RunAgent(new RunOptions(scenario, null, skill.EvalPath, config.Model, config.Verbose, - PluginRoot: null, Log: runLog)), + PluginRoot: null, Log: runLog, SessionsDir: sessionsDir, SessionId: baselineSessionId)), // 2. Skilled-isolated: single skill only (current behavior) AgentRunner.RunAgent(new RunOptions(scenario, skill, skill.EvalPath, config.Model, config.Verbose, - PluginRoot: null, Log: runLog)), + PluginRoot: null, Log: runLog, SessionsDir: sessionsDir, SessionId: isolatedSessionId)), // 3. Skilled-plugin: load entire plugin from plugin root directory AgentRunner.RunAgent(new RunOptions(scenario, skill, skill.EvalPath, config.Model, config.Verbose, - PluginRoot: pluginRoot, Log: runLog))); + PluginRoot: pluginRoot, Log: runLog, SessionsDir: sessionsDir, SessionId: pluginSessionId))); var baselineMetrics = agentTasks[0]; var isolatedMetrics = agentTasks[1]; var pluginMetrics = agentTasks[2]; + if (sessionDb is not null) + { + var baselineStatus = baselineMetrics.TimedOut ? "timed_out" : "completed"; + var isolatedStatus = isolatedMetrics.TimedOut ? "timed_out" : "completed"; + var pluginStatus = pluginMetrics.TimedOut ? "timed_out" : "completed"; + sessionDb.CompleteSession(baselineSessionId, baselineStatus, JsonSerializer.Serialize(baselineMetrics, SkillValidatorJsonContext.Default.RunMetrics)); + sessionDb.CompleteSession(isolatedSessionId, isolatedStatus, JsonSerializer.Serialize(isolatedMetrics, SkillValidatorJsonContext.Default.RunMetrics)); + sessionDb.CompleteSession(pluginSessionId, pluginStatus, JsonSerializer.Serialize(pluginMetrics, SkillValidatorJsonContext.Default.RunMetrics)); + } + // Evaluate assertions on all three runs if (scenario.Assertions is { Count: > 0 }) { @@ -786,6 +835,13 @@ private static async Task ExecuteRun( var isolatedJudge = await SafeJudge(isolatedJudgeTask, "isolated", runLog); var pluginJudge = await SafeJudge(pluginJudgeTask, "plugin", runLog); + if (sessionDb is not null) + { + sessionDb.SaveJudgeResult(baselineSessionId, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(isolatedSessionId, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(pluginSessionId, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult)); + } + var baselineResult = new RunResult(baselineMetrics, baselineJudge); var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge); var pluginResult = new RunResult(pluginMetrics, pluginJudge); @@ -805,6 +861,10 @@ private static async Task ExecuteRun( scenario, baselineMetrics, worseSkilled, new PairwiseJudgeOptions(config.JudgeModel, config.Verbose, config.JudgeTimeout, baselineMetrics.WorkDir, skill.Path, worseSkilled.WorkDir), runLog); + if (sessionDb is not null && pairwise is not null) + { + sessionDb.SavePairwiseResult(baselineSessionId, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); + } } catch (Exception error) { diff --git a/eng/skill-validator/src/Models/Models.cs b/eng/skill-validator/src/Models/Models.cs index 2ec36ce12e..c507d377f3 100644 --- a/eng/skill-validator/src/Models/Models.cs +++ b/eng/skill-validator/src/Models/Models.cs @@ -376,6 +376,7 @@ public sealed record ValidatorConfig public string? TestsDir { get; init; } public bool OverfittingCheck { get; init; } = true; public bool OverfittingFix { get; init; } + public bool KeepSessions { get; init; } public string? NoiseSkillsDir { get; init; } public double NoiseDegradationLimit { get; init; } = 0.2; public double NoiseMaxScenarioDegradation { get; init; } = 0.4; diff --git a/eng/skill-validator/src/Program.cs b/eng/skill-validator/src/Program.cs index 042ea61ce9..90b7909ca3 100644 --- a/eng/skill-validator/src/Program.cs +++ b/eng/skill-validator/src/Program.cs @@ -3,6 +3,7 @@ var rootCommand = ValidateCommand.Create(); rootCommand.Add(ConsolidateCommand.Create()); +rootCommand.Add(RejudgeCommand.Create()); var parseResult = rootCommand.Parse(args); return await parseResult.InvokeAsync(); diff --git a/eng/skill-validator/src/Services/AgentRunner.cs b/eng/skill-validator/src/Services/AgentRunner.cs index 05686f7f25..a50e1602e1 100644 --- a/eng/skill-validator/src/Services/AgentRunner.cs +++ b/eng/skill-validator/src/Services/AgentRunner.cs @@ -16,13 +16,16 @@ public sealed record RunOptions( bool Verbose, string? PluginRoot = null, Action? Log = null, - IReadOnlyList? AdditionalSkills = null); + IReadOnlyList? AdditionalSkills = null, + string? SessionsDir = null, + string? SessionId = null); public static class AgentRunner { private static readonly ConcurrentDictionary _pluginClients = new(StringComparer.OrdinalIgnoreCase); private static readonly SemaphoreSlim _clientLock = new(1, 1); private static readonly ConcurrentBag _workDirs = []; + private static readonly ConcurrentBag _configDirs = []; private static string? _capturedGitHubToken; private static bool _tokenCaptured; @@ -102,11 +105,16 @@ public static async Task StopAllClients() public static Task StopSharedClient() => StopAllClients(); /// Remove all temporary working directories created during runs. - public static Task CleanupWorkDirs() + public static Task CleanupWorkDirs(bool keepSessions = false) { var dirs = _workDirs.ToArray(); _workDirs.Clear(); - return Task.WhenAll(dirs.Select(dir => + + var configDirsToClean = keepSessions ? [] : _configDirs.ToArray(); + _configDirs.Clear(); + + var allDirs = dirs.Concat(configDirsToClean); + return Task.WhenAll(allDirs.Select(dir => { try { Directory.Delete(dir, true); } catch { } return Task.CompletedTask; @@ -206,16 +214,28 @@ internal static SessionConfig BuildSessionConfig( IReadOnlyDictionary? mcpServers = null, IReadOnlyList? additionalSkills = null, Action? log = null, - bool verbose = false) + bool verbose = false, + string? sessionsDir = null, + string? sessionId = null) { // The SDK expects SkillDirectories entries to be parent directories that // it scans for child folders containing SKILL.md. var skillPath = skill is not null ? Path.GetDirectoryName(skill.Path) : null; - // Create a unique temporary config directory for this session to not share any data - var configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}"); - Directory.CreateDirectory(configDir); - _workDirs.Add(configDir); + string configDir; + if (sessionsDir is not null) + { + var dirName = sessionId ?? Guid.NewGuid().ToString("N"); + configDir = Path.Combine(sessionsDir, dirName); + Directory.CreateDirectory(configDir); + _configDirs.Add(configDir); + } + else + { + configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}"); + Directory.CreateDirectory(configDir); + _configDirs.Add(configDir); + } if (verbose) log?.Invoke($" 📂 Config dir: {configDir} ({(skill is not null ? "skilled" : "baseline")})"); @@ -406,7 +426,7 @@ private static async Task RunAgentCore(RunOptions options, Cancellat var client = await GetPluginClient(options.PluginRoot, options.Verbose); await using var session = await client.CreateSessionAsync( - BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills, options.Log, options.Verbose)); + BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills, options.Log, options.Verbose, options.SessionsDir, options.SessionId)); var done = new TaskCompletionSource(); var effectiveTimeout = options.Scenario.Timeout; diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs new file mode 100644 index 0000000000..373b46e8f2 --- /dev/null +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -0,0 +1,200 @@ +using Microsoft.Data.Sqlite; + +namespace SkillValidator.Services; + +/// +/// Tracks eval sessions in a SQLite database for crash recovery and rejudging. +/// Thread-safe for concurrent scenario/run execution. +/// +public sealed class SessionDatabase : IDisposable +{ + private readonly SqliteConnection _connection; + private readonly SemaphoreSlim _writeLock = new(1, 1); + + public SessionDatabase(string dbPath) + { + Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); + _connection = new SqliteConnection($"Data Source={dbPath}"); + _connection.Open(); + Initialize(); + } + + private void Initialize() + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA busy_timeout=5000; + + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + skill_name TEXT NOT NULL, + skill_path TEXT NOT NULL, + scenario_name TEXT NOT NULL, + run_index INTEGER NOT NULL, + role TEXT NOT NULL, + model TEXT NOT NULL, + config_dir TEXT, + work_dir TEXT, + status TEXT NOT NULL DEFAULT 'running', + started_at TEXT NOT NULL, + completed_at TEXT + ); + + CREATE TABLE IF NOT EXISTS run_results ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id), + metrics_json TEXT NOT NULL, + judge_json TEXT, + pairwise_json TEXT + ); + """; + cmd.ExecuteNonQuery(); + } + + public void RegisterSession(string sessionId, string skillName, string skillPath, + string scenarioName, int runIndex, string role, string model, + string? configDir, string? workDir) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, status, started_at) + VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, 'running', $started_at) + """; + cmd.Parameters.AddWithValue("$id", sessionId); + cmd.Parameters.AddWithValue("$skill_name", skillName); + cmd.Parameters.AddWithValue("$skill_path", skillPath); + cmd.Parameters.AddWithValue("$scenario_name", scenarioName); + cmd.Parameters.AddWithValue("$run_index", runIndex); + cmd.Parameters.AddWithValue("$role", role); + cmd.Parameters.AddWithValue("$model", model); + cmd.Parameters.AddWithValue("$config_dir", (object?)configDir ?? DBNull.Value); + cmd.Parameters.AddWithValue("$work_dir", (object?)workDir ?? DBNull.Value); + cmd.Parameters.AddWithValue("$started_at", DateTimeOffset.UtcNow.ToString("o")); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + public void CompleteSession(string sessionId, string status, string metricsJson) + { + _writeLock.Wait(); + try + { + using var transaction = _connection.BeginTransaction(); + + using (var cmd = _connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = "UPDATE sessions SET status = $status, completed_at = $completed_at WHERE id = $id"; + cmd.Parameters.AddWithValue("$id", sessionId); + cmd.Parameters.AddWithValue("$status", status); + cmd.Parameters.AddWithValue("$completed_at", DateTimeOffset.UtcNow.ToString("o")); + cmd.ExecuteNonQuery(); + } + + using (var cmd = _connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = """ + INSERT OR REPLACE INTO run_results (session_id, metrics_json) + VALUES ($session_id, $metrics_json) + """; + cmd.Parameters.AddWithValue("$session_id", sessionId); + cmd.Parameters.AddWithValue("$metrics_json", metricsJson); + cmd.ExecuteNonQuery(); + } + + transaction.Commit(); + } + finally { _writeLock.Release(); } + } + + public void SaveJudgeResult(string sessionId, string judgeJson) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "UPDATE run_results SET judge_json = $judge_json WHERE session_id = $session_id"; + cmd.Parameters.AddWithValue("$session_id", sessionId); + cmd.Parameters.AddWithValue("$judge_json", judgeJson); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + public void SavePairwiseResult(string baselineSessionId, string pairwiseJson) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "UPDATE run_results SET pairwise_json = $pairwise_json WHERE session_id = $session_id"; + cmd.Parameters.AddWithValue("$session_id", baselineSessionId); + cmd.Parameters.AddWithValue("$pairwise_json", pairwiseJson); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + /// + /// Returns all completed sessions grouped by (skill, scenario, run_index). + /// + public List GetCompletedSessions() + { + var results = new List(); + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT s.id, s.skill_name, s.skill_path, s.scenario_name, s.run_index, s.role, s.model, + s.config_dir, s.work_dir, s.status, + r.metrics_json, r.judge_json, r.pairwise_json + FROM sessions s + LEFT JOIN run_results r ON s.id = r.session_id + WHERE s.status IN ('completed', 'timed_out') + ORDER BY s.skill_name, s.scenario_name, s.run_index, s.role + """; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + results.Add(new SessionRecord( + Id: reader.GetString(0), + SkillName: reader.GetString(1), + SkillPath: reader.GetString(2), + ScenarioName: reader.GetString(3), + RunIndex: reader.GetInt32(4), + Role: reader.GetString(5), + Model: reader.GetString(6), + ConfigDir: reader.IsDBNull(7) ? null : reader.GetString(7), + WorkDir: reader.IsDBNull(8) ? null : reader.GetString(8), + Status: reader.GetString(9), + MetricsJson: reader.IsDBNull(10) ? null : reader.GetString(10), + JudgeJson: reader.IsDBNull(11) ? null : reader.GetString(11), + PairwiseJson: reader.IsDBNull(12) ? null : reader.GetString(12))); + } + return results; + } + + public void Dispose() + { + _connection.Dispose(); + _writeLock.Dispose(); + } +} + +public sealed record SessionRecord( + string Id, + string SkillName, + string SkillPath, + string ScenarioName, + int RunIndex, + string Role, + string Model, + string? ConfigDir, + string? WorkDir, + string Status, + string? MetricsJson, + string? JudgeJson, + string? PairwiseJson); diff --git a/eng/skill-validator/src/SkillValidator.csproj b/eng/skill-validator/src/SkillValidator.csproj index 8e449a0913..657278f702 100644 --- a/eng/skill-validator/src/SkillValidator.csproj +++ b/eng/skill-validator/src/SkillValidator.csproj @@ -31,6 +31,7 @@ + diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs new file mode 100644 index 0000000000..08e7070b35 --- /dev/null +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -0,0 +1,173 @@ +using SkillValidator.Services; + +namespace SkillValidator.Tests; + +public class SessionDatabaseTests : IDisposable +{ + private readonly string _dbPath; + private readonly SessionDatabase _db; + + public SessionDatabaseTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"test-sessions-{Guid.NewGuid()}.db"); + _db = new SessionDatabase(_dbPath); + } + + public void Dispose() + { + _db.Dispose(); + // Clear SQLite connection pool so file handles are fully released + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + TryDelete(_dbPath); + TryDelete(_dbPath + "-wal"); + TryDelete(_dbPath + "-shm"); + } + + private static void TryDelete(string path) + { + try { if (File.Exists(path)) File.Delete(path); } catch { /* best-effort cleanup */ } + } + + [Fact] + public void RegisterAndComplete_RoundTrips() + { + _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "/cfg", "/work"); + _db.CompleteSession("s1", "completed", """{"TokenEstimate":100}"""); + + var sessions = _db.GetCompletedSessions(); + var s = Assert.Single(sessions); + Assert.Equal("s1", s.Id); + Assert.Equal("my-skill", s.SkillName); + Assert.Equal("/path/to/skill", s.SkillPath); + Assert.Equal("scenario-a", s.ScenarioName); + Assert.Equal(0, s.RunIndex); + Assert.Equal("baseline", s.Role); + Assert.Equal("gpt-4.1", s.Model); + Assert.Equal("completed", s.Status); + Assert.Equal("""{"TokenEstimate":100}""", s.MetricsJson); + Assert.Null(s.JudgeJson); + Assert.Null(s.PairwiseJson); + } + + [Fact] + public void SaveJudgeResult_UpdatesExistingRow() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "completed", "{}"); + _db.SaveJudgeResult("s1", """{"OverallScore":4}"""); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Equal("""{"OverallScore":4}""", s.JudgeJson); + } + + [Fact] + public void SavePairwiseResult_UpdatesExistingRow() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "completed", "{}"); + _db.SavePairwiseResult("s1", """{"Winner":"with-skill"}"""); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Equal("""{"Winner":"with-skill"}""", s.PairwiseJson); + } + + [Fact] + public void GetCompletedSessions_ExcludesRunning() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + // Never completed — should not appear + var sessions = _db.GetCompletedSessions(); + Assert.Empty(sessions); + } + + [Fact] + public void GetCompletedSessions_IncludesTimedOut() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "timed_out", "{}"); + + var sessions = _db.GetCompletedSessions(); + Assert.Single(sessions); + Assert.Equal("timed_out", sessions[0].Status); + } + + [Fact] + public void MultipleSessions_OrderedCorrectly() + { + // Register pairs for two scenarios + _db.RegisterSession("b0", "skill", "/p", "alpha", 0, "baseline", "m", null, null); + _db.RegisterSession("w0", "skill", "/p", "alpha", 0, "with-skill", "m", null, null); + _db.RegisterSession("b1", "skill", "/p", "beta", 0, "baseline", "m", null, null); + _db.RegisterSession("w1", "skill", "/p", "beta", 0, "with-skill", "m", null, null); + + _db.CompleteSession("b0", "completed", "{}"); + _db.CompleteSession("w0", "completed", "{}"); + _db.CompleteSession("b1", "completed", "{}"); + _db.CompleteSession("w1", "completed", "{}"); + + var sessions = _db.GetCompletedSessions(); + Assert.Equal(4, sessions.Count); + // Ordered by skill_name, scenario_name, run_index, role + Assert.Equal("alpha", sessions[0].ScenarioName); + Assert.Equal("baseline", sessions[0].Role); + Assert.Equal("alpha", sessions[1].ScenarioName); + Assert.Equal("with-skill", sessions[1].Role); + Assert.Equal("beta", sessions[2].ScenarioName); + } + + [Fact] + public async Task ConcurrentWrites_DoNotCorrupt() + { + const int count = 20; + var tasks = Enumerable.Range(0, count).Select(i => Task.Run(() => + { + var id = $"s{i}"; + _db.RegisterSession(id, "skill", "/p", "scn", i, i % 2 == 0 ? "baseline" : "with-skill", "m", null, null); + _db.CompleteSession(id, "completed", $"{{\"Index\":{i}}}"); + _db.SaveJudgeResult(id, $"{{\"Score\":{i}}}"); + })); + + await Task.WhenAll(tasks); + + var sessions = _db.GetCompletedSessions(); + Assert.Equal(count, sessions.Count); + Assert.All(sessions, s => + { + Assert.Equal("completed", s.Status); + Assert.NotNull(s.MetricsJson); + Assert.NotNull(s.JudgeJson); + }); + } + + [Fact] + public void SeparateDbFiles_AreIndependent() + { + // Simulates two concurrent eval processes using different result dirs + var dbPath2 = Path.Combine(Path.GetTempPath(), $"test-sessions-{Guid.NewGuid()}.db"); + try + { + using var db2 = new SessionDatabase(dbPath2); + + _db.RegisterSession("s1", "skill-a", "/a", "scn", 0, "baseline", "m", null, null); + _db.CompleteSession("s1", "completed", "{}"); + + db2.RegisterSession("s1", "skill-b", "/b", "scn", 0, "baseline", "m", null, null); + db2.CompleteSession("s1", "completed", "{}"); + + // Each DB has exactly one session with different skill names + var sessions1 = _db.GetCompletedSessions(); + var sessions2 = db2.GetCompletedSessions(); + Assert.Single(sessions1); + Assert.Single(sessions2); + Assert.Equal("skill-a", sessions1[0].SkillName); + Assert.Equal("skill-b", sessions2[0].SkillName); + } + finally + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + TryDelete(dbPath2); + TryDelete(dbPath2 + "-wal"); + TryDelete(dbPath2 + "-shm"); + } + } +} From 2567c5655dd665a1086cd16726fd2596319a7434 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 26 Feb 2026 20:07:23 -0600 Subject: [PATCH 02/10] Add prompt and skill_content columns to sessions.db Store the eval scenario prompt and SKILL.md content in the sessions table so they are available for browsing and replay without needing the original skill directory. Both columns are optional (nullable) for backward compatibility. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Services/SessionDatabase.cs | 24 ++++++++++++------- .../tests/SessionDatabaseTests.cs | 15 +++++++++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs index 373b46e8f2..92fc7f0174 100644 --- a/eng/skill-validator/src/Services/SessionDatabase.cs +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -36,6 +36,8 @@ CREATE TABLE IF NOT EXISTS sessions ( model TEXT NOT NULL, config_dir TEXT, work_dir TEXT, + prompt TEXT, + skill_content TEXT, status TEXT NOT NULL DEFAULT 'running', started_at TEXT NOT NULL, completed_at TEXT @@ -53,15 +55,15 @@ pairwise_json TEXT public void RegisterSession(string sessionId, string skillName, string skillPath, string scenarioName, int runIndex, string role, string model, - string? configDir, string? workDir) + string? configDir, string? workDir, string? prompt = null, string? skillContent = null) { _writeLock.Wait(); try { using var cmd = _connection.CreateCommand(); cmd.CommandText = """ - INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, status, started_at) - VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, 'running', $started_at) + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, prompt, skill_content, status, started_at) + VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, $prompt, $skill_content, 'running', $started_at) """; cmd.Parameters.AddWithValue("$id", sessionId); cmd.Parameters.AddWithValue("$skill_name", skillName); @@ -72,6 +74,8 @@ INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role cmd.Parameters.AddWithValue("$model", model); cmd.Parameters.AddWithValue("$config_dir", (object?)configDir ?? DBNull.Value); cmd.Parameters.AddWithValue("$work_dir", (object?)workDir ?? DBNull.Value); + cmd.Parameters.AddWithValue("$prompt", (object?)prompt ?? DBNull.Value); + cmd.Parameters.AddWithValue("$skill_content", (object?)skillContent ?? DBNull.Value); cmd.Parameters.AddWithValue("$started_at", DateTimeOffset.UtcNow.ToString("o")); cmd.ExecuteNonQuery(); } @@ -149,7 +153,7 @@ public List GetCompletedSessions() using var cmd = _connection.CreateCommand(); cmd.CommandText = """ SELECT s.id, s.skill_name, s.skill_path, s.scenario_name, s.run_index, s.role, s.model, - s.config_dir, s.work_dir, s.status, + s.config_dir, s.work_dir, s.prompt, s.skill_content, s.status, r.metrics_json, r.judge_json, r.pairwise_json FROM sessions s LEFT JOIN run_results r ON s.id = r.session_id @@ -169,10 +173,12 @@ WHERE s.status IN ('completed', 'timed_out') Model: reader.GetString(6), ConfigDir: reader.IsDBNull(7) ? null : reader.GetString(7), WorkDir: reader.IsDBNull(8) ? null : reader.GetString(8), - Status: reader.GetString(9), - MetricsJson: reader.IsDBNull(10) ? null : reader.GetString(10), - JudgeJson: reader.IsDBNull(11) ? null : reader.GetString(11), - PairwiseJson: reader.IsDBNull(12) ? null : reader.GetString(12))); + Prompt: reader.IsDBNull(9) ? null : reader.GetString(9), + SkillContent: reader.IsDBNull(10) ? null : reader.GetString(10), + Status: reader.GetString(11), + MetricsJson: reader.IsDBNull(12) ? null : reader.GetString(12), + JudgeJson: reader.IsDBNull(13) ? null : reader.GetString(13), + PairwiseJson: reader.IsDBNull(14) ? null : reader.GetString(14))); } return results; } @@ -194,6 +200,8 @@ public sealed record SessionRecord( string Model, string? ConfigDir, string? WorkDir, + string? Prompt, + string? SkillContent, string Status, string? MetricsJson, string? JudgeJson, diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs index 08e7070b35..0a90574fda 100644 --- a/eng/skill-validator/tests/SessionDatabaseTests.cs +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -31,7 +31,7 @@ private static void TryDelete(string path) [Fact] public void RegisterAndComplete_RoundTrips() { - _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "/cfg", "/work"); + _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "/cfg", "/work", "Fix the bug", "# My Skill\nInstructions here"); _db.CompleteSession("s1", "completed", """{"TokenEstimate":100}"""); var sessions = _db.GetCompletedSessions(); @@ -44,6 +44,8 @@ public void RegisterAndComplete_RoundTrips() Assert.Equal("baseline", s.Role); Assert.Equal("gpt-4.1", s.Model); Assert.Equal("completed", s.Status); + Assert.Equal("Fix the bug", s.Prompt); + Assert.Equal("# My Skill\nInstructions here", s.SkillContent); Assert.Equal("""{"TokenEstimate":100}""", s.MetricsJson); Assert.Null(s.JudgeJson); Assert.Null(s.PairwiseJson); @@ -71,6 +73,17 @@ public void SavePairwiseResult_UpdatesExistingRow() Assert.Equal("""{"Winner":"with-skill"}""", s.PairwiseJson); } + [Fact] + public void RegisterWithoutPromptOrSkillContent_StoresNulls() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "completed", "{}"); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Null(s.Prompt); + Assert.Null(s.SkillContent); + } + [Fact] public void GetCompletedSessions_ExcludesRunning() { From 6899b8c7df9a4d90a8a0255e17525e1ef2690108 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 26 Feb 2026 20:11:55 -0600 Subject: [PATCH 03/10] Replace skill_content with skill_sha for change detection Compute a SHA-256 hash (truncated to 12 hex chars) over all files in the skill directory, sorted by relative path for determinism. This gives lightweight change detection without storing potentially large skill content in the database. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Services/SessionDatabase.cs | 39 +++++++++++++++---- .../tests/SessionDatabaseTests.cs | 34 ++++++++++++++-- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs index 92fc7f0174..da0c4ae6c6 100644 --- a/eng/skill-validator/src/Services/SessionDatabase.cs +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -1,3 +1,4 @@ +using System.Security.Cryptography; using Microsoft.Data.Sqlite; namespace SkillValidator.Services; @@ -37,7 +38,7 @@ CREATE TABLE IF NOT EXISTS sessions ( config_dir TEXT, work_dir TEXT, prompt TEXT, - skill_content TEXT, + skill_sha TEXT, status TEXT NOT NULL DEFAULT 'running', started_at TEXT NOT NULL, completed_at TEXT @@ -53,17 +54,39 @@ pairwise_json TEXT cmd.ExecuteNonQuery(); } + /// + /// Computes a SHA-256 hash over all files in a directory, sorted by relative path. + /// Returns the first 12 hex characters for a short, collision-resistant identifier. + /// + public static string ComputeDirectorySha(string dirPath) + { + using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var files = Directory.GetFiles(dirPath, "*", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(dirPath, f).Replace('\\', '/')) + .OrderBy(f => f, StringComparer.Ordinal) + .ToList(); + + foreach (var relPath in files) + { + sha.AppendData(System.Text.Encoding.UTF8.GetBytes(relPath)); + sha.AppendData(File.ReadAllBytes(Path.Combine(dirPath, relPath))); + } + + var hash = sha.GetHashAndReset(); + return Convert.ToHexString(hash)[..12].ToLowerInvariant(); + } + public void RegisterSession(string sessionId, string skillName, string skillPath, string scenarioName, int runIndex, string role, string model, - string? configDir, string? workDir, string? prompt = null, string? skillContent = null) + string? configDir, string? workDir, string? prompt = null, string? skillSha = null) { _writeLock.Wait(); try { using var cmd = _connection.CreateCommand(); cmd.CommandText = """ - INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, prompt, skill_content, status, started_at) - VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, $prompt, $skill_content, 'running', $started_at) + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, prompt, skill_sha, status, started_at) + VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, $prompt, $skill_sha, 'running', $started_at) """; cmd.Parameters.AddWithValue("$id", sessionId); cmd.Parameters.AddWithValue("$skill_name", skillName); @@ -75,7 +98,7 @@ INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role cmd.Parameters.AddWithValue("$config_dir", (object?)configDir ?? DBNull.Value); cmd.Parameters.AddWithValue("$work_dir", (object?)workDir ?? DBNull.Value); cmd.Parameters.AddWithValue("$prompt", (object?)prompt ?? DBNull.Value); - cmd.Parameters.AddWithValue("$skill_content", (object?)skillContent ?? DBNull.Value); + cmd.Parameters.AddWithValue("$skill_sha", (object?)skillSha ?? DBNull.Value); cmd.Parameters.AddWithValue("$started_at", DateTimeOffset.UtcNow.ToString("o")); cmd.ExecuteNonQuery(); } @@ -153,7 +176,7 @@ public List GetCompletedSessions() using var cmd = _connection.CreateCommand(); cmd.CommandText = """ SELECT s.id, s.skill_name, s.skill_path, s.scenario_name, s.run_index, s.role, s.model, - s.config_dir, s.work_dir, s.prompt, s.skill_content, s.status, + s.config_dir, s.work_dir, s.prompt, s.skill_sha, s.status, r.metrics_json, r.judge_json, r.pairwise_json FROM sessions s LEFT JOIN run_results r ON s.id = r.session_id @@ -174,7 +197,7 @@ WHERE s.status IN ('completed', 'timed_out') ConfigDir: reader.IsDBNull(7) ? null : reader.GetString(7), WorkDir: reader.IsDBNull(8) ? null : reader.GetString(8), Prompt: reader.IsDBNull(9) ? null : reader.GetString(9), - SkillContent: reader.IsDBNull(10) ? null : reader.GetString(10), + SkillSha: reader.IsDBNull(10) ? null : reader.GetString(10), Status: reader.GetString(11), MetricsJson: reader.IsDBNull(12) ? null : reader.GetString(12), JudgeJson: reader.IsDBNull(13) ? null : reader.GetString(13), @@ -201,7 +224,7 @@ public sealed record SessionRecord( string? ConfigDir, string? WorkDir, string? Prompt, - string? SkillContent, + string? SkillSha, string Status, string? MetricsJson, string? JudgeJson, diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs index 0a90574fda..3a766cce67 100644 --- a/eng/skill-validator/tests/SessionDatabaseTests.cs +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -31,7 +31,7 @@ private static void TryDelete(string path) [Fact] public void RegisterAndComplete_RoundTrips() { - _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "/cfg", "/work", "Fix the bug", "# My Skill\nInstructions here"); + _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "/cfg", "/work", "Fix the bug", "abcdef012345"); _db.CompleteSession("s1", "completed", """{"TokenEstimate":100}"""); var sessions = _db.GetCompletedSessions(); @@ -45,7 +45,7 @@ public void RegisterAndComplete_RoundTrips() Assert.Equal("gpt-4.1", s.Model); Assert.Equal("completed", s.Status); Assert.Equal("Fix the bug", s.Prompt); - Assert.Equal("# My Skill\nInstructions here", s.SkillContent); + Assert.Equal("abcdef012345", s.SkillSha); Assert.Equal("""{"TokenEstimate":100}""", s.MetricsJson); Assert.Null(s.JudgeJson); Assert.Null(s.PairwiseJson); @@ -74,14 +74,14 @@ public void SavePairwiseResult_UpdatesExistingRow() } [Fact] - public void RegisterWithoutPromptOrSkillContent_StoresNulls() + public void RegisterWithoutPromptOrSkillSha_StoresNulls() { _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); _db.CompleteSession("s1", "completed", "{}"); var s = Assert.Single(_db.GetCompletedSessions()); Assert.Null(s.Prompt); - Assert.Null(s.SkillContent); + Assert.Null(s.SkillSha); } [Fact] @@ -152,6 +152,32 @@ public async Task ConcurrentWrites_DoNotCorrupt() }); } + [Fact] + public void ComputeDirectorySha_IsDeterministic() + { + var dir = Path.Combine(Path.GetTempPath(), $"sha-test-{Guid.NewGuid()}"); + Directory.CreateDirectory(dir); + try + { + File.WriteAllText(Path.Combine(dir, "SKILL.md"), "# Test Skill"); + File.WriteAllText(Path.Combine(dir, "plugin.json"), "{}"); + + var sha1 = SessionDatabase.ComputeDirectorySha(dir); + var sha2 = SessionDatabase.ComputeDirectorySha(dir); + Assert.Equal(sha1, sha2); + Assert.Equal(12, sha1.Length); + + // Changing content produces a different SHA + File.WriteAllText(Path.Combine(dir, "SKILL.md"), "# Modified"); + var sha3 = SessionDatabase.ComputeDirectorySha(dir); + Assert.NotEqual(sha1, sha3); + } + finally + { + Directory.Delete(dir, true); + } + } + [Fact] public void SeparateDbFiles_AreIndependent() { From a9e3e9de1fcbfe75bf78e6985de87f7294ae327b Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Sat, 28 Feb 2026 23:00:40 -0600 Subject: [PATCH 04/10] Address PR feedback: relative config_dir, schema_info table - Use session ID as config dir folder name (sessions/) so the DB record links directly to its files on disk - Store config_dir as relative path for portable results directories - Add schema_info table with type='skill-validator' and version='1' for easy DB detection by external tools like dotnet-replay - Add GetSchemaInfo() method for consumers - Extract GetSessions(whereClause) for reuse - Add tests for schema_info and relative config_dir Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Services/AgentRunner.cs | 1 + .../src/Services/SessionDatabase.cs | 30 +++++++++++++++++-- .../tests/SessionDatabaseTests.cs | 21 ++++++++++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/eng/skill-validator/src/Services/AgentRunner.cs b/eng/skill-validator/src/Services/AgentRunner.cs index a50e1602e1..ea5cf92c50 100644 --- a/eng/skill-validator/src/Services/AgentRunner.cs +++ b/eng/skill-validator/src/Services/AgentRunner.cs @@ -225,6 +225,7 @@ internal static SessionConfig BuildSessionConfig( string configDir; if (sessionsDir is not null) { + // Persistent session dir — use sessionId as folder name for DB linkage var dirName = sessionId ?? Guid.NewGuid().ToString("N"); configDir = Path.Combine(sessionsDir, dirName); Directory.CreateDirectory(configDir); diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs index da0c4ae6c6..ee3b490870 100644 --- a/eng/skill-validator/src/Services/SessionDatabase.cs +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -27,6 +27,13 @@ private void Initialize() PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; + CREATE TABLE IF NOT EXISTS schema_info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT OR IGNORE INTO schema_info (key, value) VALUES ('type', 'skill-validator'); + INSERT OR IGNORE INTO schema_info (key, value) VALUES ('version', '1'); + CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, skill_name TEXT NOT NULL, @@ -171,16 +178,35 @@ public void SavePairwiseResult(string baselineSessionId, string pairwiseJson) /// Returns all completed sessions grouped by (skill, scenario, run_index). /// public List GetCompletedSessions() + { + return GetSessions("WHERE s.status IN ('completed', 'timed_out')"); + } + + /// + /// Returns schema metadata (type, version) for DB detection by external tools. + /// + public Dictionary GetSchemaInfo() + { + var result = new Dictionary(); + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT key, value FROM schema_info"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + result[reader.GetString(0)] = reader.GetString(1); + return result; + } + + private List GetSessions(string whereClause) { var results = new List(); using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ + cmd.CommandText = $""" SELECT s.id, s.skill_name, s.skill_path, s.scenario_name, s.run_index, s.role, s.model, s.config_dir, s.work_dir, s.prompt, s.skill_sha, s.status, r.metrics_json, r.judge_json, r.pairwise_json FROM sessions s LEFT JOIN run_results r ON s.id = r.session_id - WHERE s.status IN ('completed', 'timed_out') + {whereClause} ORDER BY s.skill_name, s.scenario_name, s.run_index, s.role """; using var reader = cmd.ExecuteReader(); diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs index 3a766cce67..acd04eae1c 100644 --- a/eng/skill-validator/tests/SessionDatabaseTests.cs +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -31,7 +31,7 @@ private static void TryDelete(string path) [Fact] public void RegisterAndComplete_RoundTrips() { - _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "/cfg", "/work", "Fix the bug", "abcdef012345"); + _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "sessions/s1", "/work", "Fix the bug", "abcdef012345"); _db.CompleteSession("s1", "completed", """{"TokenEstimate":100}"""); var sessions = _db.GetCompletedSessions(); @@ -43,6 +43,7 @@ public void RegisterAndComplete_RoundTrips() Assert.Equal(0, s.RunIndex); Assert.Equal("baseline", s.Role); Assert.Equal("gpt-4.1", s.Model); + Assert.Equal("sessions/s1", s.ConfigDir); Assert.Equal("completed", s.Status); Assert.Equal("Fix the bug", s.Prompt); Assert.Equal("abcdef012345", s.SkillSha); @@ -209,4 +210,22 @@ public void SeparateDbFiles_AreIndependent() TryDelete(dbPath2 + "-shm"); } } + + [Fact] + public void SchemaInfo_ContainsTypeAndVersion() + { + var info = _db.GetSchemaInfo(); + Assert.Equal("skill-validator", info["type"]); + Assert.Equal("1", info["version"]); + } + + [Fact] + public void ConfigDir_StoredAsRelativePath() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "m", "sessions/s1", null); + _db.CompleteSession("s1", "completed", "{}"); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Equal("sessions/s1", s.ConfigDir); + } } From b7a821864f041ae15fdbd9e836111c3053a2bfbf Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Sun, 1 Mar 2026 09:44:40 -0600 Subject: [PATCH 05/10] Fix: share timestamped results dir between sessions and reporters When --keep-sessions is active, pass the already-created timestamped directory to Reporter.ReportResults so sessions.db, session files, and report outputs (results.json, summary.md) all live in the same folder instead of creating two separate timestamped directories. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/skill-validator/src/Services/Reporter.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/skill-validator/src/Services/Reporter.cs b/eng/skill-validator/src/Services/Reporter.cs index f8ab8f95db..a69edbb4fb 100644 --- a/eng/skill-validator/src/Services/Reporter.cs +++ b/eng/skill-validator/src/Services/Reporter.cs @@ -14,13 +14,15 @@ public static async Task ReportResults( string? model = null, string? judgeModel = null, string? resultsDir = null, + string? timestampedResultsDir = null, int rejectedCount = 0) { bool needsResultsDir = reporters.Any(r => r.Type is ReporterType.Json or ReporterType.Junit or ReporterType.Markdown); - string? effectiveResultsDir = resultsDir is not null && needsResultsDir - ? Path.Combine(resultsDir, FormatTimestamp(DateTime.Now)) - : null; + string? effectiveResultsDir = timestampedResultsDir + ?? (resultsDir is not null && needsResultsDir + ? Path.Combine(resultsDir, FormatTimestamp(DateTime.Now)) + : null); if (effectiveResultsDir is not null) Directory.CreateDirectory(effectiveResultsDir); From 039880560671108ddb5863b1849cba1bf383c5a5 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Sat, 14 Mar 2026 20:34:08 -0500 Subject: [PATCH 06/10] Adapt rejudge for current validator model Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Commands/RejudgeCommand.cs | 306 ++++++++++++++---- 1 file changed, 237 insertions(+), 69 deletions(-) diff --git a/eng/skill-validator/src/Commands/RejudgeCommand.cs b/eng/skill-validator/src/Commands/RejudgeCommand.cs index 241c3ce7f4..314132a463 100644 --- a/eng/skill-validator/src/Commands/RejudgeCommand.cs +++ b/eng/skill-validator/src/Commands/RejudgeCommand.cs @@ -2,7 +2,6 @@ using System.Text.Json; using SkillValidator.Models; using SkillValidator.Services; -using SkillValidator.Utilities; namespace SkillValidator.Commands; @@ -75,17 +74,14 @@ public static async Task Run( using var sessionDb = new SessionDatabase(dbPath); var sessions = sessionDb.GetCompletedSessions(); - if (sessions.Count == 0) { Console.Error.WriteLine("No completed sessions found in the database."); return 1; } - // Determine judge model from sessions if not specified var effectiveJudgeModel = judgeModel ?? sessions[0].Model; - // Validate model try { var client = await AgentRunner.GetSharedClient(verbose); @@ -105,111 +101,144 @@ public static async Task Run( Console.WriteLine($"Rejudging {sessions.Count} sessions with model: {effectiveJudgeModel}, mode: {judgeMode}"); bool usePairwise = judgeMode is JudgeMode.Pairwise or JudgeMode.Both; - var jsonOpts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; - - // Group sessions into run pairs: (skill, scenario, run_index) → (baseline, with-skill) - var runPairs = sessions + var runGroups = sessions .GroupBy(s => (s.SkillName, s.ScenarioName, s.RunIndex)) - .Where(g => g.Any(s => s.Role == "baseline") && g.Any(s => s.Role == "with-skill")) + .Where(g => g.Any(s => s.Role == "baseline") && + (g.Any(s => s.Role == "with-skill-isolated") || g.Any(s => s.Role == "with-skill"))) .ToList(); - if (runPairs.Count == 0) + if (runGroups.Count == 0) { - Console.Error.WriteLine("No complete run pairs (baseline + with-skill) found."); + Console.Error.WriteLine("No complete run groups found."); return 1; } - Console.WriteLine($"Found {runPairs.Count} run pair(s) across {runPairs.Select(g => g.Key.SkillName).Distinct().Count()} skill(s)\n"); + Console.WriteLine($"Found {runGroups.Count} run group(s) across {runGroups.Select(g => g.Key.SkillName).Distinct().Count()} skill(s)\n"); - // Group by skill → scenario for verdict computation - var skillGroups = runPairs.GroupBy(g => g.Key.SkillName); var verdicts = new List(); - - foreach (var skillGroup in skillGroups) + foreach (var skillGroup in runGroups.GroupBy(g => g.Key.SkillName)) { var skillName = skillGroup.Key; var firstSession = skillGroup.First().First(); Console.WriteLine($"[{skillName}] Rejudging..."); - var scenarioGroups = skillGroup.GroupBy(g => g.Key.ScenarioName); var comparisons = new List(); - - foreach (var scenarioGroup in scenarioGroups) + foreach (var scenarioGroup in skillGroup.GroupBy(g => g.Key.ScenarioName)) { var scenarioName = scenarioGroup.Key; - var perRunScores = new List(); - RunResult? lastBaseline = null; - RunResult? lastWithSkill = null; - PairwiseJudgeResult? lastPairwise = null; + var rejudgedRuns = new List(); foreach (var runGroup in scenarioGroup) { var baselineSess = runGroup.First(s => s.Role == "baseline"); - var skillSess = runGroup.First(s => s.Role == "with-skill"); - - var baselineMetrics = JsonSerializer.Deserialize(baselineSess.MetricsJson!, jsonOpts)!; - var withSkillMetrics = JsonSerializer.Deserialize(skillSess.MetricsJson!, jsonOpts)!; - - // Reconstruct scenario for judge (we need rubric) - // For now, create a minimal scenario from the saved data - var scenario = new EvalScenario(scenarioName, ""); + var isolatedSess = runGroup.FirstOrDefault(s => s.Role == "with-skill-isolated") + ?? runGroup.FirstOrDefault(s => s.Role == "with-skill"); + if (isolatedSess is null) + continue; + + var pluginSess = runGroup.FirstOrDefault(s => s.Role == "with-skill-plugin"); + var prompt = baselineSess.Prompt ?? isolatedSess.Prompt ?? pluginSess?.Prompt ?? ""; + var scenario = new EvalScenario(scenarioName, prompt); + Action? log = verbose ? msg => Console.WriteLine($" [{scenarioName}/{runGroup.Key.RunIndex + 1}] {msg}") : null; + + var baselineMetrics = JsonSerializer.Deserialize(baselineSess.MetricsJson!, SkillValidatorJsonContext.Default.RunMetrics)!; + var isolatedMetrics = JsonSerializer.Deserialize(isolatedSess.MetricsJson!, SkillValidatorJsonContext.Default.RunMetrics)!; + var pluginMetrics = pluginSess?.MetricsJson is not null + ? JsonSerializer.Deserialize(pluginSess.MetricsJson, SkillValidatorJsonContext.Default.RunMetrics) + : null; - // Re-judge var judgeOpts = new JudgeOptions(effectiveJudgeModel, verbose, judgeTimeout, baselineMetrics.WorkDir, firstSession.SkillPath); - var judgeTasks = await Task.WhenAll( - Judge.JudgeRun(scenario, baselineMetrics, judgeOpts), - Judge.JudgeRun(scenario, withSkillMetrics, judgeOpts with { WorkDir = withSkillMetrics.WorkDir })); - - var baselineResult = new RunResult(baselineMetrics, judgeTasks[0]); - var withSkillResult = new RunResult(withSkillMetrics, judgeTasks[1]); + var baselineJudge = await SafeJudge( + Judge.JudgeRun(scenario, baselineMetrics, judgeOpts, log), + "baseline", + log); + var isolatedJudge = await SafeJudge( + Judge.JudgeRun(scenario, isolatedMetrics, judgeOpts with { WorkDir = isolatedMetrics.WorkDir }, log), + "isolated", + log); + var pluginJudge = pluginMetrics is not null + ? await SafeJudge( + Judge.JudgeRun(scenario, pluginMetrics, judgeOpts with { WorkDir = pluginMetrics.WorkDir }, log), + "plugin", + log) + : null; + + sessionDb.SaveJudgeResult(baselineSess.Id, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(isolatedSess.Id, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult)); + if (pluginSess is not null && pluginJudge is not null) + { + sessionDb.SaveJudgeResult(pluginSess.Id, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult)); + } - // Update judge results in DB - sessionDb.SaveJudgeResult(baselineSess.Id, JsonSerializer.Serialize(judgeTasks[0])); - sessionDb.SaveJudgeResult(skillSess.Id, JsonSerializer.Serialize(judgeTasks[1])); + var baselineResult = new RunResult(baselineMetrics, baselineJudge); + var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge); + var pluginResult = pluginMetrics is not null && pluginJudge is not null + ? new RunResult(pluginMetrics, pluginJudge) + : null; - // Pairwise PairwiseJudgeResult? pairwise = null; + bool pairwiseFromPlugin = false; if (usePairwise) { try { + var pairwiseTarget = pluginResult is not null && pluginResult.JudgeResult.OverallScore < isolatedResult.JudgeResult.OverallScore + ? pluginResult + : isolatedResult; + pairwiseFromPlugin = ReferenceEquals(pairwiseTarget, pluginResult); pairwise = await PairwiseJudge.Judge( - scenario, baselineMetrics, withSkillMetrics, - new PairwiseJudgeOptions(effectiveJudgeModel, verbose, judgeTimeout, baselineMetrics.WorkDir, firstSession.SkillPath)); - sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise)); + scenario, + baselineMetrics, + pairwiseTarget.Metrics, + new PairwiseJudgeOptions( + effectiveJudgeModel, + verbose, + judgeTimeout, + baselineMetrics.WorkDir, + firstSession.SkillPath, + pairwiseTarget.Metrics.WorkDir), + log); + sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); } catch (Exception error) { - Console.Error.WriteLine($" ⚠️ Pairwise judge failed: {error.Message}"); + log?.Invoke($"⚠️ Pairwise judge failed: {error.Message}"); } } - var runComparison = Comparator.CompareScenario(scenarioName, baselineResult, withSkillResult, pairwise); - perRunScores.Add(runComparison.ImprovementScore); - - lastBaseline = baselineResult; - lastWithSkill = withSkillResult; - lastPairwise = pairwise; + var isolatedActivation = MetricsCollector.ExtractSkillActivation( + isolatedMetrics.Events, + baselineMetrics.ToolCallBreakdown, + skillName); + var pluginActivation = pluginMetrics is not null + ? MetricsCollector.ExtractSkillActivation(pluginMetrics.Events, baselineMetrics.ToolCallBreakdown, skillName) + : null; + + rejudgedRuns.Add(new RejudgedRun( + Baseline: baselineResult, + Isolated: isolatedResult, + Plugin: pluginResult, + Pairwise: pairwise, + PairwiseFromPlugin: pairwiseFromPlugin, + IsolatedActivation: isolatedActivation, + PluginActivation: pluginActivation)); } - if (lastBaseline is not null && lastWithSkill is not null) - { - var comparison = Comparator.CompareScenario(scenarioName, lastBaseline, lastWithSkill, lastPairwise); - comparison.PerRunScores = perRunScores; - comparisons.Add(comparison); - } - } + if (rejudgedRuns.Count == 0) + continue; - if (comparisons.Count > 0) - { - var skill = new SkillInfo(skillName, "", firstSession.SkillPath, firstSession.SkillPath, "", null, null); - var verdict = Comparator.ComputeVerdict(skill, comparisons, minImprovement, requireCompletion, confidenceLevel); - Console.WriteLine($"[{skillName}] {(verdict.Passed ? "✅" : "❌")} Score: {verdict.OverallImprovementScore * 100:F1}%"); - verdicts.Add(verdict); + comparisons.Add(BuildScenarioComparison(scenarioName, rejudgedRuns)); } + + if (comparisons.Count == 0) + continue; + + var skill = new SkillInfo(skillName, "", firstSession.SkillPath, firstSession.SkillPath, "", null, null); + var verdict = Comparator.ComputeVerdict(skill, comparisons, minImprovement, requireCompletion, confidenceLevel); + Console.WriteLine($"[{skillName}] {(verdict.Passed ? "✅" : "❌")} Score: {verdict.OverallImprovementScore * 100:F1}%"); + verdicts.Add(verdict); } - // Write new results var reporters = new List { new(ReporterType.Console), @@ -217,10 +246,149 @@ public static async Task Run( new(ReporterType.Markdown), }; await Reporter.ReportResults(verdicts, reporters, verbose, - effectiveJudgeModel, effectiveJudgeModel, resultsDir); - - await AgentRunner.StopSharedClient(); + effectiveJudgeModel, effectiveJudgeModel, resultsDir, resultsDir); + await AgentRunner.StopAllClients(); return verdicts.All(v => v.Passed) ? 0 : 1; } + + private static ScenarioComparison BuildScenarioComparison(string scenarioName, List runs) + { + var baselineRuns = runs.Select(r => r.Baseline).ToList(); + var isolatedRuns = runs.Select(r => r.Isolated).ToList(); + var avgBaseline = AverageResults(baselineRuns); + var avgIsolated = AverageResults(isolatedRuns); + var bestPairwise = runs.Select(r => r.Pairwise).FirstOrDefault(p => p?.PositionSwapConsistent == true) + ?? runs.Select(r => r.Pairwise).FirstOrDefault(); + + if (runs.Any(r => r.Plugin is not null)) + { + var pluginRuns = runs.Where(r => r.Plugin is not null).Select(r => r.Plugin!).ToList(); + var perRunIsolatedScores = new List(); + var perRunPluginScores = new List(); + + foreach (var run in runs) + { + var isoComp = Comparator.CompareScenario(scenarioName, run.Baseline, run.Isolated, + run.PairwiseFromPlugin ? null : run.Pairwise); + var pluginComp = run.Plugin is not null + ? Comparator.CompareScenario(scenarioName, run.Baseline, run.Plugin, + run.PairwiseFromPlugin ? run.Pairwise : null) + : isoComp; + perRunIsolatedScores.Add(isoComp.ImprovementScore); + perRunPluginScores.Add(pluginComp.ImprovementScore); + } + + var perRunScores = perRunIsolatedScores + .Zip(perRunPluginScores, (iso, plugin) => Math.Min(iso, plugin)) + .ToList(); + var avgPlugin = AverageResults(pluginRuns); + int bestPairwiseIdx = runs.FindIndex(r => r.Pairwise?.PositionSwapConsistent == true); + if (bestPairwiseIdx < 0) + bestPairwiseIdx = runs.FindIndex(r => r.Pairwise is not null); + bool pairwiseFromPlugin = bestPairwiseIdx >= 0 && runs[bestPairwiseIdx].PairwiseFromPlugin; + + var isoComparison = Comparator.CompareScenario(scenarioName, avgBaseline, avgIsolated, + pairwiseFromPlugin ? null : bestPairwise); + var pluginComparison = Comparator.CompareScenario(scenarioName, avgBaseline, avgPlugin, + pairwiseFromPlugin ? bestPairwise : null); + + var comparison = new ScenarioComparison + { + ScenarioName = scenarioName, + Baseline = avgBaseline, + SkilledIsolated = avgIsolated, + SkilledPlugin = avgPlugin, + ImprovementScore = Math.Min(isoComparison.ImprovementScore, pluginComparison.ImprovementScore), + IsolatedImprovementScore = isoComparison.ImprovementScore, + PluginImprovementScore = pluginComparison.ImprovementScore, + Breakdown = isoComparison.ImprovementScore <= pluginComparison.ImprovementScore + ? isoComparison.Breakdown + : pluginComparison.Breakdown, + IsolatedBreakdown = isoComparison.Breakdown, + PluginBreakdown = pluginComparison.Breakdown, + PairwiseResult = bestPairwise, + PerRunScores = perRunScores, + SkillActivationIsolated = new SkillActivationInfo( + Activated: runs.Any(r => r.IsolatedActivation.Activated), + DetectedSkills: runs.SelectMany(r => r.IsolatedActivation.DetectedSkills).Distinct().ToList(), + ExtraTools: runs.SelectMany(r => r.IsolatedActivation.ExtraTools).Distinct().ToList(), + SkillEventCount: runs.Sum(r => r.IsolatedActivation.SkillEventCount)), + SkillActivationPlugin = new SkillActivationInfo( + Activated: runs.Any(r => r.PluginActivation?.Activated == true), + DetectedSkills: runs.SelectMany(r => r.PluginActivation?.DetectedSkills ?? []).Distinct().ToList(), + ExtraTools: runs.SelectMany(r => r.PluginActivation?.ExtraTools ?? []).Distinct().ToList(), + SkillEventCount: runs.Sum(r => r.PluginActivation?.SkillEventCount ?? 0)), + TimedOut = runs.Any(r => r.Baseline.Metrics.TimedOut || r.Isolated.Metrics.TimedOut || r.Plugin?.Metrics.TimedOut == true), + }; + return comparison; + } + + var comparisonNoPlugin = Comparator.CompareScenario(scenarioName, avgBaseline, avgIsolated, bestPairwise); + comparisonNoPlugin.PerRunScores = runs.Select(r => Comparator.CompareScenario(scenarioName, r.Baseline, r.Isolated, r.Pairwise).ImprovementScore).ToList(); + comparisonNoPlugin.SkillActivationIsolated = new SkillActivationInfo( + Activated: runs.Any(r => r.IsolatedActivation.Activated), + DetectedSkills: runs.SelectMany(r => r.IsolatedActivation.DetectedSkills).Distinct().ToList(), + ExtraTools: runs.SelectMany(r => r.IsolatedActivation.ExtraTools).Distinct().ToList(), + SkillEventCount: runs.Sum(r => r.IsolatedActivation.SkillEventCount)); + comparisonNoPlugin.TimedOut = runs.Any(r => r.Baseline.Metrics.TimedOut || r.Isolated.Metrics.TimedOut); + return comparisonNoPlugin; + } + + private static async Task SafeJudge(Task task, string label, Action? log) + { + try + { + return await task; + } + catch (Exception error) + { + log?.Invoke($"⚠️ Judge ({label}) failed, using fallback scores: {error.Message}"); + return new JudgeResult([], 3, $"Judge failed: {error.Message}"); + } + } + + private static RunResult AverageResults(List runs) + { + if (runs.Count == 1) + return runs[0]; + + static double Avg(IEnumerable nums) => nums.Average(); + static int AvgRound(IEnumerable nums) => (int)Math.Round(nums.Average()); + + var avgMetrics = new RunMetrics + { + TokenEstimate = AvgRound(runs.Select(r => r.Metrics.TokenEstimate)), + ToolCallCount = AvgRound(runs.Select(r => r.Metrics.ToolCallCount)), + ToolCallBreakdown = runs[0].Metrics.ToolCallBreakdown, + TurnCount = AvgRound(runs.Select(r => r.Metrics.TurnCount)), + WallTimeMs = (long)Math.Round(runs.Average(r => r.Metrics.WallTimeMs)), + ErrorCount = AvgRound(runs.Select(r => r.Metrics.ErrorCount)), + TimedOut = runs.Any(r => r.Metrics.TimedOut), + AssertionResults = runs[^1].Metrics.AssertionResults, + TaskCompleted = runs.Any(r => r.Metrics.TaskCompleted), + AgentOutput = runs[^1].Metrics.AgentOutput, + Events = runs[^1].Metrics.Events, + WorkDir = runs[^1].Metrics.WorkDir, + }; + + var avgJudge = new JudgeResult( + runs[0].JudgeResult.RubricScores.Select((score, i) => new RubricScore( + score.Criterion, + Math.Round(Avg(runs.Select(r => i < r.JudgeResult.RubricScores.Count ? r.JudgeResult.RubricScores[i].Score : 3)) * 10) / 10, + score.Reasoning)).ToList(), + Math.Round(Avg(runs.Select(r => r.JudgeResult.OverallScore)) * 10) / 10, + runs[^1].JudgeResult.OverallReasoning); + + return new RunResult(avgMetrics, avgJudge); + } + + private sealed record RejudgedRun( + RunResult Baseline, + RunResult Isolated, + RunResult? Plugin, + PairwiseJudgeResult? Pairwise, + bool PairwiseFromPlugin, + SkillActivationInfo IsolatedActivation, + SkillActivationInfo? PluginActivation); } From d0e9020c815eacb2e34c3102e885938bd4524ef8 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Sun, 15 Mar 2026 09:51:23 -0500 Subject: [PATCH 07/10] Address PR #149 review feedback Persist the judge model for rejudge, use fresh judge workdirs, tighten session DB hashing/foreign key behavior, and cache skill SHA calculation during validation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Commands/RejudgeCommand.cs | 188 +++++++++++------- .../src/Commands/ValidateCommand.cs | 14 +- .../src/Services/SessionDatabase.cs | 30 ++- .../tests/SessionDatabaseTests.cs | 42 ++++ 4 files changed, 194 insertions(+), 80 deletions(-) diff --git a/eng/skill-validator/src/Commands/RejudgeCommand.cs b/eng/skill-validator/src/Commands/RejudgeCommand.cs index 314132a463..18b4699019 100644 --- a/eng/skill-validator/src/Commands/RejudgeCommand.cs +++ b/eng/skill-validator/src/Commands/RejudgeCommand.cs @@ -10,7 +10,7 @@ public static class RejudgeCommand public static Command Create() { var resultsDirArg = new Argument("results-dir") { Description = "Path to a timestamped results directory containing sessions.db" }; - var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to original model)" }; + var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to the persisted judge model when available)" }; var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }; var judgeTimeoutOpt = new Option("--judge-timeout") { Description = "Judge timeout in seconds", DefaultValueFactory = _ => 300 }; var verboseOpt = new Option("--verbose") { Description = "Show detailed output" }; @@ -80,7 +80,14 @@ public static async Task Run( return 1; } - var effectiveJudgeModel = judgeModel ?? sessions[0].Model; + var schemaInfo = sessionDb.GetSchemaInfo(); + var persistedJudgeModel = schemaInfo.GetValueOrDefault("judge_model"); + var effectiveJudgeModel = judgeModel ?? persistedJudgeModel; + if (string.IsNullOrWhiteSpace(effectiveJudgeModel)) + { + Console.Error.WriteLine("No persisted judge model found in sessions.db. Re-run with --judge-model to specify the judge explicitly."); + return 1; + } try { @@ -147,81 +154,94 @@ public static async Task Run( ? JsonSerializer.Deserialize(pluginSess.MetricsJson, SkillValidatorJsonContext.Default.RunMetrics) : null; - var judgeOpts = new JudgeOptions(effectiveJudgeModel, verbose, judgeTimeout, baselineMetrics.WorkDir, firstSession.SkillPath); - var baselineJudge = await SafeJudge( - Judge.JudgeRun(scenario, baselineMetrics, judgeOpts, log), - "baseline", - log); - var isolatedJudge = await SafeJudge( - Judge.JudgeRun(scenario, isolatedMetrics, judgeOpts with { WorkDir = isolatedMetrics.WorkDir }, log), - "isolated", - log); - var pluginJudge = pluginMetrics is not null - ? await SafeJudge( - Judge.JudgeRun(scenario, pluginMetrics, judgeOpts with { WorkDir = pluginMetrics.WorkDir }, log), - "plugin", - log) - : null; - - sessionDb.SaveJudgeResult(baselineSess.Id, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult)); - sessionDb.SaveJudgeResult(isolatedSess.Id, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult)); - if (pluginSess is not null && pluginJudge is not null) - { - sessionDb.SaveJudgeResult(pluginSess.Id, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult)); - } - - var baselineResult = new RunResult(baselineMetrics, baselineJudge); - var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge); - var pluginResult = pluginMetrics is not null && pluginJudge is not null - ? new RunResult(pluginMetrics, pluginJudge) - : null; - - PairwiseJudgeResult? pairwise = null; - bool pairwiseFromPlugin = false; - if (usePairwise) + var judgeWorkRoot = CreateJudgeWorkDir("rejudge"); + try { - try + var judgeOpts = new JudgeOptions( + effectiveJudgeModel, + verbose, + judgeTimeout, + CreateJudgeWorkDir(judgeWorkRoot, "baseline"), + firstSession.SkillPath); + var baselineJudge = await SafeJudge( + Judge.JudgeRun(scenario, baselineMetrics, judgeOpts, log), + "baseline", + log); + var isolatedJudge = await SafeJudge( + Judge.JudgeRun(scenario, isolatedMetrics, judgeOpts with { WorkDir = CreateJudgeWorkDir(judgeWorkRoot, "isolated") }, log), + "isolated", + log); + var pluginJudge = pluginMetrics is not null + ? await SafeJudge( + Judge.JudgeRun(scenario, pluginMetrics, judgeOpts with { WorkDir = CreateJudgeWorkDir(judgeWorkRoot, "plugin") }, log), + "plugin", + log) + : null; + + sessionDb.SaveJudgeResult(baselineSess.Id, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(isolatedSess.Id, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult)); + if (pluginSess is not null && pluginJudge is not null) { - var pairwiseTarget = pluginResult is not null && pluginResult.JudgeResult.OverallScore < isolatedResult.JudgeResult.OverallScore - ? pluginResult - : isolatedResult; - pairwiseFromPlugin = ReferenceEquals(pairwiseTarget, pluginResult); - pairwise = await PairwiseJudge.Judge( - scenario, - baselineMetrics, - pairwiseTarget.Metrics, - new PairwiseJudgeOptions( - effectiveJudgeModel, - verbose, - judgeTimeout, - baselineMetrics.WorkDir, - firstSession.SkillPath, - pairwiseTarget.Metrics.WorkDir), - log); - sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); + sessionDb.SaveJudgeResult(pluginSess.Id, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult)); } - catch (Exception error) + + var baselineResult = new RunResult(baselineMetrics, baselineJudge); + var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge); + var pluginResult = pluginMetrics is not null && pluginJudge is not null + ? new RunResult(pluginMetrics, pluginJudge) + : null; + + PairwiseJudgeResult? pairwise = null; + bool pairwiseFromPlugin = false; + if (usePairwise) { - log?.Invoke($"⚠️ Pairwise judge failed: {error.Message}"); + try + { + var pairwiseTarget = pluginResult is not null && pluginResult.JudgeResult.OverallScore < isolatedResult.JudgeResult.OverallScore + ? pluginResult + : isolatedResult; + pairwiseFromPlugin = ReferenceEquals(pairwiseTarget, pluginResult); + pairwise = await PairwiseJudge.Judge( + scenario, + baselineMetrics, + pairwiseTarget.Metrics, + new PairwiseJudgeOptions( + effectiveJudgeModel, + verbose, + judgeTimeout, + CreateJudgeWorkDir(judgeWorkRoot, "pairwise"), + firstSession.SkillPath, + CreateJudgeWorkDir(judgeWorkRoot, "pairwise-skilled")), + log); + sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); + } + catch (Exception error) + { + log?.Invoke($"⚠️ Pairwise judge failed: {error.Message}"); + } } - } - - var isolatedActivation = MetricsCollector.ExtractSkillActivation( - isolatedMetrics.Events, - baselineMetrics.ToolCallBreakdown, - skillName); - var pluginActivation = pluginMetrics is not null - ? MetricsCollector.ExtractSkillActivation(pluginMetrics.Events, baselineMetrics.ToolCallBreakdown, skillName) - : null; - rejudgedRuns.Add(new RejudgedRun( - Baseline: baselineResult, - Isolated: isolatedResult, - Plugin: pluginResult, - Pairwise: pairwise, - PairwiseFromPlugin: pairwiseFromPlugin, - IsolatedActivation: isolatedActivation, - PluginActivation: pluginActivation)); + var isolatedActivation = MetricsCollector.ExtractSkillActivation( + isolatedMetrics.Events, + baselineMetrics.ToolCallBreakdown, + skillName); + var pluginActivation = pluginMetrics is not null + ? MetricsCollector.ExtractSkillActivation(pluginMetrics.Events, baselineMetrics.ToolCallBreakdown, skillName) + : null; + + rejudgedRuns.Add(new RejudgedRun( + Baseline: baselineResult, + Isolated: isolatedResult, + Plugin: pluginResult, + Pairwise: pairwise, + PairwiseFromPlugin: pairwiseFromPlugin, + IsolatedActivation: isolatedActivation, + PluginActivation: pluginActivation)); + } + finally + { + TryDeleteDirectory(judgeWorkRoot); + } } if (rejudgedRuns.Count == 0) @@ -252,6 +272,32 @@ await Reporter.ReportResults(verdicts, reporters, verbose, return verdicts.All(v => v.Passed) ? 0 : 1; } + private static string CreateJudgeWorkDir(string prefix) + { + var root = Path.Combine(Path.GetTempPath(), $"sv-{prefix}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + return root; + } + + private static string CreateJudgeWorkDir(string root, string name) + { + var path = Path.Combine(root, name); + Directory.CreateDirectory(path); + return path; + } + + private static void TryDeleteDirectory(string path) + { + try + { + if (Directory.Exists(path)) + Directory.Delete(path, true); + } + catch + { + } + } + private static ScenarioComparison BuildScenarioComparison(string scenarioName, List runs) { var baselineRuns = runs.Select(r => r.Baseline).ToList(); diff --git a/eng/skill-validator/src/Commands/ValidateCommand.cs b/eng/skill-validator/src/Commands/ValidateCommand.cs index 19e814e381..3f64de683c 100644 --- a/eng/skill-validator/src/Commands/ValidateCommand.cs +++ b/eng/skill-validator/src/Commands/ValidateCommand.cs @@ -334,6 +334,7 @@ public static async Task Run(ValidatorConfig config) sessionsDir = Path.Combine(timestampedResultsDir, "sessions"); Directory.CreateDirectory(sessionsDir); sessionDb = new SessionDatabase(Path.Combine(timestampedResultsDir, "sessions.db")); + sessionDb.SetSchemaInfo("judge_model", config.JudgeModel); Console.WriteLine($"Session persistence enabled: {timestampedResultsDir}"); } @@ -520,11 +521,12 @@ internal static List CheckAggregateDescriptionLimits(IReadOnlyList - scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb))); + scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb, skillSha))); var comparisons = (await Task.WhenAll(scenarioTasks)).ToList(); // Await overfitting result (non-fatal — never blocks an otherwise-successful evaluation) @@ -608,7 +610,8 @@ private static async Task ExecuteScenario( bool singleScenario, Spinner spinner, string? sessionsDir, - SessionDatabase? sessionDb) + SessionDatabase? sessionDb, + string? skillSha) { var tag = singleScenario ? $"[{skill.Name}]" : $"[{skill.Name}/{scenario.Name}]"; var scenarioLog = (string msg) => spinner.Log($"{tag} {msg}"); @@ -618,7 +621,7 @@ private static async Task ExecuteScenario( scenarioLog("📋 Starting scenario"); var runTasks = Enumerable.Range(0, config.Runs).Select(i => - runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb))); + runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb, skillSha))); var runResults = await Task.WhenAll(runTasks); scenarioLog($"✓ All {config.Runs} run(s) complete"); @@ -740,7 +743,8 @@ private static async Task ExecuteRun( bool singleScenario, Spinner spinner, string? sessionsDir, - SessionDatabase? sessionDb) + SessionDatabase? sessionDb, + string? skillSha) { var runTag = config.Runs > 1 ? (singleScenario ? $"[{skill.Name}/{runIndex + 1}]" : $"[{skill.Name}/{scenario.Name}/{runIndex + 1}]") @@ -755,8 +759,6 @@ private static async Task ExecuteRun( var isolatedSessionId = Guid.NewGuid().ToString("N"); var pluginSessionId = Guid.NewGuid().ToString("N"); - var skillDir = Path.GetDirectoryName(skill.Path); - var skillSha = skillDir is not null ? SessionDatabase.ComputeDirectorySha(skillDir) : null; var baselineConfigDir = sessionsDir is not null ? Path.Combine("sessions", baselineSessionId) : null; var isolatedConfigDir = sessionsDir is not null ? Path.Combine("sessions", isolatedSessionId) : null; var pluginConfigDir = sessionsDir is not null ? Path.Combine("sessions", pluginSessionId) : null; diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs index ee3b490870..1a7106c98c 100644 --- a/eng/skill-validator/src/Services/SessionDatabase.cs +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Security.Cryptography; using Microsoft.Data.Sqlite; @@ -26,6 +27,7 @@ private void Initialize() cmd.CommandText = """ PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; + PRAGMA foreign_keys=ON; CREATE TABLE IF NOT EXISTS schema_info ( key TEXT PRIMARY KEY, @@ -75,14 +77,22 @@ public static string ComputeDirectorySha(string dirPath) foreach (var relPath in files) { - sha.AppendData(System.Text.Encoding.UTF8.GetBytes(relPath)); - sha.AppendData(File.ReadAllBytes(Path.Combine(dirPath, relPath))); + AppendLengthPrefixedData(sha, System.Text.Encoding.UTF8.GetBytes(relPath)); + AppendLengthPrefixedData(sha, File.ReadAllBytes(Path.Combine(dirPath, relPath))); } var hash = sha.GetHashAndReset(); return Convert.ToHexString(hash)[..12].ToLowerInvariant(); } + private static void AppendLengthPrefixedData(IncrementalHash sha, byte[] data) + { + Span lengthPrefix = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(lengthPrefix, data.Length); + sha.AppendData(lengthPrefix); + sha.AppendData(data); + } + public void RegisterSession(string sessionId, string skillName, string skillPath, string scenarioName, int runIndex, string role, string model, string? configDir, string? workDir, string? prompt = null, string? skillSha = null) @@ -174,8 +184,22 @@ public void SavePairwiseResult(string baselineSessionId, string pairwiseJson) finally { _writeLock.Release(); } } + public void SetSchemaInfo(string key, string value) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "INSERT INTO schema_info (key, value) VALUES ($key, $value) ON CONFLICT(key) DO UPDATE SET value = excluded.value"; + cmd.Parameters.AddWithValue("$key", key); + cmd.Parameters.AddWithValue("$value", value); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + /// - /// Returns all completed sessions grouped by (skill, scenario, run_index). + /// Returns all completed sessions as a flat list ordered by skill, scenario, run index, and role. /// public List GetCompletedSessions() { diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs index acd04eae1c..78823811e1 100644 --- a/eng/skill-validator/tests/SessionDatabaseTests.cs +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -179,6 +179,32 @@ public void ComputeDirectorySha_IsDeterministic() } } + [Fact] + public void ComputeDirectorySha_DistinguishesPathAndContentBoundaries() + { + var dir1 = Path.Combine(Path.GetTempPath(), $"sha-boundary-a-{Guid.NewGuid()}"); + var dir2 = Path.Combine(Path.GetTempPath(), $"sha-boundary-b-{Guid.NewGuid()}"); + Directory.CreateDirectory(dir1); + Directory.CreateDirectory(dir2); + try + { + File.WriteAllText(Path.Combine(dir1, "a"), "12"); + File.WriteAllText(Path.Combine(dir1, "b"), "34"); + + File.WriteAllText(Path.Combine(dir2, "a1"), "2"); + File.WriteAllText(Path.Combine(dir2, "b"), "34"); + + Assert.NotEqual( + SessionDatabase.ComputeDirectorySha(dir1), + SessionDatabase.ComputeDirectorySha(dir2)); + } + finally + { + Directory.Delete(dir1, true); + Directory.Delete(dir2, true); + } + } + [Fact] public void SeparateDbFiles_AreIndependent() { @@ -219,6 +245,22 @@ public void SchemaInfo_ContainsTypeAndVersion() Assert.Equal("1", info["version"]); } + [Fact] + public void SchemaInfo_CanPersistJudgeModel() + { + _db.SetSchemaInfo("judge_model", "claude-opus-4.6"); + + var info = _db.GetSchemaInfo(); + Assert.Equal("claude-opus-4.6", info["judge_model"]); + } + + [Fact] + public void CompleteSession_RequiresExistingSession() + { + Assert.Throws(() => + _db.CompleteSession("missing", "completed", "{}")); + } + [Fact] public void ConfigDir_StoredAsRelativePath() { From e5c80ab3a96e36b1ccfb3ce6ae0f8bc621677e99 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Sun, 15 Mar 2026 12:55:25 -0500 Subject: [PATCH 08/10] Fix rejudge rubric persistence Persist scenario rubrics in sessions.db so rejudge can reconstruct the original EvalScenario rubric, self-migrate older databases to the new schema, and validate --judge-mode inputs explicitly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Commands/RejudgeCommand.cs | 28 ++++++- .../src/Commands/ValidateCommand.cs | 10 ++- .../src/Services/SessionDatabase.cs | 50 +++++++++--- .../src/SkillValidatorJsonContext.cs | 1 + .../tests/SessionDatabaseTests.cs | 80 ++++++++++++++++++- 5 files changed, 151 insertions(+), 18 deletions(-) diff --git a/eng/skill-validator/src/Commands/RejudgeCommand.cs b/eng/skill-validator/src/Commands/RejudgeCommand.cs index 18b4699019..6e7361347b 100644 --- a/eng/skill-validator/src/Commands/RejudgeCommand.cs +++ b/eng/skill-validator/src/Commands/RejudgeCommand.cs @@ -11,7 +11,8 @@ public static Command Create() { var resultsDirArg = new Argument("results-dir") { Description = "Path to a timestamped results directory containing sessions.db" }; var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to the persisted judge model when available)" }; - var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }; + var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" } + .AcceptOnlyFromAmong("pairwise", "independent", "both"); var judgeTimeoutOpt = new Option("--judge-timeout") { Description = "Judge timeout in seconds", DefaultValueFactory = _ => 300 }; var verboseOpt = new Option("--verbose") { Description = "Show detailed output" }; var minImprovementOpt = new Option("--min-improvement") { Description = "Minimum improvement score to pass (0-1)", DefaultValueFactory = _ => 0.1 }; @@ -133,6 +134,7 @@ public static async Task Run( foreach (var scenarioGroup in skillGroup.GroupBy(g => g.Key.ScenarioName)) { var scenarioName = scenarioGroup.Key; + var storedRubric = GetStoredRubric(skillName, scenarioName, scenarioGroup.SelectMany(g => g)); var rejudgedRuns = new List(); foreach (var runGroup in scenarioGroup) @@ -145,7 +147,7 @@ public static async Task Run( var pluginSess = runGroup.FirstOrDefault(s => s.Role == "with-skill-plugin"); var prompt = baselineSess.Prompt ?? isolatedSess.Prompt ?? pluginSess?.Prompt ?? ""; - var scenario = new EvalScenario(scenarioName, prompt); + var scenario = new EvalScenario(scenarioName, prompt, Rubric: storedRubric); Action? log = verbose ? msg => Console.WriteLine($" [{scenarioName}/{runGroup.Key.RunIndex + 1}] {msg}") : null; var baselineMetrics = JsonSerializer.Deserialize(baselineSess.MetricsJson!, SkillValidatorJsonContext.Default.RunMetrics)!; @@ -381,6 +383,28 @@ private static ScenarioComparison BuildScenarioComparison(string scenarioName, L return comparisonNoPlugin; } + private static string[]? GetStoredRubric(string skillName, string scenarioName, IEnumerable sessions) + { + var rubricJson = sessions + .Select(s => s.RubricJson) + .FirstOrDefault(r => !string.IsNullOrWhiteSpace(r)); + if (rubricJson is null) + { + Console.WriteLine($"[{skillName}] ⚠️ Scenario '{scenarioName}' has no persisted rubric in sessions.db; falling back to the default judging rubric."); + return null; + } + + try + { + return JsonSerializer.Deserialize(rubricJson, SkillValidatorJsonContext.Default.StringArray) ?? []; + } + catch (JsonException error) + { + Console.WriteLine($"[{skillName}] ⚠️ Scenario '{scenarioName}' has an unreadable persisted rubric ({error.Message}); falling back to the default judging rubric."); + return null; + } + } + private static async Task SafeJudge(Task task, string label, Action? log) { try diff --git a/eng/skill-validator/src/Commands/ValidateCommand.cs b/eng/skill-validator/src/Commands/ValidateCommand.cs index 3f64de683c..741444481f 100644 --- a/eng/skill-validator/src/Commands/ValidateCommand.cs +++ b/eng/skill-validator/src/Commands/ValidateCommand.cs @@ -19,7 +19,8 @@ public static RootCommand Create() var verboseOpt = new Option("--verbose") { Description = "Show detailed per-scenario breakdowns" }; var modelOpt = new Option("--model") { Description = "Model to use for agent runs", DefaultValueFactory = _ => "claude-opus-4.6" }; var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to --model)" }; - var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }; + var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" } + .AcceptOnlyFromAmong("pairwise", "independent", "both"); var runsOpt = new Option("--runs") { Description = "Number of runs per scenario for averaging", DefaultValueFactory = _ => 5 }; var parallelSkillsOpt = new Option("--parallel-skills") { Description = "Max concurrent skills to evaluate", DefaultValueFactory = _ => 1 }; var parallelScenariosOpt = new Option("--parallel-scenarios") { Description = "Max concurrent scenarios per skill", DefaultValueFactory = _ => 1 }; @@ -762,13 +763,14 @@ private static async Task ExecuteRun( var baselineConfigDir = sessionsDir is not null ? Path.Combine("sessions", baselineSessionId) : null; var isolatedConfigDir = sessionsDir is not null ? Path.Combine("sessions", isolatedSessionId) : null; var pluginConfigDir = sessionsDir is not null ? Path.Combine("sessions", pluginSessionId) : null; + var rubricJson = JsonSerializer.Serialize(scenario.Rubric?.ToArray() ?? [], SkillValidatorJsonContext.Default.StringArray); sessionDb?.RegisterSession(baselineSessionId, skill.Name, skill.Path, scenario.Name, runIndex, - "baseline", config.Model, baselineConfigDir, null, scenario.Prompt, skillSha); + "baseline", config.Model, baselineConfigDir, null, scenario.Prompt, skillSha, rubricJson); sessionDb?.RegisterSession(isolatedSessionId, skill.Name, skill.Path, scenario.Name, runIndex, - "with-skill-isolated", config.Model, isolatedConfigDir, null, scenario.Prompt, skillSha); + "with-skill-isolated", config.Model, isolatedConfigDir, null, scenario.Prompt, skillSha, rubricJson); sessionDb?.RegisterSession(pluginSessionId, skill.Name, skill.Path, scenario.Name, runIndex, - "with-skill-plugin", config.Model, pluginConfigDir, null, scenario.Prompt, skillSha); + "with-skill-plugin", config.Model, pluginConfigDir, null, scenario.Prompt, skillSha, rubricJson); var agentTasks = await Task.WhenAll( // 1. Baseline: no plugin, no skills — vanilla agent diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs index 1a7106c98c..7dc18bd566 100644 --- a/eng/skill-validator/src/Services/SessionDatabase.cs +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS schema_info ( value TEXT NOT NULL ); INSERT OR IGNORE INTO schema_info (key, value) VALUES ('type', 'skill-validator'); - INSERT OR IGNORE INTO schema_info (key, value) VALUES ('version', '1'); + INSERT OR IGNORE INTO schema_info (key, value) VALUES ('version', '2'); CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, @@ -50,7 +50,8 @@ CREATE TABLE IF NOT EXISTS sessions ( skill_sha TEXT, status TEXT NOT NULL DEFAULT 'running', started_at TEXT NOT NULL, - completed_at TEXT + completed_at TEXT, + rubric TEXT ); CREATE TABLE IF NOT EXISTS run_results ( @@ -61,6 +62,32 @@ pairwise_json TEXT ); """; cmd.ExecuteNonQuery(); + EnsureSessionsRubricColumn(); + SetSchemaInfo("version", "2"); + } + + private void EnsureSessionsRubricColumn() + { + if (HasColumn("sessions", "rubric")) + return; + + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE sessions ADD COLUMN rubric TEXT"; + cmd.ExecuteNonQuery(); + } + + private bool HasColumn(string tableName, string columnName) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = $"PRAGMA table_info({tableName})"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase)) + return true; + } + + return false; } /// @@ -95,15 +122,15 @@ private static void AppendLengthPrefixedData(IncrementalHash sha, byte[] data) public void RegisterSession(string sessionId, string skillName, string skillPath, string scenarioName, int runIndex, string role, string model, - string? configDir, string? workDir, string? prompt = null, string? skillSha = null) + string? configDir, string? workDir, string? prompt = null, string? skillSha = null, string? rubric = null) { _writeLock.Wait(); try { using var cmd = _connection.CreateCommand(); cmd.CommandText = """ - INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, prompt, skill_sha, status, started_at) - VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, $prompt, $skill_sha, 'running', $started_at) + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, prompt, skill_sha, rubric, status, started_at) + VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, $prompt, $skill_sha, $rubric, 'running', $started_at) """; cmd.Parameters.AddWithValue("$id", sessionId); cmd.Parameters.AddWithValue("$skill_name", skillName); @@ -116,6 +143,7 @@ INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role cmd.Parameters.AddWithValue("$work_dir", (object?)workDir ?? DBNull.Value); cmd.Parameters.AddWithValue("$prompt", (object?)prompt ?? DBNull.Value); cmd.Parameters.AddWithValue("$skill_sha", (object?)skillSha ?? DBNull.Value); + cmd.Parameters.AddWithValue("$rubric", (object?)rubric ?? DBNull.Value); cmd.Parameters.AddWithValue("$started_at", DateTimeOffset.UtcNow.ToString("o")); cmd.ExecuteNonQuery(); } @@ -226,7 +254,7 @@ private List GetSessions(string whereClause) using var cmd = _connection.CreateCommand(); cmd.CommandText = $""" SELECT s.id, s.skill_name, s.skill_path, s.scenario_name, s.run_index, s.role, s.model, - s.config_dir, s.work_dir, s.prompt, s.skill_sha, s.status, + s.config_dir, s.work_dir, s.prompt, s.skill_sha, s.rubric, s.status, r.metrics_json, r.judge_json, r.pairwise_json FROM sessions s LEFT JOIN run_results r ON s.id = r.session_id @@ -248,10 +276,11 @@ FROM sessions s WorkDir: reader.IsDBNull(8) ? null : reader.GetString(8), Prompt: reader.IsDBNull(9) ? null : reader.GetString(9), SkillSha: reader.IsDBNull(10) ? null : reader.GetString(10), - Status: reader.GetString(11), - MetricsJson: reader.IsDBNull(12) ? null : reader.GetString(12), - JudgeJson: reader.IsDBNull(13) ? null : reader.GetString(13), - PairwiseJson: reader.IsDBNull(14) ? null : reader.GetString(14))); + RubricJson: reader.IsDBNull(11) ? null : reader.GetString(11), + Status: reader.GetString(12), + MetricsJson: reader.IsDBNull(13) ? null : reader.GetString(13), + JudgeJson: reader.IsDBNull(14) ? null : reader.GetString(14), + PairwiseJson: reader.IsDBNull(15) ? null : reader.GetString(15))); } return results; } @@ -275,6 +304,7 @@ public sealed record SessionRecord( string? WorkDir, string? Prompt, string? SkillSha, + string? RubricJson, string Status, string? MetricsJson, string? JudgeJson, diff --git a/eng/skill-validator/src/SkillValidatorJsonContext.cs b/eng/skill-validator/src/SkillValidatorJsonContext.cs index 8fd69c82a0..03d5719632 100644 --- a/eng/skill-validator/src/SkillValidatorJsonContext.cs +++ b/eng/skill-validator/src/SkillValidatorJsonContext.cs @@ -38,6 +38,7 @@ namespace SkillValidator; [JsonSerializable(typeof(JsonElement))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(Dictionary))] +[JsonSerializable(typeof(string[]))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(IReadOnlyList))] internal partial class SkillValidatorJsonContext : JsonSerializerContext; diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs index 78823811e1..c9ca84c676 100644 --- a/eng/skill-validator/tests/SessionDatabaseTests.cs +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -1,3 +1,5 @@ +using System.Text.Json; +using Microsoft.Data.Sqlite; using SkillValidator.Services; namespace SkillValidator.Tests; @@ -31,7 +33,9 @@ private static void TryDelete(string path) [Fact] public void RegisterAndComplete_RoundTrips() { - _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "sessions/s1", "/work", "Fix the bug", "abcdef012345"); + var rubricJson = JsonSerializer.Serialize(new[] { "Quality", "Completeness" }); + + _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "sessions/s1", "/work", "Fix the bug", "abcdef012345", rubricJson); _db.CompleteSession("s1", "completed", """{"TokenEstimate":100}"""); var sessions = _db.GetCompletedSessions(); @@ -47,6 +51,7 @@ public void RegisterAndComplete_RoundTrips() Assert.Equal("completed", s.Status); Assert.Equal("Fix the bug", s.Prompt); Assert.Equal("abcdef012345", s.SkillSha); + Assert.Equal(rubricJson, s.RubricJson); Assert.Equal("""{"TokenEstimate":100}""", s.MetricsJson); Assert.Null(s.JudgeJson); Assert.Null(s.PairwiseJson); @@ -83,6 +88,7 @@ public void RegisterWithoutPromptOrSkillSha_StoresNulls() var s = Assert.Single(_db.GetCompletedSessions()); Assert.Null(s.Prompt); Assert.Null(s.SkillSha); + Assert.Null(s.RubricJson); } [Fact] @@ -242,7 +248,7 @@ public void SchemaInfo_ContainsTypeAndVersion() { var info = _db.GetSchemaInfo(); Assert.Equal("skill-validator", info["type"]); - Assert.Equal("1", info["version"]); + Assert.Equal("2", info["version"]); } [Fact] @@ -254,6 +260,76 @@ public void SchemaInfo_CanPersistJudgeModel() Assert.Equal("claude-opus-4.6", info["judge_model"]); } + [Fact] + public void LegacyDatabase_UpgradesRubricColumn() + { + var legacyDbPath = Path.Combine(Path.GetTempPath(), $"legacy-sessions-{Guid.NewGuid()}.db"); + try + { + using (var connection = new SqliteConnection($"Data Source={legacyDbPath}")) + { + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = """ + CREATE TABLE schema_info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT INTO schema_info (key, value) VALUES ('type', 'skill-validator'); + INSERT INTO schema_info (key, value) VALUES ('version', '1'); + + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + skill_name TEXT NOT NULL, + skill_path TEXT NOT NULL, + scenario_name TEXT NOT NULL, + run_index INTEGER NOT NULL, + role TEXT NOT NULL, + model TEXT NOT NULL, + config_dir TEXT, + work_dir TEXT, + prompt TEXT, + skill_sha TEXT, + status TEXT NOT NULL DEFAULT 'running', + started_at TEXT NOT NULL, + completed_at TEXT + ); + + CREATE TABLE run_results ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id), + metrics_json TEXT NOT NULL, + judge_json TEXT, + pairwise_json TEXT + ); + + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, status, started_at, completed_at) + VALUES ('s1', 'skill', '/p', 'scn', 0, 'baseline', 'model', 'completed', '2026-01-01T00:00:00Z', '2026-01-01T00:01:00Z'); + INSERT INTO run_results (session_id, metrics_json) VALUES ('s1', '{}'); + """; + cmd.ExecuteNonQuery(); + } + + using var upgradedDb = new SessionDatabase(legacyDbPath); + var legacySession = Assert.Single(upgradedDb.GetCompletedSessions()); + Assert.Null(legacySession.RubricJson); + Assert.Equal("2", upgradedDb.GetSchemaInfo()["version"]); + + var rubricJson = JsonSerializer.Serialize(new[] { "Quality" }); + upgradedDb.RegisterSession("s2", "skill", "/p", "scn", 1, "with-skill", "model", null, null, "Prompt", null, rubricJson); + upgradedDb.CompleteSession("s2", "completed", "{}"); + + var upgradedSession = Assert.Single(upgradedDb.GetCompletedSessions(), s => s.Id == "s2"); + Assert.Equal(rubricJson, upgradedSession.RubricJson); + } + finally + { + SqliteConnection.ClearAllPools(); + TryDelete(legacyDbPath); + TryDelete(legacyDbPath + "-wal"); + TryDelete(legacyDbPath + "-shm"); + } + } + [Fact] public void CompleteSession_RequiresExistingSession() { From e632b63bf9b830873132f12353c78db3bcea32c0 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Sun, 15 Mar 2026 12:59:32 -0500 Subject: [PATCH 09/10] Guard SessionDatabase reads with semaphore Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Services/SessionDatabase.cs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs index 7dc18bd566..ef1950af38 100644 --- a/eng/skill-validator/src/Services/SessionDatabase.cs +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -231,7 +231,12 @@ public void SetSchemaInfo(string key, string value) /// public List GetCompletedSessions() { - return GetSessions("WHERE s.status IN ('completed', 'timed_out')"); + _writeLock.Wait(); + try + { + return GetSessions("WHERE s.status IN ('completed', 'timed_out')"); + } + finally { _writeLock.Release(); } } /// @@ -239,13 +244,18 @@ public List GetCompletedSessions() /// public Dictionary GetSchemaInfo() { - var result = new Dictionary(); - using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT key, value FROM schema_info"; - using var reader = cmd.ExecuteReader(); - while (reader.Read()) - result[reader.GetString(0)] = reader.GetString(1); - return result; + _writeLock.Wait(); + try + { + var result = new Dictionary(); + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT key, value FROM schema_info"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + result[reader.GetString(0)] = reader.GetString(1); + return result; + } + finally { _writeLock.Release(); } } private List GetSessions(string whereClause) From f505e42aba180f2d53044bb00151ef64d81b570c Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Mon, 16 Mar 2026 13:22:12 -0500 Subject: [PATCH 10/10] Fix validator session cleanup and reporting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/skill-validator/src/Commands/RejudgeCommand.cs | 11 ++++++----- eng/skill-validator/src/Commands/ValidateCommand.cs | 11 ++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/eng/skill-validator/src/Commands/RejudgeCommand.cs b/eng/skill-validator/src/Commands/RejudgeCommand.cs index 6e7361347b..2e89e67c4d 100644 --- a/eng/skill-validator/src/Commands/RejudgeCommand.cs +++ b/eng/skill-validator/src/Commands/RejudgeCommand.cs @@ -123,11 +123,12 @@ public static async Task Run( Console.WriteLine($"Found {runGroups.Count} run group(s) across {runGroups.Select(g => g.Key.SkillName).Distinct().Count()} skill(s)\n"); + var firstSession = sessions[0]; var verdicts = new List(); foreach (var skillGroup in runGroups.GroupBy(g => g.Key.SkillName)) { var skillName = skillGroup.Key; - var firstSession = skillGroup.First().First(); + var firstSkillSession = skillGroup.First().First(); Console.WriteLine($"[{skillName}] Rejudging..."); var comparisons = new List(); @@ -164,7 +165,7 @@ public static async Task Run( verbose, judgeTimeout, CreateJudgeWorkDir(judgeWorkRoot, "baseline"), - firstSession.SkillPath); + firstSkillSession.SkillPath); var baselineJudge = await SafeJudge( Judge.JudgeRun(scenario, baselineMetrics, judgeOpts, log), "baseline", @@ -212,7 +213,7 @@ public static async Task Run( verbose, judgeTimeout, CreateJudgeWorkDir(judgeWorkRoot, "pairwise"), - firstSession.SkillPath, + firstSkillSession.SkillPath, CreateJudgeWorkDir(judgeWorkRoot, "pairwise-skilled")), log); sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); @@ -255,7 +256,7 @@ public static async Task Run( if (comparisons.Count == 0) continue; - var skill = new SkillInfo(skillName, "", firstSession.SkillPath, firstSession.SkillPath, "", null, null); + var skill = new SkillInfo(skillName, "", firstSkillSession.SkillPath, firstSkillSession.SkillPath, "", null, null); var verdict = Comparator.ComputeVerdict(skill, comparisons, minImprovement, requireCompletion, confidenceLevel); Console.WriteLine($"[{skillName}] {(verdict.Passed ? "✅" : "❌")} Score: {verdict.OverallImprovementScore * 100:F1}%"); verdicts.Add(verdict); @@ -268,7 +269,7 @@ public static async Task Run( new(ReporterType.Markdown), }; await Reporter.ReportResults(verdicts, reporters, verbose, - effectiveJudgeModel, effectiveJudgeModel, resultsDir, resultsDir); + firstSession.Model, effectiveJudgeModel, resultsDir, resultsDir); await AgentRunner.StopAllClients(); return verdicts.All(v => v.Passed) ? 0 : 1; diff --git a/eng/skill-validator/src/Commands/ValidateCommand.cs b/eng/skill-validator/src/Commands/ValidateCommand.cs index 741444481f..f05c5ceb30 100644 --- a/eng/skill-validator/src/Commands/ValidateCommand.cs +++ b/eng/skill-validator/src/Commands/ValidateCommand.cs @@ -324,13 +324,14 @@ public static async Task Run(ValidatorConfig config) Console.WriteLine($"\x1b[33m⚠ Running with {config.Runs} run(s). For statistically significant results, use --runs 5 or higher.\x1b[0m"); bool usePairwise = config.JudgeMode is JudgeMode.Pairwise or JudgeMode.Both; + bool effectiveKeepSessions = config.KeepSessions && config.ResultsDir is not null; string? sessionsDir = null; SessionDatabase? sessionDb = null; string? timestampedResultsDir = null; - if (config.KeepSessions && config.ResultsDir is not null) + if (effectiveKeepSessions) { - timestampedResultsDir = Path.Combine(config.ResultsDir, Reporter.FormatTimestamp(DateTime.Now)); + timestampedResultsDir = Path.Combine(config.ResultsDir!, Reporter.FormatTimestamp(DateTime.Now)); Directory.CreateDirectory(timestampedResultsDir); sessionsDir = Path.Combine(timestampedResultsDir, "sessions"); Directory.CreateDirectory(sessionsDir); @@ -338,6 +339,10 @@ public static async Task Run(ValidatorConfig config) sessionDb.SetSchemaInfo("judge_model", config.JudgeModel); Console.WriteLine($"Session persistence enabled: {timestampedResultsDir}"); } + else if (config.KeepSessions) + { + Console.WriteLine("\x1b[33m⚠ --keep-sessions was set without --results-dir; sessions will not be persisted.\x1b[0m"); + } using var spinner = new Spinner(); using var skillLimit = new ConcurrencyLimiter(config.ParallelSkills); @@ -380,7 +385,7 @@ await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose, } await AgentRunner.StopAllClients(); - await AgentRunner.CleanupWorkDirs(config.KeepSessions); + await AgentRunner.CleanupWorkDirs(effectiveKeepSessions); sessionDb?.Dispose(); // Always fail on execution errors, even in --verdict-warn-only mode