diff --git a/eng/skill-validator/src/Commands/RejudgeCommand.cs b/eng/skill-validator/src/Commands/RejudgeCommand.cs new file mode 100644 index 0000000000..2e89e67c4d --- /dev/null +++ b/eng/skill-validator/src/Commands/RejudgeCommand.cs @@ -0,0 +1,465 @@ +using System.CommandLine; +using System.Text.Json; +using SkillValidator.Models; +using SkillValidator.Services; + +namespace SkillValidator.Commands; + +public static class RejudgeCommand +{ + public static Command Create() + { + var resultsDirArg = new Argument("results-dir") { Description = "Path to a timestamped results directory containing sessions.db" }; + var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to the persisted judge model when available)" }; + var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" } + .AcceptOnlyFromAmong("pairwise", "independent", "both"); + var judgeTimeoutOpt = new Option("--judge-timeout") { Description = "Judge timeout in seconds", DefaultValueFactory = _ => 300 }; + var verboseOpt = new Option("--verbose") { Description = "Show detailed output" }; + var minImprovementOpt = new Option("--min-improvement") { Description = "Minimum improvement score to pass (0-1)", DefaultValueFactory = _ => 0.1 }; + var requireCompletionOpt = new Option("--require-completion") { Description = "Fail if skill regresses task completion", DefaultValueFactory = _ => true }; + var confidenceLevelOpt = new Option("--confidence-level") { Description = "Confidence level for statistical intervals (0-1)", DefaultValueFactory = _ => 0.95 }; + + var command = new Command("rejudge", "Re-run judges on saved sessions without re-running agents") + { + resultsDirArg, + judgeModelOpt, + judgeModeOpt, + judgeTimeoutOpt, + verboseOpt, + minImprovementOpt, + requireCompletionOpt, + confidenceLevelOpt, + }; + + command.SetAction(async (parseResult, _) => + { + var resultsDir = parseResult.GetValue(resultsDirArg)!; + var judgeModel = parseResult.GetValue(judgeModelOpt); + var verbose = parseResult.GetValue(verboseOpt); + var judgeTimeout = parseResult.GetValue(judgeTimeoutOpt) * 1000; + var minImprovement = parseResult.GetValue(minImprovementOpt); + var requireCompletion = parseResult.GetValue(requireCompletionOpt); + var confidenceLevel = parseResult.GetValue(confidenceLevelOpt); + + var judgeMode = parseResult.GetValue(judgeModeOpt) switch + { + "independent" => JudgeMode.Independent, + "both" => JudgeMode.Both, + _ => JudgeMode.Pairwise, + }; + + return await Run(resultsDir, judgeModel, judgeMode, judgeTimeout, verbose, + minImprovement, requireCompletion, confidenceLevel); + }); + + return command; + } + + public static async Task Run( + string resultsDir, + string? judgeModel, + JudgeMode judgeMode, + int judgeTimeout, + bool verbose, + double minImprovement, + bool requireCompletion, + double confidenceLevel) + { + var dbPath = Path.Combine(resultsDir, "sessions.db"); + if (!File.Exists(dbPath)) + { + Console.Error.WriteLine($"No sessions.db found at {dbPath}"); + Console.Error.WriteLine("Use --keep-sessions during evaluation to enable rejudging."); + return 1; + } + + using var sessionDb = new SessionDatabase(dbPath); + var sessions = sessionDb.GetCompletedSessions(); + if (sessions.Count == 0) + { + Console.Error.WriteLine("No completed sessions found in the database."); + return 1; + } + + var schemaInfo = sessionDb.GetSchemaInfo(); + var persistedJudgeModel = schemaInfo.GetValueOrDefault("judge_model"); + var effectiveJudgeModel = judgeModel ?? persistedJudgeModel; + if (string.IsNullOrWhiteSpace(effectiveJudgeModel)) + { + Console.Error.WriteLine("No persisted judge model found in sessions.db. Re-run with --judge-model to specify the judge explicitly."); + return 1; + } + + try + { + var client = await AgentRunner.GetSharedClient(verbose); + var models = await client.ListModelsAsync(); + if (!models.Any(m => m.Id == effectiveJudgeModel)) + { + Console.Error.WriteLine($"Invalid model: \"{effectiveJudgeModel}\"\nAvailable models: {string.Join(", ", models.Select(m => m.Id))}"); + return 1; + } + } + catch (Exception error) + { + Console.Error.WriteLine($"Failed to validate model: {error}"); + return 1; + } + + Console.WriteLine($"Rejudging {sessions.Count} sessions with model: {effectiveJudgeModel}, mode: {judgeMode}"); + + bool usePairwise = judgeMode is JudgeMode.Pairwise or JudgeMode.Both; + var runGroups = sessions + .GroupBy(s => (s.SkillName, s.ScenarioName, s.RunIndex)) + .Where(g => g.Any(s => s.Role == "baseline") && + (g.Any(s => s.Role == "with-skill-isolated") || g.Any(s => s.Role == "with-skill"))) + .ToList(); + + if (runGroups.Count == 0) + { + Console.Error.WriteLine("No complete run groups found."); + return 1; + } + + Console.WriteLine($"Found {runGroups.Count} run group(s) across {runGroups.Select(g => g.Key.SkillName).Distinct().Count()} skill(s)\n"); + + var firstSession = sessions[0]; + var verdicts = new List(); + foreach (var skillGroup in runGroups.GroupBy(g => g.Key.SkillName)) + { + var skillName = skillGroup.Key; + var firstSkillSession = skillGroup.First().First(); + Console.WriteLine($"[{skillName}] Rejudging..."); + + var comparisons = new List(); + foreach (var scenarioGroup in skillGroup.GroupBy(g => g.Key.ScenarioName)) + { + var scenarioName = scenarioGroup.Key; + var storedRubric = GetStoredRubric(skillName, scenarioName, scenarioGroup.SelectMany(g => g)); + var rejudgedRuns = new List(); + + foreach (var runGroup in scenarioGroup) + { + var baselineSess = runGroup.First(s => s.Role == "baseline"); + var isolatedSess = runGroup.FirstOrDefault(s => s.Role == "with-skill-isolated") + ?? runGroup.FirstOrDefault(s => s.Role == "with-skill"); + if (isolatedSess is null) + continue; + + var pluginSess = runGroup.FirstOrDefault(s => s.Role == "with-skill-plugin"); + var prompt = baselineSess.Prompt ?? isolatedSess.Prompt ?? pluginSess?.Prompt ?? ""; + var scenario = new EvalScenario(scenarioName, prompt, Rubric: storedRubric); + Action? log = verbose ? msg => Console.WriteLine($" [{scenarioName}/{runGroup.Key.RunIndex + 1}] {msg}") : null; + + var baselineMetrics = JsonSerializer.Deserialize(baselineSess.MetricsJson!, SkillValidatorJsonContext.Default.RunMetrics)!; + var isolatedMetrics = JsonSerializer.Deserialize(isolatedSess.MetricsJson!, SkillValidatorJsonContext.Default.RunMetrics)!; + var pluginMetrics = pluginSess?.MetricsJson is not null + ? JsonSerializer.Deserialize(pluginSess.MetricsJson, SkillValidatorJsonContext.Default.RunMetrics) + : null; + + var judgeWorkRoot = CreateJudgeWorkDir("rejudge"); + try + { + var judgeOpts = new JudgeOptions( + effectiveJudgeModel, + verbose, + judgeTimeout, + CreateJudgeWorkDir(judgeWorkRoot, "baseline"), + firstSkillSession.SkillPath); + var baselineJudge = await SafeJudge( + Judge.JudgeRun(scenario, baselineMetrics, judgeOpts, log), + "baseline", + log); + var isolatedJudge = await SafeJudge( + Judge.JudgeRun(scenario, isolatedMetrics, judgeOpts with { WorkDir = CreateJudgeWorkDir(judgeWorkRoot, "isolated") }, log), + "isolated", + log); + var pluginJudge = pluginMetrics is not null + ? await SafeJudge( + Judge.JudgeRun(scenario, pluginMetrics, judgeOpts with { WorkDir = CreateJudgeWorkDir(judgeWorkRoot, "plugin") }, log), + "plugin", + log) + : null; + + sessionDb.SaveJudgeResult(baselineSess.Id, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(isolatedSess.Id, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult)); + if (pluginSess is not null && pluginJudge is not null) + { + sessionDb.SaveJudgeResult(pluginSess.Id, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult)); + } + + var baselineResult = new RunResult(baselineMetrics, baselineJudge); + var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge); + var pluginResult = pluginMetrics is not null && pluginJudge is not null + ? new RunResult(pluginMetrics, pluginJudge) + : null; + + PairwiseJudgeResult? pairwise = null; + bool pairwiseFromPlugin = false; + if (usePairwise) + { + try + { + var pairwiseTarget = pluginResult is not null && pluginResult.JudgeResult.OverallScore < isolatedResult.JudgeResult.OverallScore + ? pluginResult + : isolatedResult; + pairwiseFromPlugin = ReferenceEquals(pairwiseTarget, pluginResult); + pairwise = await PairwiseJudge.Judge( + scenario, + baselineMetrics, + pairwiseTarget.Metrics, + new PairwiseJudgeOptions( + effectiveJudgeModel, + verbose, + judgeTimeout, + CreateJudgeWorkDir(judgeWorkRoot, "pairwise"), + firstSkillSession.SkillPath, + CreateJudgeWorkDir(judgeWorkRoot, "pairwise-skilled")), + log); + sessionDb.SavePairwiseResult(baselineSess.Id, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); + } + catch (Exception error) + { + log?.Invoke($"⚠️ Pairwise judge failed: {error.Message}"); + } + } + + var isolatedActivation = MetricsCollector.ExtractSkillActivation( + isolatedMetrics.Events, + baselineMetrics.ToolCallBreakdown, + skillName); + var pluginActivation = pluginMetrics is not null + ? MetricsCollector.ExtractSkillActivation(pluginMetrics.Events, baselineMetrics.ToolCallBreakdown, skillName) + : null; + + rejudgedRuns.Add(new RejudgedRun( + Baseline: baselineResult, + Isolated: isolatedResult, + Plugin: pluginResult, + Pairwise: pairwise, + PairwiseFromPlugin: pairwiseFromPlugin, + IsolatedActivation: isolatedActivation, + PluginActivation: pluginActivation)); + } + finally + { + TryDeleteDirectory(judgeWorkRoot); + } + } + + if (rejudgedRuns.Count == 0) + continue; + + comparisons.Add(BuildScenarioComparison(scenarioName, rejudgedRuns)); + } + + if (comparisons.Count == 0) + continue; + + var skill = new SkillInfo(skillName, "", firstSkillSession.SkillPath, firstSkillSession.SkillPath, "", null, null); + var verdict = Comparator.ComputeVerdict(skill, comparisons, minImprovement, requireCompletion, confidenceLevel); + Console.WriteLine($"[{skillName}] {(verdict.Passed ? "✅" : "❌")} Score: {verdict.OverallImprovementScore * 100:F1}%"); + verdicts.Add(verdict); + } + + var reporters = new List + { + new(ReporterType.Console), + new(ReporterType.Json), + new(ReporterType.Markdown), + }; + await Reporter.ReportResults(verdicts, reporters, verbose, + firstSession.Model, effectiveJudgeModel, resultsDir, resultsDir); + + await AgentRunner.StopAllClients(); + return verdicts.All(v => v.Passed) ? 0 : 1; + } + + private static string CreateJudgeWorkDir(string prefix) + { + var root = Path.Combine(Path.GetTempPath(), $"sv-{prefix}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + return root; + } + + private static string CreateJudgeWorkDir(string root, string name) + { + var path = Path.Combine(root, name); + Directory.CreateDirectory(path); + return path; + } + + private static void TryDeleteDirectory(string path) + { + try + { + if (Directory.Exists(path)) + Directory.Delete(path, true); + } + catch + { + } + } + + private static ScenarioComparison BuildScenarioComparison(string scenarioName, List runs) + { + var baselineRuns = runs.Select(r => r.Baseline).ToList(); + var isolatedRuns = runs.Select(r => r.Isolated).ToList(); + var avgBaseline = AverageResults(baselineRuns); + var avgIsolated = AverageResults(isolatedRuns); + var bestPairwise = runs.Select(r => r.Pairwise).FirstOrDefault(p => p?.PositionSwapConsistent == true) + ?? runs.Select(r => r.Pairwise).FirstOrDefault(); + + if (runs.Any(r => r.Plugin is not null)) + { + var pluginRuns = runs.Where(r => r.Plugin is not null).Select(r => r.Plugin!).ToList(); + var perRunIsolatedScores = new List(); + var perRunPluginScores = new List(); + + foreach (var run in runs) + { + var isoComp = Comparator.CompareScenario(scenarioName, run.Baseline, run.Isolated, + run.PairwiseFromPlugin ? null : run.Pairwise); + var pluginComp = run.Plugin is not null + ? Comparator.CompareScenario(scenarioName, run.Baseline, run.Plugin, + run.PairwiseFromPlugin ? run.Pairwise : null) + : isoComp; + perRunIsolatedScores.Add(isoComp.ImprovementScore); + perRunPluginScores.Add(pluginComp.ImprovementScore); + } + + var perRunScores = perRunIsolatedScores + .Zip(perRunPluginScores, (iso, plugin) => Math.Min(iso, plugin)) + .ToList(); + var avgPlugin = AverageResults(pluginRuns); + int bestPairwiseIdx = runs.FindIndex(r => r.Pairwise?.PositionSwapConsistent == true); + if (bestPairwiseIdx < 0) + bestPairwiseIdx = runs.FindIndex(r => r.Pairwise is not null); + bool pairwiseFromPlugin = bestPairwiseIdx >= 0 && runs[bestPairwiseIdx].PairwiseFromPlugin; + + var isoComparison = Comparator.CompareScenario(scenarioName, avgBaseline, avgIsolated, + pairwiseFromPlugin ? null : bestPairwise); + var pluginComparison = Comparator.CompareScenario(scenarioName, avgBaseline, avgPlugin, + pairwiseFromPlugin ? bestPairwise : null); + + var comparison = new ScenarioComparison + { + ScenarioName = scenarioName, + Baseline = avgBaseline, + SkilledIsolated = avgIsolated, + SkilledPlugin = avgPlugin, + ImprovementScore = Math.Min(isoComparison.ImprovementScore, pluginComparison.ImprovementScore), + IsolatedImprovementScore = isoComparison.ImprovementScore, + PluginImprovementScore = pluginComparison.ImprovementScore, + Breakdown = isoComparison.ImprovementScore <= pluginComparison.ImprovementScore + ? isoComparison.Breakdown + : pluginComparison.Breakdown, + IsolatedBreakdown = isoComparison.Breakdown, + PluginBreakdown = pluginComparison.Breakdown, + PairwiseResult = bestPairwise, + PerRunScores = perRunScores, + SkillActivationIsolated = new SkillActivationInfo( + Activated: runs.Any(r => r.IsolatedActivation.Activated), + DetectedSkills: runs.SelectMany(r => r.IsolatedActivation.DetectedSkills).Distinct().ToList(), + ExtraTools: runs.SelectMany(r => r.IsolatedActivation.ExtraTools).Distinct().ToList(), + SkillEventCount: runs.Sum(r => r.IsolatedActivation.SkillEventCount)), + SkillActivationPlugin = new SkillActivationInfo( + Activated: runs.Any(r => r.PluginActivation?.Activated == true), + DetectedSkills: runs.SelectMany(r => r.PluginActivation?.DetectedSkills ?? []).Distinct().ToList(), + ExtraTools: runs.SelectMany(r => r.PluginActivation?.ExtraTools ?? []).Distinct().ToList(), + SkillEventCount: runs.Sum(r => r.PluginActivation?.SkillEventCount ?? 0)), + TimedOut = runs.Any(r => r.Baseline.Metrics.TimedOut || r.Isolated.Metrics.TimedOut || r.Plugin?.Metrics.TimedOut == true), + }; + return comparison; + } + + var comparisonNoPlugin = Comparator.CompareScenario(scenarioName, avgBaseline, avgIsolated, bestPairwise); + comparisonNoPlugin.PerRunScores = runs.Select(r => Comparator.CompareScenario(scenarioName, r.Baseline, r.Isolated, r.Pairwise).ImprovementScore).ToList(); + comparisonNoPlugin.SkillActivationIsolated = new SkillActivationInfo( + Activated: runs.Any(r => r.IsolatedActivation.Activated), + DetectedSkills: runs.SelectMany(r => r.IsolatedActivation.DetectedSkills).Distinct().ToList(), + ExtraTools: runs.SelectMany(r => r.IsolatedActivation.ExtraTools).Distinct().ToList(), + SkillEventCount: runs.Sum(r => r.IsolatedActivation.SkillEventCount)); + comparisonNoPlugin.TimedOut = runs.Any(r => r.Baseline.Metrics.TimedOut || r.Isolated.Metrics.TimedOut); + return comparisonNoPlugin; + } + + private static string[]? GetStoredRubric(string skillName, string scenarioName, IEnumerable sessions) + { + var rubricJson = sessions + .Select(s => s.RubricJson) + .FirstOrDefault(r => !string.IsNullOrWhiteSpace(r)); + if (rubricJson is null) + { + Console.WriteLine($"[{skillName}] ⚠️ Scenario '{scenarioName}' has no persisted rubric in sessions.db; falling back to the default judging rubric."); + return null; + } + + try + { + return JsonSerializer.Deserialize(rubricJson, SkillValidatorJsonContext.Default.StringArray) ?? []; + } + catch (JsonException error) + { + Console.WriteLine($"[{skillName}] ⚠️ Scenario '{scenarioName}' has an unreadable persisted rubric ({error.Message}); falling back to the default judging rubric."); + return null; + } + } + + private static async Task SafeJudge(Task task, string label, Action? log) + { + try + { + return await task; + } + catch (Exception error) + { + log?.Invoke($"⚠️ Judge ({label}) failed, using fallback scores: {error.Message}"); + return new JudgeResult([], 3, $"Judge failed: {error.Message}"); + } + } + + private static RunResult AverageResults(List runs) + { + if (runs.Count == 1) + return runs[0]; + + static double Avg(IEnumerable nums) => nums.Average(); + static int AvgRound(IEnumerable nums) => (int)Math.Round(nums.Average()); + + var avgMetrics = new RunMetrics + { + TokenEstimate = AvgRound(runs.Select(r => r.Metrics.TokenEstimate)), + ToolCallCount = AvgRound(runs.Select(r => r.Metrics.ToolCallCount)), + ToolCallBreakdown = runs[0].Metrics.ToolCallBreakdown, + TurnCount = AvgRound(runs.Select(r => r.Metrics.TurnCount)), + WallTimeMs = (long)Math.Round(runs.Average(r => r.Metrics.WallTimeMs)), + ErrorCount = AvgRound(runs.Select(r => r.Metrics.ErrorCount)), + TimedOut = runs.Any(r => r.Metrics.TimedOut), + AssertionResults = runs[^1].Metrics.AssertionResults, + TaskCompleted = runs.Any(r => r.Metrics.TaskCompleted), + AgentOutput = runs[^1].Metrics.AgentOutput, + Events = runs[^1].Metrics.Events, + WorkDir = runs[^1].Metrics.WorkDir, + }; + + var avgJudge = new JudgeResult( + runs[0].JudgeResult.RubricScores.Select((score, i) => new RubricScore( + score.Criterion, + Math.Round(Avg(runs.Select(r => i < r.JudgeResult.RubricScores.Count ? r.JudgeResult.RubricScores[i].Score : 3)) * 10) / 10, + score.Reasoning)).ToList(), + Math.Round(Avg(runs.Select(r => r.JudgeResult.OverallScore)) * 10) / 10, + runs[^1].JudgeResult.OverallReasoning); + + return new RunResult(avgMetrics, avgJudge); + } + + private sealed record RejudgedRun( + RunResult Baseline, + RunResult Isolated, + RunResult? Plugin, + PairwiseJudgeResult? Pairwise, + bool PairwiseFromPlugin, + SkillActivationInfo IsolatedActivation, + SkillActivationInfo? PluginActivation); +} diff --git a/eng/skill-validator/src/Commands/ValidateCommand.cs b/eng/skill-validator/src/Commands/ValidateCommand.cs index dcd4b87f14..f05c5ceb30 100644 --- a/eng/skill-validator/src/Commands/ValidateCommand.cs +++ b/eng/skill-validator/src/Commands/ValidateCommand.cs @@ -19,7 +19,8 @@ public static RootCommand Create() var verboseOpt = new Option("--verbose") { Description = "Show detailed per-scenario breakdowns" }; var modelOpt = new Option("--model") { Description = "Model to use for agent runs", DefaultValueFactory = _ => "claude-opus-4.6" }; var judgeModelOpt = new Option("--judge-model") { Description = "Model to use for judging (defaults to --model)" }; - var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }; + var judgeModeOpt = new Option("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" } + .AcceptOnlyFromAmong("pairwise", "independent", "both"); var runsOpt = new Option("--runs") { Description = "Number of runs per scenario for averaging", DefaultValueFactory = _ => 5 }; var parallelSkillsOpt = new Option("--parallel-skills") { Description = "Max concurrent skills to evaluate", DefaultValueFactory = _ => 1 }; var parallelScenariosOpt = new Option("--parallel-scenarios") { Description = "Max concurrent scenarios per skill", DefaultValueFactory = _ => 1 }; @@ -31,6 +32,7 @@ public static RootCommand Create() var reporterOpt = new Option("--reporter") { Description = "Reporter (console, json, junit, markdown). Can be repeated.", AllowMultipleArgumentsPerToken = true }; var noOverfittingCheckOpt = new Option("--no-overfitting-check") { Description = "Disable LLM-based overfitting analysis (on by default)" }; var overfittingFixOpt = new Option("--overfitting-fix") { Description = "Generate a fixed eval.yaml with improved rubric items/assertions" }; + var keepSessionsOpt = new Option("--keep-sessions") { Description = "Preserve agent session data in the results directory for later rejudging" }; var noiseSkillsDirOpt = new Option("--noise-skills-dir") { Description = "Directory containing skills to load as noise. Enables the noise test: re-runs scenarios with all noise skills loaded and measures degradation." }; var noiseMaxDegradationOpt = new Option("--noise-max-degradation") { Description = "Maximum acceptable average quality degradation (0-1) in noise test (only positive degradations count)", DefaultValueFactory = _ => 0.2 }; var noiseMaxScenarioDegradationOpt = new Option("--noise-max-scenario-degradation") { Description = "Maximum acceptable quality degradation (0-1) for any single noise-test scenario", DefaultValueFactory = _ => 0.4 }; @@ -57,6 +59,7 @@ public static RootCommand Create() reporterOpt, noOverfittingCheckOpt, overfittingFixOpt, + keepSessionsOpt, noiseSkillsDirOpt, noiseMaxDegradationOpt, noiseMaxScenarioDegradationOpt, @@ -105,6 +108,7 @@ public static RootCommand Create() TestsDir = parseResult.GetValue(testsDirOpt), OverfittingCheck = !parseResult.GetValue(noOverfittingCheckOpt), OverfittingFix = parseResult.GetValue(overfittingFixOpt), + KeepSessions = parseResult.GetValue(keepSessionsOpt), NoiseSkillsDir = parseResult.GetValue(noiseSkillsDirOpt), NoiseDegradationLimit = parseResult.GetValue(noiseMaxDegradationOpt), NoiseMaxScenarioDegradation = parseResult.GetValue(noiseMaxScenarioDegradationOpt), @@ -320,6 +324,25 @@ public static async Task Run(ValidatorConfig config) Console.WriteLine($"\x1b[33m⚠ Running with {config.Runs} run(s). For statistically significant results, use --runs 5 or higher.\x1b[0m"); bool usePairwise = config.JudgeMode is JudgeMode.Pairwise or JudgeMode.Both; + bool effectiveKeepSessions = config.KeepSessions && config.ResultsDir is not null; + + string? sessionsDir = null; + SessionDatabase? sessionDb = null; + string? timestampedResultsDir = null; + if (effectiveKeepSessions) + { + timestampedResultsDir = Path.Combine(config.ResultsDir!, Reporter.FormatTimestamp(DateTime.Now)); + Directory.CreateDirectory(timestampedResultsDir); + sessionsDir = Path.Combine(timestampedResultsDir, "sessions"); + Directory.CreateDirectory(sessionsDir); + sessionDb = new SessionDatabase(Path.Combine(timestampedResultsDir, "sessions.db")); + sessionDb.SetSchemaInfo("judge_model", config.JudgeModel); + Console.WriteLine($"Session persistence enabled: {timestampedResultsDir}"); + } + else if (config.KeepSessions) + { + Console.WriteLine("\x1b[33m⚠ --keep-sessions was set without --results-dir; sessions will not be persisted.\x1b[0m"); + } using var spinner = new Spinner(); using var skillLimit = new ConcurrencyLimiter(config.ParallelSkills); @@ -327,7 +350,7 @@ public static async Task Run(ValidatorConfig config) // Evaluate skills spinner.Start($"Evaluating {allSkills.Count} skill(s)..."); var skillTasks = allSkills.Select(skill => - skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills))); + skillLimit.RunAsync(() => EvaluateSkill(skill, config, usePairwise, spinner, noiseSkills, sessionsDir, sessionDb))); var settled = await Task.WhenAll(skillTasks.Select(async t => { try { return (Result: await t, Error: (Exception?)null); } @@ -350,7 +373,7 @@ public static async Task Run(ValidatorConfig config) } await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose, - config.Model, config.JudgeModel, config.ResultsDir, + config.Model, config.JudgeModel, config.ResultsDir, timestampedResultsDir, rejectedCount: rejectionMessages.Count); if (rejectionMessages.Count > 0) @@ -362,7 +385,8 @@ await Reporter.ReportResults(verdicts, config.Reporters, config.Verbose, } await AgentRunner.StopAllClients(); - await AgentRunner.CleanupWorkDirs(); + await AgentRunner.CleanupWorkDirs(effectiveKeepSessions); + sessionDb?.Dispose(); // Always fail on execution errors, even in --verdict-warn-only mode if (rejectionMessages.Count > 0) return 1; @@ -432,7 +456,9 @@ internal static List CheckAggregateDescriptionLimits(IReadOnlyList noiseSkills) + IReadOnlyList noiseSkills, + string? sessionsDir, + SessionDatabase? sessionDb) { var prefix = $"[{skill.Name}]"; var log = (string msg) => spinner.Log($"{prefix} {msg}"); @@ -501,11 +527,12 @@ internal static List CheckAggregateDescriptionLimits(IReadOnlyList - scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner))); + scenarioLimit.RunAsync(() => ExecuteScenario(scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb, skillSha))); var comparisons = (await Task.WhenAll(scenarioTasks)).ToList(); // Await overfitting result (non-fatal — never blocks an otherwise-successful evaluation) @@ -587,7 +614,10 @@ private static async Task ExecuteScenario( ValidatorConfig config, bool usePairwise, bool singleScenario, - Spinner spinner) + Spinner spinner, + string? sessionsDir, + SessionDatabase? sessionDb, + string? skillSha) { var tag = singleScenario ? $"[{skill.Name}]" : $"[{skill.Name}/{scenario.Name}]"; var scenarioLog = (string msg) => spinner.Log($"{tag} {msg}"); @@ -597,7 +627,7 @@ private static async Task ExecuteScenario( scenarioLog("📋 Starting scenario"); var runTasks = Enumerable.Range(0, config.Runs).Select(i => - runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner))); + runLimit.RunAsync(() => ExecuteRun(i, scenario, skill, config, usePairwise, singleScenario, spinner, sessionsDir, sessionDb, skillSha))); var runResults = await Task.WhenAll(runTasks); scenarioLog($"✓ All {config.Runs} run(s) complete"); @@ -717,7 +747,10 @@ private static async Task ExecuteRun( ValidatorConfig config, bool usePairwise, bool singleScenario, - Spinner spinner) + Spinner spinner, + string? sessionsDir, + SessionDatabase? sessionDb, + string? skillSha) { var runTag = config.Runs > 1 ? (singleScenario ? $"[{skill.Name}/{runIndex + 1}]" : $"[{skill.Name}/{scenario.Name}/{runIndex + 1}]") @@ -728,21 +761,46 @@ private static async Task ExecuteRun( runLog("running agents..."); var pluginRoot = SkillDiscovery.FindPluginRoot(skill.Path); + var baselineSessionId = Guid.NewGuid().ToString("N"); + var isolatedSessionId = Guid.NewGuid().ToString("N"); + var pluginSessionId = Guid.NewGuid().ToString("N"); + + var baselineConfigDir = sessionsDir is not null ? Path.Combine("sessions", baselineSessionId) : null; + var isolatedConfigDir = sessionsDir is not null ? Path.Combine("sessions", isolatedSessionId) : null; + var pluginConfigDir = sessionsDir is not null ? Path.Combine("sessions", pluginSessionId) : null; + var rubricJson = JsonSerializer.Serialize(scenario.Rubric?.ToArray() ?? [], SkillValidatorJsonContext.Default.StringArray); + + sessionDb?.RegisterSession(baselineSessionId, skill.Name, skill.Path, scenario.Name, runIndex, + "baseline", config.Model, baselineConfigDir, null, scenario.Prompt, skillSha, rubricJson); + sessionDb?.RegisterSession(isolatedSessionId, skill.Name, skill.Path, scenario.Name, runIndex, + "with-skill-isolated", config.Model, isolatedConfigDir, null, scenario.Prompt, skillSha, rubricJson); + sessionDb?.RegisterSession(pluginSessionId, skill.Name, skill.Path, scenario.Name, runIndex, + "with-skill-plugin", config.Model, pluginConfigDir, null, scenario.Prompt, skillSha, rubricJson); var agentTasks = await Task.WhenAll( // 1. Baseline: no plugin, no skills — vanilla agent AgentRunner.RunAgent(new RunOptions(scenario, null, skill.EvalPath, config.Model, config.Verbose, - PluginRoot: null, Log: runLog)), + PluginRoot: null, Log: runLog, SessionsDir: sessionsDir, SessionId: baselineSessionId)), // 2. Skilled-isolated: single skill only (current behavior) AgentRunner.RunAgent(new RunOptions(scenario, skill, skill.EvalPath, config.Model, config.Verbose, - PluginRoot: null, Log: runLog)), + PluginRoot: null, Log: runLog, SessionsDir: sessionsDir, SessionId: isolatedSessionId)), // 3. Skilled-plugin: load entire plugin from plugin root directory AgentRunner.RunAgent(new RunOptions(scenario, skill, skill.EvalPath, config.Model, config.Verbose, - PluginRoot: pluginRoot, Log: runLog))); + PluginRoot: pluginRoot, Log: runLog, SessionsDir: sessionsDir, SessionId: pluginSessionId))); var baselineMetrics = agentTasks[0]; var isolatedMetrics = agentTasks[1]; var pluginMetrics = agentTasks[2]; + if (sessionDb is not null) + { + var baselineStatus = baselineMetrics.TimedOut ? "timed_out" : "completed"; + var isolatedStatus = isolatedMetrics.TimedOut ? "timed_out" : "completed"; + var pluginStatus = pluginMetrics.TimedOut ? "timed_out" : "completed"; + sessionDb.CompleteSession(baselineSessionId, baselineStatus, JsonSerializer.Serialize(baselineMetrics, SkillValidatorJsonContext.Default.RunMetrics)); + sessionDb.CompleteSession(isolatedSessionId, isolatedStatus, JsonSerializer.Serialize(isolatedMetrics, SkillValidatorJsonContext.Default.RunMetrics)); + sessionDb.CompleteSession(pluginSessionId, pluginStatus, JsonSerializer.Serialize(pluginMetrics, SkillValidatorJsonContext.Default.RunMetrics)); + } + // Evaluate assertions on all three runs if (scenario.Assertions is { Count: > 0 }) { @@ -786,6 +844,13 @@ private static async Task ExecuteRun( var isolatedJudge = await SafeJudge(isolatedJudgeTask, "isolated", runLog); var pluginJudge = await SafeJudge(pluginJudgeTask, "plugin", runLog); + if (sessionDb is not null) + { + sessionDb.SaveJudgeResult(baselineSessionId, JsonSerializer.Serialize(baselineJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(isolatedSessionId, JsonSerializer.Serialize(isolatedJudge, SkillValidatorJsonContext.Default.JudgeResult)); + sessionDb.SaveJudgeResult(pluginSessionId, JsonSerializer.Serialize(pluginJudge, SkillValidatorJsonContext.Default.JudgeResult)); + } + var baselineResult = new RunResult(baselineMetrics, baselineJudge); var isolatedResult = new RunResult(isolatedMetrics, isolatedJudge); var pluginResult = new RunResult(pluginMetrics, pluginJudge); @@ -805,6 +870,10 @@ private static async Task ExecuteRun( scenario, baselineMetrics, worseSkilled, new PairwiseJudgeOptions(config.JudgeModel, config.Verbose, config.JudgeTimeout, baselineMetrics.WorkDir, skill.Path, worseSkilled.WorkDir), runLog); + if (sessionDb is not null && pairwise is not null) + { + sessionDb.SavePairwiseResult(baselineSessionId, JsonSerializer.Serialize(pairwise, SkillValidatorJsonContext.Default.PairwiseJudgeResult)); + } } catch (Exception error) { diff --git a/eng/skill-validator/src/Models/Models.cs b/eng/skill-validator/src/Models/Models.cs index 2ec36ce12e..c507d377f3 100644 --- a/eng/skill-validator/src/Models/Models.cs +++ b/eng/skill-validator/src/Models/Models.cs @@ -376,6 +376,7 @@ public sealed record ValidatorConfig public string? TestsDir { get; init; } public bool OverfittingCheck { get; init; } = true; public bool OverfittingFix { get; init; } + public bool KeepSessions { get; init; } public string? NoiseSkillsDir { get; init; } public double NoiseDegradationLimit { get; init; } = 0.2; public double NoiseMaxScenarioDegradation { get; init; } = 0.4; diff --git a/eng/skill-validator/src/Program.cs b/eng/skill-validator/src/Program.cs index 042ea61ce9..90b7909ca3 100644 --- a/eng/skill-validator/src/Program.cs +++ b/eng/skill-validator/src/Program.cs @@ -3,6 +3,7 @@ var rootCommand = ValidateCommand.Create(); rootCommand.Add(ConsolidateCommand.Create()); +rootCommand.Add(RejudgeCommand.Create()); var parseResult = rootCommand.Parse(args); return await parseResult.InvokeAsync(); diff --git a/eng/skill-validator/src/Services/AgentRunner.cs b/eng/skill-validator/src/Services/AgentRunner.cs index 05686f7f25..ea5cf92c50 100644 --- a/eng/skill-validator/src/Services/AgentRunner.cs +++ b/eng/skill-validator/src/Services/AgentRunner.cs @@ -16,13 +16,16 @@ public sealed record RunOptions( bool Verbose, string? PluginRoot = null, Action? Log = null, - IReadOnlyList? AdditionalSkills = null); + IReadOnlyList? AdditionalSkills = null, + string? SessionsDir = null, + string? SessionId = null); public static class AgentRunner { private static readonly ConcurrentDictionary _pluginClients = new(StringComparer.OrdinalIgnoreCase); private static readonly SemaphoreSlim _clientLock = new(1, 1); private static readonly ConcurrentBag _workDirs = []; + private static readonly ConcurrentBag _configDirs = []; private static string? _capturedGitHubToken; private static bool _tokenCaptured; @@ -102,11 +105,16 @@ public static async Task StopAllClients() public static Task StopSharedClient() => StopAllClients(); /// Remove all temporary working directories created during runs. - public static Task CleanupWorkDirs() + public static Task CleanupWorkDirs(bool keepSessions = false) { var dirs = _workDirs.ToArray(); _workDirs.Clear(); - return Task.WhenAll(dirs.Select(dir => + + var configDirsToClean = keepSessions ? [] : _configDirs.ToArray(); + _configDirs.Clear(); + + var allDirs = dirs.Concat(configDirsToClean); + return Task.WhenAll(allDirs.Select(dir => { try { Directory.Delete(dir, true); } catch { } return Task.CompletedTask; @@ -206,16 +214,29 @@ internal static SessionConfig BuildSessionConfig( IReadOnlyDictionary? mcpServers = null, IReadOnlyList? additionalSkills = null, Action? log = null, - bool verbose = false) + bool verbose = false, + string? sessionsDir = null, + string? sessionId = null) { // The SDK expects SkillDirectories entries to be parent directories that // it scans for child folders containing SKILL.md. var skillPath = skill is not null ? Path.GetDirectoryName(skill.Path) : null; - // Create a unique temporary config directory for this session to not share any data - var configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}"); - Directory.CreateDirectory(configDir); - _workDirs.Add(configDir); + string configDir; + if (sessionsDir is not null) + { + // Persistent session dir — use sessionId as folder name for DB linkage + var dirName = sessionId ?? Guid.NewGuid().ToString("N"); + configDir = Path.Combine(sessionsDir, dirName); + Directory.CreateDirectory(configDir); + _configDirs.Add(configDir); + } + else + { + configDir = Path.Combine(Path.GetTempPath(), $"sv-cfg-{Guid.NewGuid():N}"); + Directory.CreateDirectory(configDir); + _configDirs.Add(configDir); + } if (verbose) log?.Invoke($" 📂 Config dir: {configDir} ({(skill is not null ? "skilled" : "baseline")})"); @@ -406,7 +427,7 @@ private static async Task RunAgentCore(RunOptions options, Cancellat var client = await GetPluginClient(options.PluginRoot, options.Verbose); await using var session = await client.CreateSessionAsync( - BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills, options.Log, options.Verbose)); + BuildSessionConfig(options.Skill, options.PluginRoot, options.Model, workDir, options.Skill?.McpServers, options.AdditionalSkills, options.Log, options.Verbose, options.SessionsDir, options.SessionId)); var done = new TaskCompletionSource(); var effectiveTimeout = options.Scenario.Timeout; diff --git a/eng/skill-validator/src/Services/Reporter.cs b/eng/skill-validator/src/Services/Reporter.cs index f8ab8f95db..a69edbb4fb 100644 --- a/eng/skill-validator/src/Services/Reporter.cs +++ b/eng/skill-validator/src/Services/Reporter.cs @@ -14,13 +14,15 @@ public static async Task ReportResults( string? model = null, string? judgeModel = null, string? resultsDir = null, + string? timestampedResultsDir = null, int rejectedCount = 0) { bool needsResultsDir = reporters.Any(r => r.Type is ReporterType.Json or ReporterType.Junit or ReporterType.Markdown); - string? effectiveResultsDir = resultsDir is not null && needsResultsDir - ? Path.Combine(resultsDir, FormatTimestamp(DateTime.Now)) - : null; + string? effectiveResultsDir = timestampedResultsDir + ?? (resultsDir is not null && needsResultsDir + ? Path.Combine(resultsDir, FormatTimestamp(DateTime.Now)) + : null); if (effectiveResultsDir is not null) Directory.CreateDirectory(effectiveResultsDir); diff --git a/eng/skill-validator/src/Services/SessionDatabase.cs b/eng/skill-validator/src/Services/SessionDatabase.cs new file mode 100644 index 0000000000..ef1950af38 --- /dev/null +++ b/eng/skill-validator/src/Services/SessionDatabase.cs @@ -0,0 +1,321 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using Microsoft.Data.Sqlite; + +namespace SkillValidator.Services; + +/// +/// Tracks eval sessions in a SQLite database for crash recovery and rejudging. +/// Thread-safe for concurrent scenario/run execution. +/// +public sealed class SessionDatabase : IDisposable +{ + private readonly SqliteConnection _connection; + private readonly SemaphoreSlim _writeLock = new(1, 1); + + public SessionDatabase(string dbPath) + { + Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); + _connection = new SqliteConnection($"Data Source={dbPath}"); + _connection.Open(); + Initialize(); + } + + private void Initialize() + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA busy_timeout=5000; + PRAGMA foreign_keys=ON; + + CREATE TABLE IF NOT EXISTS schema_info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT OR IGNORE INTO schema_info (key, value) VALUES ('type', 'skill-validator'); + INSERT OR IGNORE INTO schema_info (key, value) VALUES ('version', '2'); + + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + skill_name TEXT NOT NULL, + skill_path TEXT NOT NULL, + scenario_name TEXT NOT NULL, + run_index INTEGER NOT NULL, + role TEXT NOT NULL, + model TEXT NOT NULL, + config_dir TEXT, + work_dir TEXT, + prompt TEXT, + skill_sha TEXT, + status TEXT NOT NULL DEFAULT 'running', + started_at TEXT NOT NULL, + completed_at TEXT, + rubric TEXT + ); + + CREATE TABLE IF NOT EXISTS run_results ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id), + metrics_json TEXT NOT NULL, + judge_json TEXT, + pairwise_json TEXT + ); + """; + cmd.ExecuteNonQuery(); + EnsureSessionsRubricColumn(); + SetSchemaInfo("version", "2"); + } + + private void EnsureSessionsRubricColumn() + { + if (HasColumn("sessions", "rubric")) + return; + + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE sessions ADD COLUMN rubric TEXT"; + cmd.ExecuteNonQuery(); + } + + private bool HasColumn(string tableName, string columnName) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = $"PRAGMA table_info({tableName})"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase)) + return true; + } + + return false; + } + + /// + /// Computes a SHA-256 hash over all files in a directory, sorted by relative path. + /// Returns the first 12 hex characters for a short, collision-resistant identifier. + /// + public static string ComputeDirectorySha(string dirPath) + { + using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var files = Directory.GetFiles(dirPath, "*", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(dirPath, f).Replace('\\', '/')) + .OrderBy(f => f, StringComparer.Ordinal) + .ToList(); + + foreach (var relPath in files) + { + AppendLengthPrefixedData(sha, System.Text.Encoding.UTF8.GetBytes(relPath)); + AppendLengthPrefixedData(sha, File.ReadAllBytes(Path.Combine(dirPath, relPath))); + } + + var hash = sha.GetHashAndReset(); + return Convert.ToHexString(hash)[..12].ToLowerInvariant(); + } + + private static void AppendLengthPrefixedData(IncrementalHash sha, byte[] data) + { + Span lengthPrefix = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(lengthPrefix, data.Length); + sha.AppendData(lengthPrefix); + sha.AppendData(data); + } + + public void RegisterSession(string sessionId, string skillName, string skillPath, + string scenarioName, int runIndex, string role, string model, + string? configDir, string? workDir, string? prompt = null, string? skillSha = null, string? rubric = null) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, config_dir, work_dir, prompt, skill_sha, rubric, status, started_at) + VALUES ($id, $skill_name, $skill_path, $scenario_name, $run_index, $role, $model, $config_dir, $work_dir, $prompt, $skill_sha, $rubric, 'running', $started_at) + """; + cmd.Parameters.AddWithValue("$id", sessionId); + cmd.Parameters.AddWithValue("$skill_name", skillName); + cmd.Parameters.AddWithValue("$skill_path", skillPath); + cmd.Parameters.AddWithValue("$scenario_name", scenarioName); + cmd.Parameters.AddWithValue("$run_index", runIndex); + cmd.Parameters.AddWithValue("$role", role); + cmd.Parameters.AddWithValue("$model", model); + cmd.Parameters.AddWithValue("$config_dir", (object?)configDir ?? DBNull.Value); + cmd.Parameters.AddWithValue("$work_dir", (object?)workDir ?? DBNull.Value); + cmd.Parameters.AddWithValue("$prompt", (object?)prompt ?? DBNull.Value); + cmd.Parameters.AddWithValue("$skill_sha", (object?)skillSha ?? DBNull.Value); + cmd.Parameters.AddWithValue("$rubric", (object?)rubric ?? DBNull.Value); + cmd.Parameters.AddWithValue("$started_at", DateTimeOffset.UtcNow.ToString("o")); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + public void CompleteSession(string sessionId, string status, string metricsJson) + { + _writeLock.Wait(); + try + { + using var transaction = _connection.BeginTransaction(); + + using (var cmd = _connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = "UPDATE sessions SET status = $status, completed_at = $completed_at WHERE id = $id"; + cmd.Parameters.AddWithValue("$id", sessionId); + cmd.Parameters.AddWithValue("$status", status); + cmd.Parameters.AddWithValue("$completed_at", DateTimeOffset.UtcNow.ToString("o")); + cmd.ExecuteNonQuery(); + } + + using (var cmd = _connection.CreateCommand()) + { + cmd.Transaction = transaction; + cmd.CommandText = """ + INSERT OR REPLACE INTO run_results (session_id, metrics_json) + VALUES ($session_id, $metrics_json) + """; + cmd.Parameters.AddWithValue("$session_id", sessionId); + cmd.Parameters.AddWithValue("$metrics_json", metricsJson); + cmd.ExecuteNonQuery(); + } + + transaction.Commit(); + } + finally { _writeLock.Release(); } + } + + public void SaveJudgeResult(string sessionId, string judgeJson) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "UPDATE run_results SET judge_json = $judge_json WHERE session_id = $session_id"; + cmd.Parameters.AddWithValue("$session_id", sessionId); + cmd.Parameters.AddWithValue("$judge_json", judgeJson); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + public void SavePairwiseResult(string baselineSessionId, string pairwiseJson) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "UPDATE run_results SET pairwise_json = $pairwise_json WHERE session_id = $session_id"; + cmd.Parameters.AddWithValue("$session_id", baselineSessionId); + cmd.Parameters.AddWithValue("$pairwise_json", pairwiseJson); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + public void SetSchemaInfo(string key, string value) + { + _writeLock.Wait(); + try + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "INSERT INTO schema_info (key, value) VALUES ($key, $value) ON CONFLICT(key) DO UPDATE SET value = excluded.value"; + cmd.Parameters.AddWithValue("$key", key); + cmd.Parameters.AddWithValue("$value", value); + cmd.ExecuteNonQuery(); + } + finally { _writeLock.Release(); } + } + + /// + /// Returns all completed sessions as a flat list ordered by skill, scenario, run index, and role. + /// + public List GetCompletedSessions() + { + _writeLock.Wait(); + try + { + return GetSessions("WHERE s.status IN ('completed', 'timed_out')"); + } + finally { _writeLock.Release(); } + } + + /// + /// Returns schema metadata (type, version) for DB detection by external tools. + /// + public Dictionary GetSchemaInfo() + { + _writeLock.Wait(); + try + { + var result = new Dictionary(); + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT key, value FROM schema_info"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + result[reader.GetString(0)] = reader.GetString(1); + return result; + } + finally { _writeLock.Release(); } + } + + private List GetSessions(string whereClause) + { + var results = new List(); + using var cmd = _connection.CreateCommand(); + cmd.CommandText = $""" + SELECT s.id, s.skill_name, s.skill_path, s.scenario_name, s.run_index, s.role, s.model, + s.config_dir, s.work_dir, s.prompt, s.skill_sha, s.rubric, s.status, + r.metrics_json, r.judge_json, r.pairwise_json + FROM sessions s + LEFT JOIN run_results r ON s.id = r.session_id + {whereClause} + ORDER BY s.skill_name, s.scenario_name, s.run_index, s.role + """; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + results.Add(new SessionRecord( + Id: reader.GetString(0), + SkillName: reader.GetString(1), + SkillPath: reader.GetString(2), + ScenarioName: reader.GetString(3), + RunIndex: reader.GetInt32(4), + Role: reader.GetString(5), + Model: reader.GetString(6), + ConfigDir: reader.IsDBNull(7) ? null : reader.GetString(7), + WorkDir: reader.IsDBNull(8) ? null : reader.GetString(8), + Prompt: reader.IsDBNull(9) ? null : reader.GetString(9), + SkillSha: reader.IsDBNull(10) ? null : reader.GetString(10), + RubricJson: reader.IsDBNull(11) ? null : reader.GetString(11), + Status: reader.GetString(12), + MetricsJson: reader.IsDBNull(13) ? null : reader.GetString(13), + JudgeJson: reader.IsDBNull(14) ? null : reader.GetString(14), + PairwiseJson: reader.IsDBNull(15) ? null : reader.GetString(15))); + } + return results; + } + + public void Dispose() + { + _connection.Dispose(); + _writeLock.Dispose(); + } +} + +public sealed record SessionRecord( + string Id, + string SkillName, + string SkillPath, + string ScenarioName, + int RunIndex, + string Role, + string Model, + string? ConfigDir, + string? WorkDir, + string? Prompt, + string? SkillSha, + string? RubricJson, + string Status, + string? MetricsJson, + string? JudgeJson, + string? PairwiseJson); diff --git a/eng/skill-validator/src/SkillValidator.csproj b/eng/skill-validator/src/SkillValidator.csproj index 8e449a0913..657278f702 100644 --- a/eng/skill-validator/src/SkillValidator.csproj +++ b/eng/skill-validator/src/SkillValidator.csproj @@ -31,6 +31,7 @@ + diff --git a/eng/skill-validator/src/SkillValidatorJsonContext.cs b/eng/skill-validator/src/SkillValidatorJsonContext.cs index 8fd69c82a0..03d5719632 100644 --- a/eng/skill-validator/src/SkillValidatorJsonContext.cs +++ b/eng/skill-validator/src/SkillValidatorJsonContext.cs @@ -38,6 +38,7 @@ namespace SkillValidator; [JsonSerializable(typeof(JsonElement))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(Dictionary))] +[JsonSerializable(typeof(string[]))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(IReadOnlyList))] internal partial class SkillValidatorJsonContext : JsonSerializerContext; diff --git a/eng/skill-validator/tests/SessionDatabaseTests.cs b/eng/skill-validator/tests/SessionDatabaseTests.cs new file mode 100644 index 0000000000..c9ca84c676 --- /dev/null +++ b/eng/skill-validator/tests/SessionDatabaseTests.cs @@ -0,0 +1,349 @@ +using System.Text.Json; +using Microsoft.Data.Sqlite; +using SkillValidator.Services; + +namespace SkillValidator.Tests; + +public class SessionDatabaseTests : IDisposable +{ + private readonly string _dbPath; + private readonly SessionDatabase _db; + + public SessionDatabaseTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"test-sessions-{Guid.NewGuid()}.db"); + _db = new SessionDatabase(_dbPath); + } + + public void Dispose() + { + _db.Dispose(); + // Clear SQLite connection pool so file handles are fully released + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + TryDelete(_dbPath); + TryDelete(_dbPath + "-wal"); + TryDelete(_dbPath + "-shm"); + } + + private static void TryDelete(string path) + { + try { if (File.Exists(path)) File.Delete(path); } catch { /* best-effort cleanup */ } + } + + [Fact] + public void RegisterAndComplete_RoundTrips() + { + var rubricJson = JsonSerializer.Serialize(new[] { "Quality", "Completeness" }); + + _db.RegisterSession("s1", "my-skill", "/path/to/skill", "scenario-a", 0, "baseline", "gpt-4.1", "sessions/s1", "/work", "Fix the bug", "abcdef012345", rubricJson); + _db.CompleteSession("s1", "completed", """{"TokenEstimate":100}"""); + + var sessions = _db.GetCompletedSessions(); + var s = Assert.Single(sessions); + Assert.Equal("s1", s.Id); + Assert.Equal("my-skill", s.SkillName); + Assert.Equal("/path/to/skill", s.SkillPath); + Assert.Equal("scenario-a", s.ScenarioName); + Assert.Equal(0, s.RunIndex); + Assert.Equal("baseline", s.Role); + Assert.Equal("gpt-4.1", s.Model); + Assert.Equal("sessions/s1", s.ConfigDir); + Assert.Equal("completed", s.Status); + Assert.Equal("Fix the bug", s.Prompt); + Assert.Equal("abcdef012345", s.SkillSha); + Assert.Equal(rubricJson, s.RubricJson); + Assert.Equal("""{"TokenEstimate":100}""", s.MetricsJson); + Assert.Null(s.JudgeJson); + Assert.Null(s.PairwiseJson); + } + + [Fact] + public void SaveJudgeResult_UpdatesExistingRow() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "completed", "{}"); + _db.SaveJudgeResult("s1", """{"OverallScore":4}"""); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Equal("""{"OverallScore":4}""", s.JudgeJson); + } + + [Fact] + public void SavePairwiseResult_UpdatesExistingRow() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "completed", "{}"); + _db.SavePairwiseResult("s1", """{"Winner":"with-skill"}"""); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Equal("""{"Winner":"with-skill"}""", s.PairwiseJson); + } + + [Fact] + public void RegisterWithoutPromptOrSkillSha_StoresNulls() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "completed", "{}"); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Null(s.Prompt); + Assert.Null(s.SkillSha); + Assert.Null(s.RubricJson); + } + + [Fact] + public void GetCompletedSessions_ExcludesRunning() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + // Never completed — should not appear + var sessions = _db.GetCompletedSessions(); + Assert.Empty(sessions); + } + + [Fact] + public void GetCompletedSessions_IncludesTimedOut() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "model", null, null); + _db.CompleteSession("s1", "timed_out", "{}"); + + var sessions = _db.GetCompletedSessions(); + Assert.Single(sessions); + Assert.Equal("timed_out", sessions[0].Status); + } + + [Fact] + public void MultipleSessions_OrderedCorrectly() + { + // Register pairs for two scenarios + _db.RegisterSession("b0", "skill", "/p", "alpha", 0, "baseline", "m", null, null); + _db.RegisterSession("w0", "skill", "/p", "alpha", 0, "with-skill", "m", null, null); + _db.RegisterSession("b1", "skill", "/p", "beta", 0, "baseline", "m", null, null); + _db.RegisterSession("w1", "skill", "/p", "beta", 0, "with-skill", "m", null, null); + + _db.CompleteSession("b0", "completed", "{}"); + _db.CompleteSession("w0", "completed", "{}"); + _db.CompleteSession("b1", "completed", "{}"); + _db.CompleteSession("w1", "completed", "{}"); + + var sessions = _db.GetCompletedSessions(); + Assert.Equal(4, sessions.Count); + // Ordered by skill_name, scenario_name, run_index, role + Assert.Equal("alpha", sessions[0].ScenarioName); + Assert.Equal("baseline", sessions[0].Role); + Assert.Equal("alpha", sessions[1].ScenarioName); + Assert.Equal("with-skill", sessions[1].Role); + Assert.Equal("beta", sessions[2].ScenarioName); + } + + [Fact] + public async Task ConcurrentWrites_DoNotCorrupt() + { + const int count = 20; + var tasks = Enumerable.Range(0, count).Select(i => Task.Run(() => + { + var id = $"s{i}"; + _db.RegisterSession(id, "skill", "/p", "scn", i, i % 2 == 0 ? "baseline" : "with-skill", "m", null, null); + _db.CompleteSession(id, "completed", $"{{\"Index\":{i}}}"); + _db.SaveJudgeResult(id, $"{{\"Score\":{i}}}"); + })); + + await Task.WhenAll(tasks); + + var sessions = _db.GetCompletedSessions(); + Assert.Equal(count, sessions.Count); + Assert.All(sessions, s => + { + Assert.Equal("completed", s.Status); + Assert.NotNull(s.MetricsJson); + Assert.NotNull(s.JudgeJson); + }); + } + + [Fact] + public void ComputeDirectorySha_IsDeterministic() + { + var dir = Path.Combine(Path.GetTempPath(), $"sha-test-{Guid.NewGuid()}"); + Directory.CreateDirectory(dir); + try + { + File.WriteAllText(Path.Combine(dir, "SKILL.md"), "# Test Skill"); + File.WriteAllText(Path.Combine(dir, "plugin.json"), "{}"); + + var sha1 = SessionDatabase.ComputeDirectorySha(dir); + var sha2 = SessionDatabase.ComputeDirectorySha(dir); + Assert.Equal(sha1, sha2); + Assert.Equal(12, sha1.Length); + + // Changing content produces a different SHA + File.WriteAllText(Path.Combine(dir, "SKILL.md"), "# Modified"); + var sha3 = SessionDatabase.ComputeDirectorySha(dir); + Assert.NotEqual(sha1, sha3); + } + finally + { + Directory.Delete(dir, true); + } + } + + [Fact] + public void ComputeDirectorySha_DistinguishesPathAndContentBoundaries() + { + var dir1 = Path.Combine(Path.GetTempPath(), $"sha-boundary-a-{Guid.NewGuid()}"); + var dir2 = Path.Combine(Path.GetTempPath(), $"sha-boundary-b-{Guid.NewGuid()}"); + Directory.CreateDirectory(dir1); + Directory.CreateDirectory(dir2); + try + { + File.WriteAllText(Path.Combine(dir1, "a"), "12"); + File.WriteAllText(Path.Combine(dir1, "b"), "34"); + + File.WriteAllText(Path.Combine(dir2, "a1"), "2"); + File.WriteAllText(Path.Combine(dir2, "b"), "34"); + + Assert.NotEqual( + SessionDatabase.ComputeDirectorySha(dir1), + SessionDatabase.ComputeDirectorySha(dir2)); + } + finally + { + Directory.Delete(dir1, true); + Directory.Delete(dir2, true); + } + } + + [Fact] + public void SeparateDbFiles_AreIndependent() + { + // Simulates two concurrent eval processes using different result dirs + var dbPath2 = Path.Combine(Path.GetTempPath(), $"test-sessions-{Guid.NewGuid()}.db"); + try + { + using var db2 = new SessionDatabase(dbPath2); + + _db.RegisterSession("s1", "skill-a", "/a", "scn", 0, "baseline", "m", null, null); + _db.CompleteSession("s1", "completed", "{}"); + + db2.RegisterSession("s1", "skill-b", "/b", "scn", 0, "baseline", "m", null, null); + db2.CompleteSession("s1", "completed", "{}"); + + // Each DB has exactly one session with different skill names + var sessions1 = _db.GetCompletedSessions(); + var sessions2 = db2.GetCompletedSessions(); + Assert.Single(sessions1); + Assert.Single(sessions2); + Assert.Equal("skill-a", sessions1[0].SkillName); + Assert.Equal("skill-b", sessions2[0].SkillName); + } + finally + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + TryDelete(dbPath2); + TryDelete(dbPath2 + "-wal"); + TryDelete(dbPath2 + "-shm"); + } + } + + [Fact] + public void SchemaInfo_ContainsTypeAndVersion() + { + var info = _db.GetSchemaInfo(); + Assert.Equal("skill-validator", info["type"]); + Assert.Equal("2", info["version"]); + } + + [Fact] + public void SchemaInfo_CanPersistJudgeModel() + { + _db.SetSchemaInfo("judge_model", "claude-opus-4.6"); + + var info = _db.GetSchemaInfo(); + Assert.Equal("claude-opus-4.6", info["judge_model"]); + } + + [Fact] + public void LegacyDatabase_UpgradesRubricColumn() + { + var legacyDbPath = Path.Combine(Path.GetTempPath(), $"legacy-sessions-{Guid.NewGuid()}.db"); + try + { + using (var connection = new SqliteConnection($"Data Source={legacyDbPath}")) + { + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = """ + CREATE TABLE schema_info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT INTO schema_info (key, value) VALUES ('type', 'skill-validator'); + INSERT INTO schema_info (key, value) VALUES ('version', '1'); + + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + skill_name TEXT NOT NULL, + skill_path TEXT NOT NULL, + scenario_name TEXT NOT NULL, + run_index INTEGER NOT NULL, + role TEXT NOT NULL, + model TEXT NOT NULL, + config_dir TEXT, + work_dir TEXT, + prompt TEXT, + skill_sha TEXT, + status TEXT NOT NULL DEFAULT 'running', + started_at TEXT NOT NULL, + completed_at TEXT + ); + + CREATE TABLE run_results ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id), + metrics_json TEXT NOT NULL, + judge_json TEXT, + pairwise_json TEXT + ); + + INSERT INTO sessions (id, skill_name, skill_path, scenario_name, run_index, role, model, status, started_at, completed_at) + VALUES ('s1', 'skill', '/p', 'scn', 0, 'baseline', 'model', 'completed', '2026-01-01T00:00:00Z', '2026-01-01T00:01:00Z'); + INSERT INTO run_results (session_id, metrics_json) VALUES ('s1', '{}'); + """; + cmd.ExecuteNonQuery(); + } + + using var upgradedDb = new SessionDatabase(legacyDbPath); + var legacySession = Assert.Single(upgradedDb.GetCompletedSessions()); + Assert.Null(legacySession.RubricJson); + Assert.Equal("2", upgradedDb.GetSchemaInfo()["version"]); + + var rubricJson = JsonSerializer.Serialize(new[] { "Quality" }); + upgradedDb.RegisterSession("s2", "skill", "/p", "scn", 1, "with-skill", "model", null, null, "Prompt", null, rubricJson); + upgradedDb.CompleteSession("s2", "completed", "{}"); + + var upgradedSession = Assert.Single(upgradedDb.GetCompletedSessions(), s => s.Id == "s2"); + Assert.Equal(rubricJson, upgradedSession.RubricJson); + } + finally + { + SqliteConnection.ClearAllPools(); + TryDelete(legacyDbPath); + TryDelete(legacyDbPath + "-wal"); + TryDelete(legacyDbPath + "-shm"); + } + } + + [Fact] + public void CompleteSession_RequiresExistingSession() + { + Assert.Throws(() => + _db.CompleteSession("missing", "completed", "{}")); + } + + [Fact] + public void ConfigDir_StoredAsRelativePath() + { + _db.RegisterSession("s1", "skill", "/p", "scn", 0, "baseline", "m", "sessions/s1", null); + _db.CompleteSession("s1", "completed", "{}"); + + var s = Assert.Single(_db.GetCompletedSessions()); + Assert.Equal("sessions/s1", s.ConfigDir); + } +}