diff --git a/eng/skill-validator/src/Commands/EvaluateCommand.cs b/eng/skill-validator/src/Commands/EvaluateCommand.cs index b419b285fb..93c17ebce7 100644 --- a/eng/skill-validator/src/Commands/EvaluateCommand.cs +++ b/eng/skill-validator/src/Commands/EvaluateCommand.cs @@ -641,9 +641,9 @@ private static async Task ExecuteRun( // Evaluate assertions on all three runs if (scenario.Assertions is { Count: > 0 }) { - baselineMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, baselineMetrics.AgentOutput, baselineMetrics.WorkDir); - isolatedMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, isolatedMetrics.AgentOutput, isolatedMetrics.WorkDir); - pluginMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, pluginMetrics.AgentOutput, pluginMetrics.WorkDir); + baselineMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, baselineMetrics.AgentOutput, baselineMetrics.WorkDir, scenario.Timeout); + isolatedMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, isolatedMetrics.AgentOutput, isolatedMetrics.WorkDir, scenario.Timeout); + pluginMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, pluginMetrics.AgentOutput, pluginMetrics.WorkDir, scenario.Timeout); } // Evaluate constraints on all three runs @@ -889,9 +889,9 @@ private static async Task ExecuteNoiseTest( if (scenario.Assertions is { Count: > 0 }) { skillOnlyMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions( - scenario.Assertions, skillOnlyMetrics.AgentOutput, skillOnlyMetrics.WorkDir); + scenario.Assertions, skillOnlyMetrics.AgentOutput, skillOnlyMetrics.WorkDir, scenario.Timeout); allSkillsMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions( - scenario.Assertions, allSkillsMetrics.AgentOutput, allSkillsMetrics.WorkDir); + scenario.Assertions, allSkillsMetrics.AgentOutput, allSkillsMetrics.WorkDir, scenario.Timeout); } var soConstraints = AssertionEvaluator.EvaluateConstraints(scenario, skillOnlyMetrics); var asConstraints = AssertionEvaluator.EvaluateConstraints(scenario, allSkillsMetrics); diff --git a/eng/skill-validator/src/Models/Models.cs b/eng/skill-validator/src/Models/Models.cs index 2641a10380..ce0ba64672 100644 --- a/eng/skill-validator/src/Models/Models.cs +++ b/eng/skill-validator/src/Models/Models.cs @@ -16,17 +16,29 @@ public enum AssertionType OutputMatches, OutputNotMatches, ExitSuccess, + RunCommandAndAssert, ExpectTools, RejectTools, MaxTurns, MaxTokens, } +public sealed record CommandAssertionArgs( + string CommandToRun, + string? CommandArguments = null, + int? ExpectedExitCode = null, + string? ExpectedStdOutContains = null, + string? ExpectedStdErrorContains = null, + string? ExpectedStdOutMatches = null, + string? ExpectedStdErrorMatches = null, + int? Timeout = null); + public sealed record Assertion( AssertionType Type, string? Path = null, string? Value = null, - string? Pattern = null); + string? Pattern = null, + CommandAssertionArgs? CommandArgs = null); public sealed record AssertionResult( Assertion Assertion, diff --git a/eng/skill-validator/src/Services/AssertionEvaluator.cs b/eng/skill-validator/src/Services/AssertionEvaluator.cs index eab00ced4a..b6f569ed05 100644 --- a/eng/skill-validator/src/Services/AssertionEvaluator.cs +++ b/eng/skill-validator/src/Services/AssertionEvaluator.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Text.RegularExpressions; using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.FileSystemGlobbing.Abstractions; @@ -10,12 +11,13 @@ public static class AssertionEvaluator public static async Task> EvaluateAssertions( IReadOnlyList assertions, string agentOutput, - string workDir) + string workDir, + int scenarioTimeoutSeconds = 120) { var results = new List(); foreach (var assertion in assertions) { - var result = await EvaluateAssertion(assertion, agentOutput, workDir); + var result = await EvaluateAssertion(assertion, agentOutput, workDir, scenarioTimeoutSeconds); results.Add(result); } return results; @@ -84,7 +86,8 @@ public static List EvaluateConstraints( private static async Task EvaluateAssertion( Assertion assertion, string agentOutput, - string workDir) + string workDir, + int scenarioTimeoutSeconds) { return assertion.Type switch { @@ -97,6 +100,7 @@ private static async Task EvaluateAssertion( AssertionType.OutputMatches => EvalOutputMatches(assertion, agentOutput), AssertionType.OutputNotMatches => EvalOutputNotMatches(assertion, agentOutput), AssertionType.ExitSuccess => EvalExitSuccess(assertion, agentOutput), + AssertionType.RunCommandAndAssert => await EvalRunCommandAndAssert(assertion, workDir, scenarioTimeoutSeconds), _ => new AssertionResult(assertion, false, $"Unknown assertion type: {assertion.Type}"), }; } @@ -236,6 +240,126 @@ private static AssertionResult EvalExitSuccess(Assertion a, string agentOutput) : "Agent produced no output"); } + private const int MaxOutputLength = 4096; + + private static string TruncateOutput(string output) + { + if (output.Length <= MaxOutputLength) + return output; + return $"{output[..MaxOutputLength]}... [truncated, {output.Length} chars total]"; + } + + private static async Task EvalRunCommandAndAssert(Assertion a, string workDir, int scenarioTimeoutSeconds) + { + var cmd = a.CommandArgs ?? throw new UnreachableException(); + var command = cmd.CommandToRun; + var timeoutSeconds = cmd.Timeout ?? scenarioTimeoutSeconds; + + if (timeoutSeconds <= 0) + { + return new AssertionResult(a, false, $"Invalid timeout value {timeoutSeconds}s. Timeout must be greater than 0."); + } + + var processStartInfo = new ProcessStartInfo(command, cmd.CommandArguments ?? string.Empty) + { + WorkingDirectory = workDir, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + AgentRunner.ScrubSensitiveEnvironment(processStartInfo); + + Process process; + try + { + var started = Process.Start(processStartInfo); + if (started is null) + { + return new AssertionResult(a, false, $"Failed to start process '{command}' {cmd.CommandArguments}"); + } + process = started; + } + catch (Exception ex) + { + return new AssertionResult(a, false, $"Failed to start process '{command}' {cmd.CommandArguments}: {ex.Message}"); + } + + using (process) + { + var stdOutTask = process.StandardOutput.ReadToEndAsync(); + var stdErrTask = process.StandardError.ReadToEndAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); + try + { + await process.WaitForExitAsync(cts.Token); + } + catch (OperationCanceledException) + { + try { process.Kill(entireProcessTree: true); } catch { } + return new AssertionResult(a, false, $"Command timed out after {timeoutSeconds}s"); + } + + var actualStdOut = await stdOutTask; + var actualStdErr = await stdErrTask; + + var actualExitCode = process.ExitCode; + if (cmd.ExpectedExitCode.HasValue && cmd.ExpectedExitCode.Value != actualExitCode) + { + return new AssertionResult(a, false, $"Command exited with code {actualExitCode} but expected {cmd.ExpectedExitCode.Value}"); + } + + if (cmd.ExpectedStdOutContains is not null) + { + if (!actualStdOut.Contains(cmd.ExpectedStdOutContains, StringComparison.OrdinalIgnoreCase)) + { + return new AssertionResult(a, false, $"Command stdout did not contain expected value. Stdout: {TruncateOutput(actualStdOut)}"); + } + } + + if (cmd.ExpectedStdOutMatches is not null) + { + try + { + if (!Regex.IsMatch(actualStdOut, cmd.ExpectedStdOutMatches, RegexOptions.IgnoreCase, RegexTimeout)) + { + return new AssertionResult(a, false, $"Command stdout did not match pattern '{cmd.ExpectedStdOutMatches}'. Stdout: {TruncateOutput(actualStdOut)}"); + } + } + catch (RegexMatchTimeoutException) + { + return new AssertionResult(a, false, $"Regex pattern '{cmd.ExpectedStdOutMatches}' timed out after {RegexTimeout.TotalSeconds}s"); + } + } + + if (cmd.ExpectedStdErrorContains is not null) + { + if (!actualStdErr.Contains(cmd.ExpectedStdErrorContains, StringComparison.OrdinalIgnoreCase)) + { + return new AssertionResult(a, false, $"Command stderr did not contain expected value. Stderr: {TruncateOutput(actualStdErr)}"); + } + } + + if (cmd.ExpectedStdErrorMatches is not null) + { + try + { + if (!Regex.IsMatch(actualStdErr, cmd.ExpectedStdErrorMatches, RegexOptions.IgnoreCase, RegexTimeout)) + { + return new AssertionResult(a, false, $"Command stderr did not match pattern '{cmd.ExpectedStdErrorMatches}'. Stderr: {TruncateOutput(actualStdErr)}"); + } + } + catch (RegexMatchTimeoutException) + { + return new AssertionResult(a, false, $"Regex pattern '{cmd.ExpectedStdErrorMatches}' timed out after {RegexTimeout.TotalSeconds}s"); + } + } + + return new AssertionResult(a, true, string.Empty); + } + } + private static Task FileExistsGlob(string pattern, string workDir) { try diff --git a/eng/skill-validator/src/Services/EvalSchema.cs b/eng/skill-validator/src/Services/EvalSchema.cs index 6130da3a08..81e3f5d4a9 100644 --- a/eng/skill-validator/src/Services/EvalSchema.cs +++ b/eng/skill-validator/src/Services/EvalSchema.cs @@ -84,6 +84,7 @@ private static Assertion ParseAssertion(RawAssertion raw) "output_matches" => AssertionType.OutputMatches, "output_not_matches" => AssertionType.OutputNotMatches, "exit_success" => AssertionType.ExitSuccess, + "run_command_and_assert" => AssertionType.RunCommandAndAssert, _ => throw new InvalidOperationException($"Unknown assertion type: {raw.Type}"), }; @@ -108,11 +109,37 @@ private static Assertion ParseAssertion(RawAssertion raw) if (string.IsNullOrWhiteSpace(raw.Pattern)) throw new InvalidOperationException($"Assertion '{raw.Type}' requires 'pattern'"); break; + case AssertionType.RunCommandAndAssert: + if (string.IsNullOrWhiteSpace(raw.CommandToRun)) + throw new InvalidOperationException($"Assertion '{raw.Type}' requires 'command_to_run'"); + + if (raw.ExpectedExitCode is null && + string.IsNullOrWhiteSpace(raw.ExpectedStdOutputContains) && + string.IsNullOrWhiteSpace(raw.ExpectedStdErrorContains) && + string.IsNullOrWhiteSpace(raw.ExpectedStdOutputMatches) && + string.IsNullOrWhiteSpace(raw.ExpectedStdErrorMatches)) + throw new InvalidOperationException($"Assertion '{raw.Type}' requires one or more of 'expected_exit_code', 'expected_std_output_contains', 'expected_std_error_contains', 'expected_std_output_matches', or 'expected_std_error_matches'"); + break; } - return new Assertion(type, raw.Path, raw.Value, raw.Pattern); + CommandAssertionArgs? commandArgs = type == AssertionType.RunCommandAndAssert + ? new CommandAssertionArgs( + raw.CommandToRun!, + NullIfWhiteSpace(raw.CommandArguments), + raw.ExpectedExitCode, + NullIfWhiteSpace(raw.ExpectedStdOutputContains), + NullIfWhiteSpace(raw.ExpectedStdErrorContains), + NullIfWhiteSpace(raw.ExpectedStdOutputMatches), + NullIfWhiteSpace(raw.ExpectedStdErrorMatches), + raw.CommandTimeout) + : null; + + return new Assertion(type, raw.Path, raw.Value, raw.Pattern, commandArgs); } + private static string? NullIfWhiteSpace(string? value) => + string.IsNullOrWhiteSpace(value) ? null : value; + // Raw YAML deserialization models internal sealed class RawFrontmatter @@ -176,5 +203,14 @@ internal sealed class RawAssertion public string? Path { get; set; } public string? Value { get; set; } public string? Pattern { get; set; } + + public string? CommandToRun { get; set; } + public string? CommandArguments { get; set; } + public int? ExpectedExitCode { get; set; } + public string? ExpectedStdOutputContains { get; set; } + public string? ExpectedStdErrorContains { get; set; } + public string? ExpectedStdOutputMatches { get; set; } + public string? ExpectedStdErrorMatches { get; set; } + public int? CommandTimeout { get; set; } } } diff --git a/eng/skill-validator/tests/AssertionsTests.cs b/eng/skill-validator/tests/AssertionsTests.cs index 3bf3f1003a..ab7686a725 100644 --- a/eng/skill-validator/tests/AssertionsTests.cs +++ b/eng/skill-validator/tests/AssertionsTests.cs @@ -327,3 +327,238 @@ public void ExpectToolsChecksEachToolIndependently() Assert.True(results[2].Passed); // create_file: used } } + +public class RunCommandAndAssertTests : IDisposable +{ + private readonly string _tmpDir; + + public RunCommandAndAssertTests() + { + _tmpDir = Path.Combine(Path.GetTempPath(), $"run-cmd-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tmpDir); + } + + public void Dispose() + { + try { Directory.Delete(_tmpDir, true); } catch { } + } + + private static string Shell => OperatingSystem.IsWindows() ? "cmd" : "/bin/sh"; + + private static string ShellArgs(string command) => + OperatingSystem.IsWindows() + ? $"/c {command}" + : $"-c \"{command}\""; + + private static Assertion CmdAssertion( + string? commandArguments = null, + int? expectedExitCode = null, + string? expectedStdOutContains = null, + string? expectedStdErrorContains = null, + string? expectedStdOutMatches = null, + string? expectedStdErrorMatches = null, + int? timeout = null) => + new(AssertionType.RunCommandAndAssert, + CommandArgs: new CommandAssertionArgs( + Shell, + commandArguments, + expectedExitCode, + expectedStdOutContains, + expectedStdErrorContains, + expectedStdOutMatches, + expectedStdErrorMatches, + timeout)); + + [Fact] + public async Task PassesWhenExitCodeMatches() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("exit 0"), expectedExitCode: 0)], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task FailsWhenExitCodeDoesNotMatch() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("exit 1"), expectedExitCode: 0)], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("exited with code 1 but expected 0", results[0].Message); + } + + [Fact] + public async Task PassesWithNonZeroExpectedExitCode() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("exit 42"), expectedExitCode: 42)], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task PassesWhenStdOutContainsExpectedValue() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo hello_world"), expectedStdOutContains: "hello_world")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task FailsWhenStdOutDoesNotContainExpectedValue() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo hello"), expectedStdOutContains: "goodbye")], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("stdout did not contain expected value", results[0].Message); + } + + [Fact] + public async Task PassesWhenStdErrContainsExpectedValue() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo error_marker 1>&2"), expectedStdErrorContains: "error_marker")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task FailsWhenStdErrDoesNotContainExpectedValue() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo some_error 1>&2"), expectedStdErrorContains: "different_error")], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("stderr did not contain expected value", results[0].Message); + } + + [Fact] + public async Task PassesWhenStdOutMatchesRegex() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo build_v2.3.1_ok"), expectedStdOutMatches: @"build_v\d+\.\d+\.\d+_ok")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task FailsWhenStdOutDoesNotMatchRegex() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo no_version_here"), expectedStdOutMatches: @"v\d+\.\d+\.\d+")], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("stdout did not match pattern", results[0].Message); + } + + [Fact] + public async Task PassesWhenStdErrMatchesRegex() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo warn_code_42 1>&2"), expectedStdErrorMatches: @"warn_code_\d+")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task FailsWhenStdErrDoesNotMatchRegex() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs("echo some_text 1>&2"), expectedStdErrorMatches: @"error_\d+")], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("stderr did not match pattern", results[0].Message); + } + + [Fact] + public async Task PassesWhenAllChecksPass() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion( + commandArguments: ShellArgs("echo stdout_text && echo stderr_text 1>&2"), + expectedExitCode: 0, + expectedStdOutContains: "stdout_text", + expectedStdErrorContains: "stderr_text")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task ExitCodeFailureShortCircuitsOtherChecks() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion( + commandArguments: ShellArgs("echo hello && exit 1"), + expectedExitCode: 0, + expectedStdOutContains: "hello")], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("exited with code 1 but expected 0", results[0].Message); + } + + [Fact] + public async Task StdOutFailureShortCircuitsStdErrCheck() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion( + commandArguments: ShellArgs("echo wrong && echo expected_error 1>&2"), + expectedStdOutContains: "expected_output", + expectedStdErrorContains: "expected_error")], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("stdout did not contain expected value", results[0].Message); + } + + [Fact] + public async Task IgnoresExitCodeWhenNotSpecified() + { + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion( + commandArguments: ShellArgs("echo output_text && exit 1"), + expectedStdOutContains: "output_text")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task UsesWorkDirAsProcessWorkingDirectory() + { + File.WriteAllText(Path.Combine(_tmpDir, "marker.txt"), "test_content"); + var catCmd = OperatingSystem.IsWindows() ? "type marker.txt" : "cat marker.txt"; + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs(catCmd), expectedStdOutContains: "test_content")], + "", _tmpDir); + Assert.True(results[0].Passed); + } + + [Fact] + public async Task UsesCustomTimeoutFromAssertion() + { + // Use a very short timeout (1 second) so a long-running command times out. + // Note: 'timeout' is not reliable in non-interactive contexts (exits immediately on CI), + // so we use 'ping -n 11 127.0.0.1' which sleeps ~10s reliably everywhere. + var sleepCmd = OperatingSystem.IsWindows() ? "ping -n 11 127.0.0.1 >nul" : "sleep 10"; + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs(sleepCmd), expectedExitCode: 0, timeout: 1)], + "", _tmpDir); + Assert.False(results[0].Passed); + Assert.Contains("Command timed out after 1s", results[0].Message); + } + + [Fact] + public async Task FallsBackToScenarioTimeout() + { + // Pass a short scenario timeout (1 second) - the command should time out. + // Note: 'timeout' is not reliable in non-interactive contexts (exits immediately on CI), + // so we use 'ping -n 11 127.0.0.1' which sleeps ~10s reliably everywhere. + var sleepCmd = OperatingSystem.IsWindows() ? "ping -n 11 127.0.0.1 >nul" : "sleep 10"; + var results = await AssertionEvaluator.EvaluateAssertions( + [CmdAssertion(commandArguments: ShellArgs(sleepCmd), expectedExitCode: 0)], + "", _tmpDir, scenarioTimeoutSeconds: 1); + Assert.False(results[0].Passed); + Assert.Contains("Command timed out after 1s", results[0].Message); + } +} diff --git a/eng/skill-validator/tests/EvalSchemaTests.cs b/eng/skill-validator/tests/EvalSchemaTests.cs index 2b4eaeade4..d3d22d2847 100644 --- a/eng/skill-validator/tests/EvalSchemaTests.cs +++ b/eng/skill-validator/tests/EvalSchemaTests.cs @@ -177,6 +177,70 @@ public void ParsesPartialConfigSection() Assert.Equal(2, config.MaxParallelScenarios); Assert.Null(config.MaxParallelRuns); } + + [Fact] + public void ParsesRunCommandAndAssertAssertion() + { + var yaml = """ + scenarios: + - name: "Build check" + prompt: "Build the project" + assertions: + - type: "run_command_and_assert" + command_to_run: "dotnet" + command_arguments: "build" + expected_exit_code: 0 + expected_std_output_contains: "Build succeeded" + expected_std_error_contains: "warning" + expected_std_output_matches: "Build \\w+" + expected_std_error_matches: "warn.*" + command_timeout: 60 + """; + var config = EvalSchema.ParseEvalConfig(yaml); + var assertion = config.Scenarios[0].Assertions![0]; + Assert.Equal(AssertionType.RunCommandAndAssert, assertion.Type); + Assert.NotNull(assertion.CommandArgs); + var cmd = assertion.CommandArgs!; + Assert.Equal("dotnet", cmd.CommandToRun); + Assert.Equal("build", cmd.CommandArguments); + Assert.Equal(0, cmd.ExpectedExitCode); + Assert.Equal("Build succeeded", cmd.ExpectedStdOutContains); + Assert.Equal("warning", cmd.ExpectedStdErrorContains); + Assert.Equal("Build \\w+", cmd.ExpectedStdOutMatches); + Assert.Equal("warn.*", cmd.ExpectedStdErrorMatches); + Assert.Equal(60, cmd.Timeout); + } + + [Fact] + public void RejectsRunCommandAndAssertWithoutCommandToRun() + { + var yaml = """ + scenarios: + - name: "Test" + prompt: "Do it" + assertions: + - type: "run_command_and_assert" + expected_exit_code: 0 + """; + var ex = Assert.Throws(() => EvalSchema.ParseEvalConfig(yaml)); + Assert.Contains("command_to_run", ex.Message); + } + + [Fact] + public void RejectsRunCommandAndAssertWithoutAnyExpectedChecks() + { + var yaml = """ + scenarios: + - name: "Test" + prompt: "Do it" + assertions: + - type: "run_command_and_assert" + command_to_run: "dotnet" + """; + var ex = Assert.Throws(() => EvalSchema.ParseEvalConfig(yaml)); + Assert.Contains("expected_exit_code", ex.Message); + Assert.Contains("expected_std_output_contains", ex.Message); + } } public class ValidateEvalConfigTests