Add 'run_command_and_assert' assertion

eng/skill-validator/src/Commands/EvaluateCommand.cs

@@ -641,9 +641,9 @@ private static async Task<RunExecutionResult> ExecuteRun(
         // Evaluate assertions on all three runs
         if (scenario.Assertions is { Count: > 0 })
         {
-            baselineMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, baselineMetrics.AgentOutput, baselineMetrics.WorkDir);
-            isolatedMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, isolatedMetrics.AgentOutput, isolatedMetrics.WorkDir);
-            pluginMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, pluginMetrics.AgentOutput, pluginMetrics.WorkDir);
+            baselineMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, baselineMetrics.AgentOutput, baselineMetrics.WorkDir, scenario.Timeout);
+            isolatedMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, isolatedMetrics.AgentOutput, isolatedMetrics.WorkDir, scenario.Timeout);
+            pluginMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(scenario.Assertions, pluginMetrics.AgentOutput, pluginMetrics.WorkDir, scenario.Timeout);
         }
 
         // Evaluate constraints on all three runs
@@ -889,9 +889,9 @@ private static async Task<NoiseTestResult> ExecuteNoiseTest(
                         if (scenario.Assertions is { Count: > 0 })
                         {
                             skillOnlyMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(
-                                scenario.Assertions, skillOnlyMetrics.AgentOutput, skillOnlyMetrics.WorkDir);
+                                scenario.Assertions, skillOnlyMetrics.AgentOutput, skillOnlyMetrics.WorkDir, scenario.Timeout);
                             allSkillsMetrics.AssertionResults = await AssertionEvaluator.EvaluateAssertions(
-                                scenario.Assertions, allSkillsMetrics.AgentOutput, allSkillsMetrics.WorkDir);
+                                scenario.Assertions, allSkillsMetrics.AgentOutput, allSkillsMetrics.WorkDir, scenario.Timeout);
                         }
                         var soConstraints = AssertionEvaluator.EvaluateConstraints(scenario, skillOnlyMetrics);
                         var asConstraints = AssertionEvaluator.EvaluateConstraints(scenario, allSkillsMetrics);

eng/skill-validator/src/Models/Models.cs

@@ -16,17 +16,29 @@ public enum AssertionType
     OutputMatches,
     OutputNotMatches,
     ExitSuccess,
+    RunCommandAndAssert,
     ExpectTools,
     RejectTools,
     MaxTurns,
     MaxTokens,
 }
 
+public sealed record CommandAssertionArgs(
+    string CommandToRun,
+    string? CommandArguments = null,
+    int? ExpectedExitCode = null,
+    string? ExpectedStdOutContains = null,
+    string? ExpectedStdErrorContains = null,
+    string? ExpectedStdOutMatches = null,
+    string? ExpectedStdErrorMatches = null,
+    int? Timeout = null);
+
 public sealed record Assertion(
     AssertionType Type,
     string? Path = null,
     string? Value = null,
-    string? Pattern = null);
+    string? Pattern = null,
+    CommandAssertionArgs? CommandArgs = null);
 
 public sealed record AssertionResult(
     Assertion Assertion,

eng/skill-validator/src/Services/AssertionEvaluator.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+using System.Text;
 using System.Text.RegularExpressions;
 using Microsoft.Extensions.FileSystemGlobbing;
 using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
@@ -10,12 +12,13 @@ public static class AssertionEvaluator
     public static async Task<List<AssertionResult>> EvaluateAssertions(
         IReadOnlyList<Assertion> assertions,
         string agentOutput,
-        string workDir)
+        string workDir,
+        int scenarioTimeoutSeconds = 120)
     {
         var results = new List<AssertionResult>();
         foreach (var assertion in assertions)
         {
-            var result = await EvaluateAssertion(assertion, agentOutput, workDir);
+            var result = await EvaluateAssertion(assertion, agentOutput, workDir, scenarioTimeoutSeconds);
             results.Add(result);
         }
         return results;
@@ -84,7 +87,8 @@ public static List<AssertionResult> EvaluateConstraints(
     private static async Task<AssertionResult> EvaluateAssertion(
         Assertion assertion,
         string agentOutput,
-        string workDir)
+        string workDir,
+        int scenarioTimeoutSeconds)
     {
         return assertion.Type switch
         {
@@ -97,6 +101,7 @@ private static async Task<AssertionResult> EvaluateAssertion(
             AssertionType.OutputMatches => EvalOutputMatches(assertion, agentOutput),
             AssertionType.OutputNotMatches => EvalOutputNotMatches(assertion, agentOutput),
             AssertionType.ExitSuccess => EvalExitSuccess(assertion, agentOutput),
+            AssertionType.RunCommandAndAssert => await EvalRunCommandAndAssert(assertion, workDir, scenarioTimeoutSeconds),
             _ => new AssertionResult(assertion, false, $"Unknown assertion type: {assertion.Type}"),
         };
     }
@@ -236,6 +241,135 @@ private static AssertionResult EvalExitSuccess(Assertion a, string agentOutput)
                 : "Agent produced no output");
     }
 
+    private const int MaxOutputLength = 4096;
+
+    private static string TruncateOutput(string output)
+    {
+        if (output.Length <= MaxOutputLength)
+            return output;
+        return $"{output[..MaxOutputLength]}... [truncated, {output.Length} chars total]";
+    }
+
+    private static async Task<AssertionResult> EvalRunCommandAndAssert(Assertion a, string workDir, int scenarioTimeoutSeconds)
+    {
+        var cmd = a.CommandArgs ?? throw new UnreachableException();
+        var command = cmd.CommandToRun;
+        var timeoutSeconds = cmd.Timeout ?? scenarioTimeoutSeconds;
+
+        if (timeoutSeconds <= 0)
+        {
+            return new AssertionResult(a, false, $"Invalid timeout value {timeoutSeconds}s. Timeout must be greater than 0.");
+        }
+
+        var processStartInfo = new ProcessStartInfo(command, cmd.CommandArguments ?? string.Empty)
+        {
+            WorkingDirectory = workDir,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+        };
+
+        Process process;
+        try
+        {
+            var started = Process.Start(processStartInfo);
+            if (started is null)
+            {
+                return new AssertionResult(a, false, $"Failed to start process '{command}' {cmd.CommandArguments}");
+            }
+            process = started;
+        }
+        catch (Exception ex)
+        {
+            return new AssertionResult(a, false, $"Failed to start process '{command}' {cmd.CommandArguments}: {ex.Message}");
+        }
+
+        using (process)
+        {
+            var outBuilder = new StringBuilder();
+            var errBuilder = new StringBuilder();
+            process.OutputDataReceived += (_, e) =>
+            {
+                if (e.Data is not null)
+                    outBuilder.AppendLine(e.Data);
+            };
+            process.ErrorDataReceived += (_, e) =>
+            {
+                if (e.Data is not null)
+                    errBuilder.AppendLine(e.Data);
+            };
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+
+            using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds));
+            try
+            {
+                await process.WaitForExitAsync(cts.Token);
+            }
+            catch (OperationCanceledException)
+            {
+                try { process.Kill(entireProcessTree: true); } catch { }
+                return new AssertionResult(a, false, $"Command timed out after {timeoutSeconds}s");
+            }
+
+            var actualExitCode = process.ExitCode;
+            if (cmd.ExpectedExitCode.HasValue && cmd.ExpectedExitCode.Value != actualExitCode)
+            {
+                return new AssertionResult(a, false, $"Command exited with code {actualExitCode} but expected {cmd.ExpectedExitCode.Value}");
+            }
+
+            var actualStdOut = outBuilder.ToString();
+            if (cmd.ExpectedStdOutContains is not null)
+            {
+                if (!actualStdOut.Contains(cmd.ExpectedStdOutContains, StringComparison.Ordinal))
+                {
+                    return new AssertionResult(a, false, $"Command stdout did not contain expected value. Stdout: {TruncateOutput(actualStdOut)}");
+                }
+            }
+
+            if (cmd.ExpectedStdOutMatches is not null)
+            {
+                try
+                {
+                    if (!Regex.IsMatch(actualStdOut, cmd.ExpectedStdOutMatches, RegexOptions.IgnoreCase, RegexTimeout))
+                    {
+                        return new AssertionResult(a, false, $"Command stdout did not match pattern '{cmd.ExpectedStdOutMatches}'. Stdout: {TruncateOutput(actualStdOut)}");
+                    }
+                }
+                catch (RegexMatchTimeoutException)
+                {
+                    return new AssertionResult(a, false, $"Regex pattern '{cmd.ExpectedStdOutMatches}' timed out after {RegexTimeout.TotalSeconds}s");
+                }
+            }
+
+            var actualStdErr = errBuilder.ToString();
+            if (cmd.ExpectedStdErrorContains is not null)
+            {
+                if (!actualStdErr.Contains(cmd.ExpectedStdErrorContains, StringComparison.Ordinal))
+                {
+                    return new AssertionResult(a, false, $"Command stderr did not contain expected value. Stderr: {TruncateOutput(actualStdErr)}");
+                }
+            }
+
+            if (cmd.ExpectedStdErrorMatches is not null)
+            {
+                try
+                {
+                    if (!Regex.IsMatch(actualStdErr, cmd.ExpectedStdErrorMatches, RegexOptions.IgnoreCase, RegexTimeout))
+                    {
+                        return new AssertionResult(a, false, $"Command stderr did not match pattern '{cmd.ExpectedStdErrorMatches}'. Stderr: {TruncateOutput(actualStdErr)}");
+                    }
+                }
+                catch (RegexMatchTimeoutException)
+                {
+                    return new AssertionResult(a, false, $"Regex pattern '{cmd.ExpectedStdErrorMatches}' timed out after {RegexTimeout.TotalSeconds}s");
+                }
+            }
+
+            return new AssertionResult(a, true, string.Empty);
+        }
+    }
+
     private static Task<bool> FileExistsGlob(string pattern, string workDir)
     {
         try

eng/skill-validator/src/Services/EvalSchema.cs

@@ -84,6 +84,7 @@ private static Assertion ParseAssertion(RawAssertion raw)
             "output_matches" => AssertionType.OutputMatches,
             "output_not_matches" => AssertionType.OutputNotMatches,
             "exit_success" => AssertionType.ExitSuccess,
+            "run_command_and_assert" => AssertionType.RunCommandAndAssert,
             _ => throw new InvalidOperationException($"Unknown assertion type: {raw.Type}"),
         };
 
@@ -108,11 +109,37 @@ private static Assertion ParseAssertion(RawAssertion raw)
                 if (string.IsNullOrWhiteSpace(raw.Pattern))
                     throw new InvalidOperationException($"Assertion '{raw.Type}' requires 'pattern'");
                 break;
+            case AssertionType.RunCommandAndAssert:
+                if (string.IsNullOrWhiteSpace(raw.CommandToRun))
+                    throw new InvalidOperationException($"Assertion '{raw.Type}' requires 'command_to_run'");
+
+                if (raw.ExpectedExitCode is null &&
+                    string.IsNullOrWhiteSpace(raw.ExpectedStdOutputContains) &&
+                    string.IsNullOrWhiteSpace(raw.ExpectedStdErrorContains) &&
+                    string.IsNullOrWhiteSpace(raw.ExpectedStdOutputMatches) &&
+                    string.IsNullOrWhiteSpace(raw.ExpectedStdErrorMatches))
+                    throw new InvalidOperationException($"Assertion '{raw.Type}' requires one or more of 'expected_exit_code', 'expected_std_output_contains', 'expected_std_error_contains', 'expected_std_output_matches', or 'expected_std_error_matches'");
+                break;
         }
 
-        return new Assertion(type, raw.Path, raw.Value, raw.Pattern);
+        CommandAssertionArgs? commandArgs = type == AssertionType.RunCommandAndAssert
+            ? new CommandAssertionArgs(
+                raw.CommandToRun!,
+                NullIfWhiteSpace(raw.CommandArguments),
+                raw.ExpectedExitCode,
+                NullIfWhiteSpace(raw.ExpectedStdOutputContains),
+                NullIfWhiteSpace(raw.ExpectedStdErrorContains),
+                NullIfWhiteSpace(raw.ExpectedStdOutputMatches),
+                NullIfWhiteSpace(raw.ExpectedStdErrorMatches),
+                raw.CommandTimeout)
+            : null;
+
+        return new Assertion(type, raw.Path, raw.Value, raw.Pattern, commandArgs);
     }
 
+    private static string? NullIfWhiteSpace(string? value) =>
+        string.IsNullOrWhiteSpace(value) ? null : value;
+
     // Raw YAML deserialization models
 
     internal sealed class RawFrontmatter
@@ -176,5 +203,14 @@ internal sealed class RawAssertion
         public string? Path { get; set; }
         public string? Value { get; set; }
         public string? Pattern { get; set; }
+
+        public string? CommandToRun { get; set; }
+        public string? CommandArguments { get; set; }
+        public int? ExpectedExitCode { get; set; }
+        public string? ExpectedStdOutputContains { get; set; }
+        public string? ExpectedStdErrorContains { get; set; }
+        public string? ExpectedStdOutputMatches { get; set; }
+        public string? ExpectedStdErrorMatches { get; set; }
+        public int? CommandTimeout { get; set; }
     }
 }