Add per-stage summaries to benchmark output

cmd/nyann-bench/common.go

@@ -458,6 +458,7 @@ func runScenario(ctx context.Context, cancel context.CancelFunc, opts scenarioOp
 
 	summary := analysis.Compute(records, 0, 0)
 	summary.Timestamps = timestamps
+	summary.Stages = analysis.ComputeStages(records, stageTimestamps)
 	return summary, nil
 }
 

cmd/nyann-bench/generate.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log/slog"
 	"os"
 	"os/signal"
@@ -27,6 +28,7 @@ func generateCmd() *cobra.Command {
 		workerID    int
 		workersFlag string
 		metricsAddr string
+		jsonOnly    bool
 		kubeFlags   kube.Flags
 	)
 
@@ -63,6 +65,10 @@ Workload types:
   corpus      Sliding window over real text files
   gsm8k       GSM8K math problems with streaming eval`,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if jsonOnly {
+				slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil)))
+			}
+
 			// Parse config early — needed to resolve --workers auto
 			sc, err := config.Parse(cfgInput)
 			if err != nil {
@@ -152,8 +158,10 @@ Workload types:
 			}
 
 			if summary.TotalRequests > 0 {
-				fmt.Fprint(os.Stderr, "\n")
-				fmt.Fprint(os.Stderr, analysis.FormatSummary(summary))
+				if !jsonOnly {
+					fmt.Fprint(os.Stderr, "\n")
+					fmt.Fprint(os.Stderr, analysis.FormatSummary(summary))
+				}
 
 				jsonOut, err := json.MarshalIndent(summary, "", "  ")
 				if err != nil {
@@ -173,6 +181,7 @@ Workload types:
 	cmd.Flags().IntVar(&workerID, "worker-id", 0, "Worker identifier (for multi-container runs)")
 	cmd.Flags().StringVar(&workersFlag, "workers", "1", `Number of workers: integer or "auto" (auto = ceil(max_concurrency/1024))`)
 	cmd.Flags().StringVar(&metricsAddr, "metrics", "", "Prometheus metrics listen address (e.g. :9090)")
+	cmd.Flags().BoolVar(&jsonOnly, "json", false, "Output only the JSON summary (suppress logs and human-readable output)")
 
 	kube.RegisterFlags(cmd, &kubeFlags)
 

pkg/analysis/stats.go

@@ -38,9 +38,24 @@ type Summary struct {
 	EvalIncorrect int     `json:"eval_incorrect,omitempty"`
 	EvalAccuracy  float64 `json:"eval_accuracy,omitempty"`
 
+	Stages []StageSummary `json:"stages,omitempty"`
+
 	Timestamps *recorder.Timestamps `json:"timestamps,omitempty"`
 }
 
+// StageSummary holds per-stage statistics for multi-stage benchmarks (sweeps).
+type StageSummary struct {
+	Stage       int          `json:"stage"`
+	Concurrency int          `json:"concurrency"`
+	DurationS   float64      `json:"duration_seconds"`
+	Requests    int          `json:"requests"`
+	ErrorRate   float64      `json:"error_rate"`
+	OutputTokS  float64      `json:"output_tokens_per_second"`
+	TTFTMs      LatencyStats `json:"ttft_ms"`
+	ITLMs       LatencyStats `json:"itl_ms"`
+	E2EMs       LatencyStats `json:"e2e_latency_ms"`
+}
+
 // LatencyStats holds percentile statistics for a latency metric.
 type LatencyStats struct {
 	Mean float64 `json:"mean"`
@@ -215,6 +230,34 @@ func Compute(records []recorder.Record, startTime, endTime float64) *Summary {
 	return s
 }
 
+// ComputeStages computes per-stage summaries by filtering records to each stage's time window.
+func ComputeStages(records []recorder.Record, stages []recorder.StageTimestamp) []StageSummary {
+	if len(stages) == 0 {
+		return nil
+	}
+
+	result := make([]StageSummary, 0, len(stages))
+	for _, st := range stages {
+		s := Compute(records, st.StartTime, st.EndTime)
+		errRate := 0.0
+		if s.TotalRequests > 0 {
+			errRate = float64(s.ErrorRequests) / float64(s.TotalRequests)
+		}
+		result = append(result, StageSummary{
+			Stage:       st.Stage,
+			Concurrency: st.Concurrency,
+			DurationS:   st.EndTime - st.StartTime,
+			Requests:    s.TotalRequests,
+			ErrorRate:   errRate,
+			OutputTokS:  s.OutputTokensPerS,
+			TTFTMs:      s.TTFTMs,
+			ITLMs:       s.ITLMs,
+			E2EMs:       s.E2EMs,
+		})
+	}
+	return result
+}
+
 func computeLatencyStats(values []float64) LatencyStats {
 	if len(values) == 0 {
 		return LatencyStats{}

pkg/analysis/stats_test.go

@@ -129,6 +129,62 @@ func TestLoadRecordsAndTimestamps(t *testing.T) {
 	}
 }
 
+func TestComputeStages(t *testing.T) {
+	records := []recorder.Record{
+		// Stage 0 (c=10, 100-110s)
+		{RequestID: "s0-r1", ConversationID: "c1", StartTime: 101.0, EndTime: 103.0,
+			TTFT: 50.0, ITLs: []float64{10.0}, TotalLatencyMs: 2000, OutputTokens: 100, Status: "ok"},
+		{RequestID: "s0-r2", ConversationID: "c2", StartTime: 102.0, EndTime: 105.0,
+			TTFT: 40.0, ITLs: []float64{8.0}, TotalLatencyMs: 3000, OutputTokens: 150, Status: "ok"},
+		// Stage 1 (c=20, 110-120s)
+		{RequestID: "s1-r1", ConversationID: "c3", StartTime: 111.0, EndTime: 114.0,
+			TTFT: 30.0, ITLs: []float64{5.0}, TotalLatencyMs: 3000, OutputTokens: 200, Status: "ok"},
+		{RequestID: "s1-r2", ConversationID: "c4", StartTime: 112.0, EndTime: 115.0,
+			TTFT: 35.0, ITLs: []float64{6.0}, TotalLatencyMs: 3000, OutputTokens: 180, Status: "ok"},
+		{RequestID: "s1-err", ConversationID: "c5", StartTime: 113.0, EndTime: 114.5,
+			Status: "error", Error: "timeout"},
+	}
+
+	stages := []recorder.StageTimestamp{
+		{Stage: 0, Concurrency: 10, StartTime: 100.0, EndTime: 110.0},
+		{Stage: 1, Concurrency: 20, StartTime: 110.0, EndTime: 120.0},
+	}
+
+	result := analysis.ComputeStages(records, stages)
+	if len(result) != 2 {
+		t.Fatalf("expected 2 stages, got %d", len(result))
+	}
+
+	// Stage 0
+	if result[0].Concurrency != 10 {
+		t.Errorf("stage 0: expected concurrency 10, got %d", result[0].Concurrency)
+	}
+	if result[0].Requests != 2 {
+		t.Errorf("stage 0: expected 2 requests, got %d", result[0].Requests)
+	}
+	if result[0].TTFTMs.Mean != 45.0 {
+		t.Errorf("stage 0: expected TTFT mean 45.0, got %.1f", result[0].TTFTMs.Mean)
+	}
+
+	// Stage 1
+	if result[1].Concurrency != 20 {
+		t.Errorf("stage 1: expected concurrency 20, got %d", result[1].Concurrency)
+	}
+	if result[1].Requests != 3 {
+		t.Errorf("stage 1: expected 3 requests, got %d", result[1].Requests)
+	}
+	if result[1].ErrorRate != 1.0/3.0 {
+		t.Errorf("stage 1: expected error rate 0.333, got %.3f", result[1].ErrorRate)
+	}
+}
+
+func TestComputeStagesEmpty(t *testing.T) {
+	result := analysis.ComputeStages(nil, nil)
+	if result != nil {
+		t.Errorf("expected nil for empty stages, got %v", result)
+	}
+}
+
 func TestFormatSummary(t *testing.T) {
 	s := &analysis.Summary{
 		TotalRequests:   100,

pkg/kube/deploy.go

@@ -376,7 +376,13 @@ func CollectArgs(cmd *cobra.Command, prefix []string) []string {
 		if strings.HasPrefix(f.Name, "kube") {
 			return
 		}
-		args = append(args, "--"+f.Name, f.Value.String())
+		if f.Value.Type() == "bool" {
+			if f.Value.String() == "true" {
+				args = append(args, "--"+f.Name)
+			}
+		} else {
+			args = append(args, "--"+f.Name, f.Value.String())
+		}
 	})
 	return args
 }