fix(#6014,#6015): real rebuild in-flight across the process boundary; dashboard stops serving a stale graph.json

internal/cli/status.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -129,130 +128,7 @@ func runStatus(w io.Writer, filter string, ref string, showAll bool) error {
 					fmt.Fprintf(w, "  mem_limit: unbounded (%s)\n", src)
 				}
 			}
-			if st.WatcherRepos > 0 || st.WatcherEvents > 0 {
-				fmt.Fprintf(w, "  watcher: repos=%d dirs=%d events=%d dropped=%d",
-					st.WatcherRepos, st.WatcherDirs, st.WatcherEvents, st.WatcherDropped)
-				// PH2a (#2096): show pause/resume slot counts when available.
-				if st.WatcherActiveSlots > 0 || st.WatcherPausedSlots > 0 {
-					fmt.Fprintf(w, " slots_active=%d slots_paused=%d",
-						st.WatcherActiveSlots, st.WatcherPausedSlots)
-				}
-				fmt.Fprintln(w)
-			}
-			if st.QueueLen > 0 || len(st.IndexInFlight) > 0 ||
-				len(st.PendingAlgo) > 0 || len(st.PendingLinks) > 0 {
-				fmt.Fprintf(w, "  scheduler: queue=%d in_flight=%d pending_algo=%d pending_links=%d\n",
-					st.QueueLen, len(st.IndexInFlight), len(st.PendingAlgo), len(st.PendingLinks))
-			}
-			printAnnotationStatus(w, st)
-			// #5954 heavy write-stage gate. Printed whenever the gate is doing
-			// ANYTHING (holding, deferring, or being barged) so an operator — or
-			// a peak-RSS measurement run — can confirm from outside the process
-			// that it fired, instead of inferring it from RSS shape. Silent when
-			// the gate is idle, which is the overwhelmingly common case.
-			if st.StageGateHolder != "" || len(st.StageGateDeferred) > 0 || len(st.StageGateBarging) > 0 ||
-				st.StageGateForfeits > 0 {
-				fmt.Fprint(w, "  stage_gate:")
-				if len(st.StageGateBarging) > 0 {
-					fmt.Fprintf(w, " barging=%s", strings.Join(st.StageGateBarging, ","))
-				}
-				if st.StageGateHolder != "" {
-					fmt.Fprintf(w, " holder=%s", st.StageGateHolder)
-					// LIVE, unlike FORFEITS below: this holder blew the 4h
-					// hold-max and the gate is inside its forfeit grace right
-					// now. It is the state an operator staring at a stalled
-					// daemon is trying to identify, so it rides on the holder.
-					if st.StageGateForfeitedHolder {
-						fmt.Fprint(w, "(FORFEITED,awaiting-cancel)")
-					}
-				}
-				if len(st.StageGateDeferred) > 0 {
-					fmt.Fprintf(w, " deferred=%s", strings.Join(st.StageGateDeferred, ","))
-				}
-				// Sticky: a forfeit is a failure, so it stays on the line for
-				// the life of the daemon rather than vanishing with the holder.
-				if st.StageGateForfeits > 0 {
-					fmt.Fprintf(w, " FORFEITS=%d", st.StageGateForfeits)
-				}
-				fmt.Fprintln(w)
-			}
-			if st.RSSBudgetMB > 0 {
-				// Two separate lines: daemon idle RSS (informational) vs.
-				// admission budget (delta-based predicted in-flight sum).
-				// These are intentionally distinct — idle RSS can exceed the
-				// budget without blocking jobs, because jobs are only blocked
-				// when sum(predicted_in_flight) + new_job_pred > budget.
-				// #3648: report the honest process footprint (resident set
-				// size), not the old MemStats.Sys mislabel. On macOS RSS
-				// under-counts swapped/compressed pages — note that, plus the
-				// Go-heap breakdown, so the number is interpretable.
-				fmt.Fprintf(w, "  mem: footprint=%dMB heap_inuse=%dMB heap_released=%dMB go_sys=%dMB\n",
-					st.RSSUsedMB,
-					st.HeapInuseBytes/(1024*1024),
-					st.HeapReleasedBytes/(1024*1024),
-					st.SysBytes/(1024*1024))
-				if st.FootprintLabel != "" {
-					fmt.Fprintf(w, "       (footprint = %s)\n", st.FootprintLabel)
-				}
-				admHeadroom := st.RSSBudgetMB - st.AdmissionUsedMB
-				if admHeadroom < 0 {
-					admHeadroom = 0
-				}
-				fmt.Fprintf(w, "  admission: queued=%d admitted=%d predicted=%dMB / budget=%dMB (headroom=%dMB)\n",
-					len(st.BlockedJobs), len(st.InFlightJobs),
-					st.AdmissionUsedMB, st.RSSBudgetMB, admHeadroom)
-				if st.RebuildConcurrencyCap > 0 {
-					fmt.Fprintf(w, "  rebuild: in_flight=%d / cap=%d\n",
-						st.RebuildInFlight, st.RebuildConcurrencyCap)
-				}
-				if len(st.InFlightJobs) > 0 {
-					for _, j := range st.InFlightJobs {
-						fmt.Fprintf(w, "    admitted: %s (predicted=%dMB)\n", j.Path, j.PredictedMB)
-					}
-				}
-				if len(st.BlockedJobs) > 0 {
-					for _, p := range st.BlockedJobs {
-						fmt.Fprintf(w, "    queued:   %s\n", p)
-					}
-				}
-			}
-			if len(st.IndexedRepos) > 0 {
-				fmt.Fprintln(w, "  indexed repos:")
-				for _, r := range st.IndexedRepos {
-					last := r.LastIndex
-					if last == "" {
-						last = "(never)"
-					}
-					fmt.Fprintf(w, "    %s  last_index=%s  indexes=%d  algos=%d",
-						r.Path, last, r.IndexCount, r.AlgoCount)
-					if r.LastErr != "" {
-						fmt.Fprintf(w, "  err=%s", r.LastErr)
-					}
-					// #5727/#5729-W1: show the exact indexed commit + whether
-					// the on-disk graph still matches HEAD.
-					if r.IndexedCommitShort != "" {
-						fmt.Fprintf(w, "  commit=%s at_head=%v", r.IndexedCommitShort, r.AtHead)
-					}
-					fmt.Fprintln(w)
-				}
-			}
-			if n := len(st.RecentLog); n > 0 {
-				start := n - 5
-				if start < 0 {
-					start = 0
-				}
-				fmt.Fprintln(w, "  recent events:")
-				for _, e := range st.RecentLog[start:] {
-					line := fmt.Sprintf("    %s  %s", e.Time, e.Kind)
-					if e.Repo != "" {
-						line += "  " + e.Repo
-					}
-					if e.Msg != "" {
-						line += "  " + e.Msg
-					}
-					fmt.Fprintln(w, line)
-				}
-			}
+			printDaemonDetail(w, st)
 		}
 	case errors.Is(err, client.ErrDaemonNotRunning):
 		fmt.Fprintln(w, "Daemon: not running")

internal/cli/status_daemon_detail.go

@@ -0,0 +1,157 @@
+package cli
+
+// status_daemon_detail.go -- the daemon-section renderer for `grafel status`,
+// split out of runStatus so the print gates are unit-testable without dialing a
+// real daemon (#6014). runStatus itself needs a live client.Dial, which put
+// every one of these conditionals out of reach of a test -- and an unreachable
+// print gate is precisely how #6014's engine-sourced rebuild count came to be
+// computed correctly and then never shown to anyone.
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/cajasmota/grafel/internal/daemon/proto"
+)
+
+// printDaemonDetail renders the per-subsystem detail lines of the daemon
+// section: watcher, scheduler, annotation pass, stage gate, rebuild counters,
+// memory/admission, indexed repos and the recent-event tail. Every line is
+// individually gated on its subsystem being present or busy, so an idle daemon
+// prints only the header lines runStatus emits directly.
+func printDaemonDetail(w io.Writer, st proto.StatusReply) {
+	if st.WatcherRepos > 0 || st.WatcherEvents > 0 {
+		fmt.Fprintf(w, "  watcher: repos=%d dirs=%d events=%d dropped=%d",
+			st.WatcherRepos, st.WatcherDirs, st.WatcherEvents, st.WatcherDropped)
+		// PH2a (#2096): show pause/resume slot counts when available.
+		if st.WatcherActiveSlots > 0 || st.WatcherPausedSlots > 0 {
+			fmt.Fprintf(w, " slots_active=%d slots_paused=%d",
+				st.WatcherActiveSlots, st.WatcherPausedSlots)
+		}
+		fmt.Fprintln(w)
+	}
+	if st.QueueLen > 0 || len(st.IndexInFlight) > 0 ||
+		len(st.PendingAlgo) > 0 || len(st.PendingLinks) > 0 {
+		fmt.Fprintf(w, "  scheduler: queue=%d in_flight=%d pending_algo=%d pending_links=%d\n",
+			st.QueueLen, len(st.IndexInFlight), len(st.PendingAlgo), len(st.PendingLinks))
+	}
+	printAnnotationStatus(w, st)
+	// #5954 heavy write-stage gate. Printed whenever the gate is doing
+	// ANYTHING (holding, deferring, or being barged) so an operator — or
+	// a peak-RSS measurement run — can confirm from outside the process
+	// that it fired, instead of inferring it from RSS shape. Silent when
+	// the gate is idle, which is the overwhelmingly common case.
+	if st.StageGateHolder != "" || len(st.StageGateDeferred) > 0 || len(st.StageGateBarging) > 0 ||
+		st.StageGateForfeits > 0 {
+		fmt.Fprint(w, "  stage_gate:")
+		if len(st.StageGateBarging) > 0 {
+			fmt.Fprintf(w, " barging=%s", strings.Join(st.StageGateBarging, ","))
+		}
+		if st.StageGateHolder != "" {
+			fmt.Fprintf(w, " holder=%s", st.StageGateHolder)
+			// LIVE, unlike FORFEITS below: this holder blew the 4h
+			// hold-max and the gate is inside its forfeit grace right
+			// now. It is the state an operator staring at a stalled
+			// daemon is trying to identify, so it rides on the holder.
+			if st.StageGateForfeitedHolder {
+				fmt.Fprint(w, "(FORFEITED,awaiting-cancel)")
+			}
+		}
+		if len(st.StageGateDeferred) > 0 {
+			fmt.Fprintf(w, " deferred=%s", strings.Join(st.StageGateDeferred, ","))
+		}
+		// Sticky: a forfeit is a failure, so it stays on the line for
+		// the life of the daemon rather than vanishing with the holder.
+		if st.StageGateForfeits > 0 {
+			fmt.Fprintf(w, " FORFEITS=%d", st.StageGateForfeits)
+		}
+		fmt.Fprintln(w)
+	}
+	// #6014: the rebuild counters are printed OUTSIDE the RSSBudgetMB block
+	// below. RSSBudgetMB is populated only from an in-process scheduler
+	// (Service.Status' `if s.scheduler != nil` branch), and in SPLIT MODE -- the
+	// default since ADR-0024 -- serve has no scheduler, so it is 0 and that
+	// whole block never prints. Nesting this line there made the engine-sourced
+	// rebuild count unreachable on the one surface an operator actually reads:
+	// the number crossed the process boundary correctly and was then discarded
+	// by the print gate. RebuildConcurrencyCap is set unconditionally by
+	// Service.Status, so gating on it alone is both sufficient and mode-neutral.
+	if st.RebuildConcurrencyCap > 0 {
+		fmt.Fprintf(w, "  rebuild: in_flight=%d / cap=%d\n",
+			st.RebuildInFlight, st.RebuildConcurrencyCap)
+	}
+	if st.RSSBudgetMB > 0 {
+		// Two separate lines: daemon idle RSS (informational) vs.
+		// admission budget (delta-based predicted in-flight sum).
+		// These are intentionally distinct — idle RSS can exceed the
+		// budget without blocking jobs, because jobs are only blocked
+		// when sum(predicted_in_flight) + new_job_pred > budget.
+		// #3648: report the honest process footprint (resident set
+		// size), not the old MemStats.Sys mislabel. On macOS RSS
+		// under-counts swapped/compressed pages — note that, plus the
+		// Go-heap breakdown, so the number is interpretable.
+		fmt.Fprintf(w, "  mem: footprint=%dMB heap_inuse=%dMB heap_released=%dMB go_sys=%dMB\n",
+			st.RSSUsedMB,
+			st.HeapInuseBytes/(1024*1024),
+			st.HeapReleasedBytes/(1024*1024),
+			st.SysBytes/(1024*1024))
+		if st.FootprintLabel != "" {
+			fmt.Fprintf(w, "       (footprint = %s)\n", st.FootprintLabel)
+		}
+		admHeadroom := st.RSSBudgetMB - st.AdmissionUsedMB
+		if admHeadroom < 0 {
+			admHeadroom = 0
+		}
+		fmt.Fprintf(w, "  admission: queued=%d admitted=%d predicted=%dMB / budget=%dMB (headroom=%dMB)\n",
+			len(st.BlockedJobs), len(st.InFlightJobs),
+			st.AdmissionUsedMB, st.RSSBudgetMB, admHeadroom)
+		if len(st.InFlightJobs) > 0 {
+			for _, j := range st.InFlightJobs {
+				fmt.Fprintf(w, "    admitted: %s (predicted=%dMB)\n", j.Path, j.PredictedMB)
+			}
+		}
+		if len(st.BlockedJobs) > 0 {
+			for _, p := range st.BlockedJobs {
+				fmt.Fprintf(w, "    queued:   %s\n", p)
+			}
+		}
+	}
+	if len(st.IndexedRepos) > 0 {
+		fmt.Fprintln(w, "  indexed repos:")
+		for _, r := range st.IndexedRepos {
+			last := r.LastIndex
+			if last == "" {
+				last = "(never)"
+			}
+			fmt.Fprintf(w, "    %s  last_index=%s  indexes=%d  algos=%d",
+				r.Path, last, r.IndexCount, r.AlgoCount)
+			if r.LastErr != "" {
+				fmt.Fprintf(w, "  err=%s", r.LastErr)
+			}
+			// #5727/#5729-W1: show the exact indexed commit + whether
+			// the on-disk graph still matches HEAD.
+			if r.IndexedCommitShort != "" {
+				fmt.Fprintf(w, "  commit=%s at_head=%v", r.IndexedCommitShort, r.AtHead)
+			}
+			fmt.Fprintln(w)
+		}
+	}
+	if n := len(st.RecentLog); n > 0 {
+		start := n - 5
+		if start < 0 {
+			start = 0
+		}
+		fmt.Fprintln(w, "  recent events:")
+		for _, e := range st.RecentLog[start:] {
+			line := fmt.Sprintf("    %s  %s", e.Time, e.Kind)
+			if e.Repo != "" {
+				line += "  " + e.Repo
+			}
+			if e.Msg != "" {
+				line += "  " + e.Msg
+			}
+			fmt.Fprintln(w, line)
+		}
+	}
+}

internal/cli/status_daemon_detail_6014_test.go

@@ -0,0 +1,81 @@
+package cli
+
+// status_daemon_detail_6014_test.go — #6014, the print gate.
+//
+// The engine-sourced rebuild count reached StatusReply.RebuildInFlight
+// correctly and was then thrown away by the renderer: the `rebuild:` line lived
+// inside `if st.RSSBudgetMB > 0`, and RSSBudgetMB is populated only from an
+// IN-PROCESS scheduler (Service.Status' `if s.scheduler != nil` branch). In
+// split mode — the default since ADR-0024 — serve has no scheduler, so
+// RSSBudgetMB is 0 and the whole block, rebuild line included, never printed.
+// `grafel status` showed no `rebuild:` line at all.
+//
+// That is the same shape as the bug being fixed: a number that is right on the
+// wire and absent on the screen. So the gate is pinned here, in the one
+// configuration that matters — no scheduler, no RSS budget.
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/cajasmota/grafel/internal/daemon/proto"
+)
+
+// TestPrintDaemonDetail_RebuildLinePrintsWithNoRSSBudget is the regression: a
+// split-mode serve reply (scheduler-derived fields all zero) must still show
+// the rebuild counters. Re-nesting the line under RSSBudgetMB kills this test.
+func TestPrintDaemonDetail_RebuildLinePrintsWithNoRSSBudget(t *testing.T) {
+	var buf bytes.Buffer
+	printDaemonDetail(&buf, proto.StatusReply{
+		// Split-mode serve: no scheduler, so every scheduler-sourced field is
+		// zero — RSSBudgetMB above all.
+		RSSBudgetMB:           0,
+		RebuildInFlight:       1,
+		RebuildConcurrencyCap: 2,
+	})
+	out := buf.String()
+
+	if !strings.Contains(out, "rebuild: in_flight=1 / cap=2") {
+		t.Errorf("`rebuild:` line missing from `grafel status` output with RSSBudgetMB=0.\n"+
+			"This is the DEFAULT deployment mode: serve has no scheduler, so a rebuild line gated on the "+
+			"RSS-budget block is unreachable and the engine-sourced count is computed, transmitted, and "+
+			"then discarded before anyone sees it.\ngot:\n%s", out)
+	}
+	// Guard the hoist rather than the copy: the RSS/admission block must still
+	// be suppressed when there is no budget to report.
+	if strings.Contains(out, "admission:") {
+		t.Errorf("admission block printed with RSSBudgetMB=0; the rebuild line should have been hoisted "+
+			"OUT of that block, not the block un-gated.\ngot:\n%s", out)
+	}
+}
+
+// TestPrintDaemonDetail_RebuildLineStillPrintsWithRSSBudget: the monolith path
+// (scheduler attached, budget reported) must be unchanged by the hoist.
+func TestPrintDaemonDetail_RebuildLineStillPrintsWithRSSBudget(t *testing.T) {
+	var buf bytes.Buffer
+	printDaemonDetail(&buf, proto.StatusReply{
+		RSSBudgetMB:           4096,
+		RebuildInFlight:       3,
+		RebuildConcurrencyCap: 4,
+	})
+	out := buf.String()
+	if !strings.Contains(out, "rebuild: in_flight=3 / cap=4") {
+		t.Errorf("`rebuild:` line missing in monolith mode (RSSBudgetMB > 0).\ngot:\n%s", out)
+	}
+	if !strings.Contains(out, "admission:") {
+		t.Errorf("admission block missing with RSSBudgetMB > 0 — the hoist changed monolith output.\ngot:\n%s", out)
+	}
+}
+
+// TestPrintDaemonDetail_NoRebuildLineWithoutCap: RebuildConcurrencyCap is set
+// unconditionally by Service.Status, so a zero means the reply did not come
+// from a daemon that reports it (an old daemon, or a zero-valued fixture).
+// Printing `cap=0` there would be noise, not information.
+func TestPrintDaemonDetail_NoRebuildLineWithoutCap(t *testing.T) {
+	var buf bytes.Buffer
+	printDaemonDetail(&buf, proto.StatusReply{})
+	if out := buf.String(); strings.Contains(out, "rebuild:") {
+		t.Errorf("`rebuild:` line printed with RebuildConcurrencyCap=0.\ngot:\n%s", out)
+	}
+}