fix: prevent orphan VMM via socket tiebreaker + status one-shot default

Closes #48 and #49 — both were triggers for orphan cocoon/CH processes
observed on cocoonset-gke-private (8 alive CH/FC without DB entries,
plus 4-day-old cocoon-CLI watchers from prior vk-cocoon restarts).

#49: vm rm --force could clean the DB but leave CH running when the
pidfile/cmdline check returned a false negative (stale pidfile, recycled
PID, manual rundir tampering). Add Backend.IsAPISocketLive as a
pidfile-independent tiebreaker (utils.CheckSocket) and call it in
DeleteAll after the stop step. Fires unconditionally — AF_UNIX has no
TIME_WAIT and TerminateProcess only returns nil after full pid reap, so
a successful stop guarantees the listener is gone; firing also catches
the pidfd_linux.go's own VerifyProcessCmdline false-negative path that
otherwise returns (true, nil) without sending a signal.

#48: cocoon vm status defaulted to a 5s polling loop. When invoked from
a shell that disconnected without SIGHUP propagation (sudo wrapper,
bash -c pipeline, tmux killed without huponexit), the loop survived
indefinitely — observed 21-day-old orphans on prod. Add --watch flag;
default is one-shot snapshot via statusOnce. --event still implies
streaming events. statusOnce + List share a renderVMList helper so the
JSON/table/empty-list branch lives in one place.

Author: CMGS

cmd/vm/commands.go

@@ -165,12 +165,13 @@ func Command(h Actions) *cobra.Command {
 
 	statusCmd := &cobra.Command{
 		Use:   "status [VM...]",
-		Short: "Watch VM status in real time",
+		Short: "Show VM status; --watch for refresh loop, --event for streaming",
 		RunE:  h.Status,
 	}
-	statusCmd.Flags().IntP("interval", "n", 5, "poll interval in seconds") //nolint:mnd
-	statusCmd.Flags().Bool("event", false, "event stream mode (append changes instead of refreshing)")
-	statusCmd.Flags().String("format", "", "output format: json (event mode only)")
+	statusCmd.Flags().IntP("interval", "n", 5, "poll interval in seconds (only with --watch or --event)") //nolint:mnd
+	statusCmd.Flags().BoolP("watch", "w", false, "refresh-loop mode (full-screen redraw each tick); omit for one-shot snapshot")
+	statusCmd.Flags().Bool("event", false, "event stream mode (append changes instead of refreshing); implies polling")
+	statusCmd.Flags().String("format", "", "output format: json (one-shot + event modes; --watch always renders a table)")
 
 	vmCmd.AddCommand(
 		createCmd,

cmd/vm/status.go

@@ -56,16 +56,23 @@ func (h Handler) List(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return fmt.Errorf("list: %w", err)
 	}
+	sortVMs(vms)
+	format, _ := cmd.Flags().GetString("format")
+	return renderVMList(vms, format)
+}
+
+// renderVMList emits vms as JSON or table; "No VMs found." for empty in table mode.
+func renderVMList(vms []*types.VM, format string) error {
+	if format == "json" {
+		return cmdcore.OutputJSON(vms)
+	}
 	if len(vms) == 0 {
 		fmt.Println("No VMs found.")
 		return nil
 	}
-
-	sortVMs(vms)
-
-	return cmdcore.OutputFormatted(cmd, vms, func(w *tabwriter.Writer) {
-		printVMTable(w, vms)
-	})
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	printVMTable(w, vms)
+	return w.Flush()
 }
 
 func (h Handler) Status(cmd *cobra.Command, args []string) error {
@@ -79,19 +86,22 @@ func (h Handler) Status(cmd *cobra.Command, args []string) error {
 		interval = 5 //nolint:mnd
 	}
 	eventMode, _ := cmd.Flags().GetBool("event")
+	watchMode, _ := cmd.Flags().GetBool("watch")
+	format, _ := cmd.Flags().GetString("format")
 
 	hypers, hyperErr := cmdcore.InitAllHypervisors(conf)
 	if hyperErr != nil {
 		return hyperErr
 	}
 
-	watchCh := mergeWatchChannels(ctx, hypers)
+	if !eventMode && !watchMode {
+		return statusOnce(ctx, hypers, args, format)
+	}
 
+	watchCh := mergeWatchChannels(ctx, hypers)
 	ticker := time.NewTicker(time.Duration(interval) * time.Second)
 	defer ticker.Stop()
 
-	format, _ := cmd.Flags().GetString("format")
-
 	if eventMode {
 		if format == "json" {
 			statusEventLoopJSON(ctx, hypers, args, watchCh, ticker.C)
@@ -105,6 +115,17 @@ func (h Handler) Status(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+// statusOnce prints a single snapshot then returns; propagates ListAllVMs error (loop callers swallow).
+func statusOnce(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string, format string) error {
+	vms, err := cmdcore.ListAllVMs(ctx, hypers)
+	if err != nil {
+		return fmt.Errorf("status: %w", err)
+	}
+	vms = applyFilters(vms, filters)
+	sortVMs(vms)
+	return renderVMList(vms, format)
+}
+
 func mergeWatchChannels(ctx context.Context, hypers []hypervisor.Hypervisor) <-chan struct{} {
 	var channels []<-chan struct{}
 	for _, h := range hypers {
@@ -266,23 +287,28 @@ func printEventRow(w *tabwriter.Writer, event string, snap vmSnapshot) {
 		snap.ip, snap.image)
 }
 
+// listAndFilter swallows backend errors with a warn so a transient hiccup can't break the polling tick; one-shot callers must use cmdcore.ListAllVMs directly.
 func listAndFilter(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string) []*types.VM {
 	vms, err := cmdcore.ListAllVMs(ctx, hypers)
 	if err != nil {
 		log.WithFunc("cmd.vm.listAndFilter").Warnf(ctx, "list: %v", err)
 		return nil
 	}
 	sortVMs(vms)
+	return applyFilters(vms, filters)
+}
+
+func applyFilters(vms []*types.VM, filters []string) []*types.VM {
 	if len(filters) == 0 {
 		return vms
 	}
-	var result []*types.VM
+	out := make([]*types.VM, 0, len(vms))
 	for _, vm := range vms {
 		if matchesFilter(vm, filters) {
-			result = append(result, vm)
+			out = append(out, vm)
 		}
 	}
-	return result
+	return out
 }
 
 func matchesFilter(vm *types.VM, filters []string) bool {

hypervisor/state.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"syscall"
 	"time"
 
 	"github.com/projecteru2/core/log"
@@ -25,6 +26,20 @@ func (b *Backend) WithRunningVM(ctx context.Context, rec *VMRecord, fn func(pid
 	return fn(pid)
 }
 
+// IsAPISocketLive is a pidfile-independent liveness tiebreaker; fails closed on uncertain dial errors.
+func (b *Backend) IsAPISocketLive(ctx context.Context, rec *VMRecord) bool {
+	sock := SocketPath(rec.RunDir)
+	err := utils.CheckSocket(sock)
+	if err == nil {
+		return true
+	}
+	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ECONNREFUSED) {
+		return false
+	}
+	log.WithFunc(b.Typ+".IsAPISocketLive").Warnf(ctx, "uncertain socket probe %s: %v — failing closed", sock, err)
+	return true
+}
+
 // WithPausedVM pauses, runs fn, resumes; eager resume on success promotes its error, deferred resume on fn-error only logs.
 func (b *Backend) WithPausedVM(ctx context.Context, rec *VMRecord, pause, resume, fn func() error) error {
 	return b.WithRunningVM(ctx, rec, func(_ int) error {

hypervisor/stop.go

@@ -64,6 +64,10 @@ func (b *Backend) DeleteAll(ctx context.Context, refs []string, force bool, stop
 		}); runningErr != nil && !errors.Is(runningErr, ErrNotRunning) {
 			return fmt.Errorf("stop before delete: %w", runningErr)
 		}
+		// Socket probe runs regardless of stop outcome: AF_UNIX has no TIME_WAIT so a real stop guarantees no listener; firing always also catches false-negative pidfile/cmdline shortcuts.
+		if b.IsAPISocketLive(ctx, &rec) {
+			return fmt.Errorf("refuse delete: api socket %s still responsive (suspected orphan vmm; kill the vmm process then retry)", SocketPath(rec.RunDir))
+		}
 		if rmErr := RemoveVMDirs(rec.RunDir, rec.LogDir); rmErr != nil {
 			return fmt.Errorf("cleanup VM dirs: %w", rmErr)
 		}