fix: prevent orphan VMM via socket tiebreaker + status one-shot default

Closes #48 and #49 — both were triggers for orphan cocoon/CH processes
observed on cocoonset-gke-private (8 alive CH/FC without DB entries,
plus 4-day-old cocoon-CLI watchers from prior vk-cocoon restarts).

#49: vm rm --force could clean the DB but leave CH running when the
pidfile/cmdline check returned a false negative (stale pidfile, recycled
PID, manual rundir tampering). Add Backend.IsAPISocketLive as a
pidfile-independent tiebreaker (utils.CheckSocket) and call it in
DeleteAll after the stop step. Fires unconditionally — AF_UNIX has no
TIME_WAIT and TerminateProcess only returns nil after full pid reap, so
a successful stop guarantees the listener is gone; firing also catches
the pidfd_linux.go's own VerifyProcessCmdline false-negative path that
otherwise returns (true, nil) without sending a signal.

#48: cocoon vm status defaulted to a 5s polling loop. When invoked from
a shell that disconnected without SIGHUP propagation (sudo wrapper,
bash -c pipeline, tmux killed without huponexit), the loop survived
indefinitely — observed 21-day-old orphans on prod. Add --watch flag;
default is one-shot snapshot via statusOnce. --event still implies
streaming events. statusOnce + List share a renderVMList helper so the
JSON/table/empty-list branch lives in one place.

Author: CMGS

cmd/vm/commands.go

@@ -165,12 +165,13 @@ func Command(h Actions) *cobra.Command {
 
 	statusCmd := &cobra.Command{
 		Use:   "status [VM...]",
-		Short: "Watch VM status in real time",
+		Short: "Show VM status; --watch for refresh loop, --event for streaming",
 		RunE:  h.Status,
 	}
-	statusCmd.Flags().IntP("interval", "n", 5, "poll interval in seconds") //nolint:mnd
-	statusCmd.Flags().Bool("event", false, "event stream mode (append changes instead of refreshing)")
-	statusCmd.Flags().String("format", "", "output format: json (event mode only)")
+	statusCmd.Flags().IntP("interval", "n", 5, "poll interval in seconds (only with --watch or --event)") //nolint:mnd
+	statusCmd.Flags().BoolP("watch", "w", false, "refresh-loop mode (full-screen redraw each tick); omit for one-shot snapshot")
+	statusCmd.Flags().Bool("event", false, "event stream mode (append changes instead of refreshing); implies polling")
+	statusCmd.Flags().String("format", "", "output format: json (one-shot + event modes; --watch always renders a table)")
 
 	vmCmd.AddCommand(
 		createCmd,

cmd/vm/status.go

@@ -56,16 +56,23 @@ func (h Handler) List(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return fmt.Errorf("list: %w", err)
 	}
+	sortVMs(vms)
+	format, _ := cmd.Flags().GetString("format")
+	return renderVMList(vms, format)
+}
+
+// renderVMList emits vms as JSON or table; "No VMs found." for empty in table mode.
+func renderVMList(vms []*types.VM, format string) error {
+	if format == "json" {
+		return cmdcore.OutputJSON(vms)
+	}
 	if len(vms) == 0 {
 		fmt.Println("No VMs found.")
 		return nil
 	}
-
-	sortVMs(vms)
-
-	return cmdcore.OutputFormatted(cmd, vms, func(w *tabwriter.Writer) {
-		printVMTable(w, vms)
-	})
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	printVMTable(w, vms)
+	return w.Flush()
 }
 
 func (h Handler) Status(cmd *cobra.Command, args []string) error {
@@ -79,19 +86,25 @@ func (h Handler) Status(cmd *cobra.Command, args []string) error {
 		interval = 5 //nolint:mnd
 	}
 	eventMode, _ := cmd.Flags().GetBool("event")
+	watchMode, _ := cmd.Flags().GetBool("watch")
+	if eventMode && watchMode {
+		return fmt.Errorf("--event and --watch are mutually exclusive")
+	}
+	format, _ := cmd.Flags().GetString("format")
 
 	hypers, hyperErr := cmdcore.InitAllHypervisors(conf)
 	if hyperErr != nil {
 		return hyperErr
 	}
 
-	watchCh := mergeWatchChannels(ctx, hypers)
+	if !eventMode && !watchMode {
+		return statusOnce(ctx, hypers, args, format)
+	}
 
+	watchCh := mergeWatchChannels(ctx, hypers)
 	ticker := time.NewTicker(time.Duration(interval) * time.Second)
 	defer ticker.Stop()
 
-	format, _ := cmd.Flags().GetString("format")
-
 	if eventMode {
 		if format == "json" {
 			statusEventLoopJSON(ctx, hypers, args, watchCh, ticker.C)
@@ -105,6 +118,17 @@ func (h Handler) Status(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+// statusOnce prints a single snapshot then returns; propagates ListAllVMs error (loop callers swallow).
+func statusOnce(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string, format string) error {
+	vms, err := cmdcore.ListAllVMs(ctx, hypers)
+	if err != nil {
+		return fmt.Errorf("status: %w", err)
+	}
+	vms = applyFilters(vms, filters)
+	sortVMs(vms)
+	return renderVMList(vms, format)
+}
+
 func mergeWatchChannels(ctx context.Context, hypers []hypervisor.Hypervisor) <-chan struct{} {
 	var channels []<-chan struct{}
 	for _, h := range hypers {
@@ -266,23 +290,28 @@ func printEventRow(w *tabwriter.Writer, event string, snap vmSnapshot) {
 		snap.ip, snap.image)
 }
 
+// listAndFilter swallows backend errors with a warn so a transient hiccup can't break the polling tick; one-shot callers must use cmdcore.ListAllVMs directly.
 func listAndFilter(ctx context.Context, hypers []hypervisor.Hypervisor, filters []string) []*types.VM {
 	vms, err := cmdcore.ListAllVMs(ctx, hypers)
 	if err != nil {
 		log.WithFunc("cmd.vm.listAndFilter").Warnf(ctx, "list: %v", err)
 		return nil
 	}
 	sortVMs(vms)
+	return applyFilters(vms, filters)
+}
+
+func applyFilters(vms []*types.VM, filters []string) []*types.VM {
 	if len(filters) == 0 {
 		return vms
 	}
-	var result []*types.VM
+	out := make([]*types.VM, 0, len(vms))
 	for _, vm := range vms {
 		if matchesFilter(vm, filters) {
-			result = append(result, vm)
+			out = append(out, vm)
 		}
 	}
-	return result
+	return out
 }
 
 func matchesFilter(vm *types.VM, filters []string) bool {

cmd/vm/status_test.go

@@ -1,12 +1,31 @@
 package vm
 
 import (
+	"io"
+	"os"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/cocoonstack/cocoon/types"
 )
 
+// captureStdout swaps os.Stdout for a pipe, runs fn, returns what fn wrote.
+func captureStdout(t *testing.T, fn func()) string {
+	t.Helper()
+	orig := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatalf("pipe: %v", err)
+	}
+	os.Stdout = w
+	fn()
+	_ = w.Close()
+	os.Stdout = orig
+	buf, _ := io.ReadAll(r)
+	return string(buf)
+}
+
 func TestMatchesFilter(t *testing.T) {
 	vm := &types.VM{
 		ID: "abcdef123456",
@@ -85,3 +104,83 @@ func TestVMIPsAndSort(t *testing.T) {
 		t.Fatalf("takeSnapshot() = %+v", snap)
 	}
 }
+
+func TestRenderVMList(t *testing.T) {
+	vm := &types.VM{
+		ID:        "abc",
+		Config:    types.VMConfig{Name: "demo", Config: types.Config{CPU: 1, Memory: 1 << 30, Image: "img"}},
+		CreatedAt: time.Now(),
+	}
+
+	tests := []struct {
+		name    string
+		vms     []*types.VM
+		format  string
+		want    string
+		notWant string
+	}{
+		{name: "empty table → No VMs found", vms: nil, format: "", want: "No VMs found."},
+		{name: "empty json → null", vms: nil, format: "json", want: "null"},
+		{name: "table with vm contains name", vms: []*types.VM{vm}, format: "", want: "demo"},
+		{name: "json with vm contains id", vms: []*types.VM{vm}, format: "json", want: `"id": "abc"`},
+		{name: "table mode skips json marker", vms: []*types.VM{vm}, format: "", notWant: `"id":`},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out := captureStdout(t, func() {
+				if err := renderVMList(tt.vms, tt.format); err != nil {
+					t.Fatalf("renderVMList: %v", err)
+				}
+			})
+			if tt.want != "" && !strings.Contains(out, tt.want) {
+				t.Errorf("output %q does not contain %q", out, tt.want)
+			}
+			if tt.notWant != "" && strings.Contains(out, tt.notWant) {
+				t.Errorf("output %q unexpectedly contains %q", out, tt.notWant)
+			}
+		})
+	}
+}
+
+func TestApplyFilters(t *testing.T) {
+	vms := []*types.VM{
+		{ID: "abcdef123456", Config: types.VMConfig{Name: "alpha"}},
+		{ID: "beadbeef0000", Config: types.VMConfig{Name: "beta"}},
+	}
+	tests := []struct {
+		name    string
+		filters []string
+		wantIDs []string
+	}{
+		{name: "no filter returns all", filters: nil, wantIDs: []string{"abcdef123456", "beadbeef0000"}},
+		{name: "exact name", filters: []string{"alpha"}, wantIDs: []string{"abcdef123456"}},
+		{name: "id prefix", filters: []string{"bead"}, wantIDs: []string{"beadbeef0000"}},
+		{name: "multi filter union", filters: []string{"alpha", "bead"}, wantIDs: []string{"abcdef123456", "beadbeef0000"}},
+		{name: "no match", filters: []string{"zzz"}, wantIDs: nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := applyFilters(vms, tt.filters)
+			gotIDs := make([]string, 0, len(got))
+			for _, vm := range got {
+				gotIDs = append(gotIDs, vm.ID)
+			}
+			if !equalStrings(gotIDs, tt.wantIDs) {
+				t.Errorf("applyFilters(%v) = %v, want %v", tt.filters, gotIDs, tt.wantIDs)
+			}
+		})
+	}
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}

hypervisor/state.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"net"
+	"syscall"
 	"time"
 
 	"github.com/projecteru2/core/log"
@@ -13,6 +15,8 @@ import (
 	"github.com/cocoonstack/cocoon/utils"
 )
 
+const socketProbeTimeout = 2 * time.Second
+
 // WithRunningVM calls fn if rec still points to a live VM process.
 func (b *Backend) WithRunningVM(ctx context.Context, rec *VMRecord, fn func(pid int) error) error {
 	pid, pidErr := utils.ReadPIDFile(b.PIDFilePath(rec.RunDir))
@@ -25,6 +29,23 @@ func (b *Backend) WithRunningVM(ctx context.Context, rec *VMRecord, fn func(pid
 	return fn(pid)
 }
 
+// IsAPISocketLive is a pidfile-independent liveness tiebreaker; honors ctx cancellation, fails closed on uncertain dial errors.
+func (b *Backend) IsAPISocketLive(ctx context.Context, rec *VMRecord) bool {
+	sock := SocketPath(rec.RunDir)
+	dialCtx, cancel := context.WithTimeout(ctx, socketProbeTimeout)
+	defer cancel()
+	conn, err := (&net.Dialer{}).DialContext(dialCtx, "unix", sock)
+	if err == nil {
+		_ = conn.Close()
+		return true
+	}
+	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ECONNREFUSED) {
+		return false
+	}
+	log.WithFunc(b.Typ+".IsAPISocketLive").Warnf(ctx, "uncertain socket probe %s: %v — failing closed", sock, err)
+	return true
+}
+
 // WithPausedVM pauses, runs fn, resumes; eager resume on success promotes its error, deferred resume on fn-error only logs.
 func (b *Backend) WithPausedVM(ctx context.Context, rec *VMRecord, pause, resume, fn func() error) error {
 	return b.WithRunningVM(ctx, rec, func(_ int) error {

hypervisor/stop.go

@@ -64,6 +64,10 @@ func (b *Backend) DeleteAll(ctx context.Context, refs []string, force bool, stop
 		}); runningErr != nil && !errors.Is(runningErr, ErrNotRunning) {
 			return fmt.Errorf("stop before delete: %w", runningErr)
 		}
+		// Socket probe runs regardless of stop outcome: AF_UNIX has no TIME_WAIT so a real stop guarantees no listener; firing always also catches false-negative pidfile/cmdline shortcuts.
+		if b.IsAPISocketLive(ctx, &rec) {
+			return fmt.Errorf("refuse delete: api socket %s still responsive (suspected orphan vmm; kill the vmm process then retry)", SocketPath(rec.RunDir))
+		}
 		if rmErr := RemoveVMDirs(rec.RunDir, rec.LogDir); rmErr != nil {
 			return fmt.Errorf("cleanup VM dirs: %w", rmErr)
 		}