fix: address Copilot round-1 findings on orphan VMM PR

- utils/process_linux.go: slices.Sort the returned pids so callers get a deterministic smallest-pid choice (Copilot caught the /proc lexicographic ordering trap, e.g. "100" < "11").
- hypervisor/state.go: fail-closed when /proc scan errors after pidfile-based check fails; previously returned ErrNotRunning on inconclusive state, which could let start/delete proceed against a still-running VM.
- hypervisor/stop.go: fail-closed in DeleteAll second-pass when /proc scan errors; previously dropped scanErr and risked re-introducing the orphan leak the PR is trying to fix.
- utils/process_test.go: replace flaky "sleep marker 60" (sleep rejects non-numeric arg and exits immediately) with "sh -c 'sleep 60 && :' marker" (compound prevents sh tail-exec into sleep). Gate on runtime.GOOS == "linux".

Author: CMGS

hypervisor/state.go

@@ -28,10 +28,10 @@ func (b *Backend) WithRunningVM(ctx context.Context, rec *VMRecord, fn func(pid
 	if utils.VerifyProcessCmdline(pid, b.Conf.BinaryName(), sockPath) {
 		return fn(pid)
 	}
-	// Covers pidfile/socket cleaned up before VMM exited.
+	// Covers pidfile/socket cleaned up before VMM exited. Fail-closed if scan errors so callers don't treat inconclusive state as ErrNotRunning.
 	scanned, scanErr := utils.FindVMMByCmdline(b.Conf.BinaryName(), sockPath)
 	if scanErr != nil {
-		logger.Warnf(ctx, "scan /proc for VM %s: %v", rec.ID, scanErr)
+		return fmt.Errorf("VM %s: pidfile-based check failed and /proc scan errored: %w", rec.ID, scanErr)
 	}
 	if len(scanned) == 0 {
 		return ErrNotRunning

hypervisor/stop.go

@@ -75,14 +75,16 @@ func (b *Backend) DeleteAll(ctx context.Context, refs []string, force bool, stop
 			}
 			return fmt.Errorf("refuse delete: api socket %s still responsive (suspected orphan vmm; kill the vmm process then retry)", sockPath)
 		}
-		// Catches workers/siblings the pidfile-based stop didn't see.
-		if scanned, _ := utils.FindVMMByCmdline(b.Conf.BinaryName(), sockPath); len(scanned) > 0 {
-			for _, pid := range scanned {
-				if termErr := utils.TerminateProcess(ctx, pid, b.Conf.BinaryName(), sockPath, b.Conf.TerminateGracePeriod()); termErr != nil {
-					return fmt.Errorf("terminate orphan VMM pid=%d for VM %s: %w", pid, id, termErr)
-				}
-				log.WithFunc(b.Typ+".Delete").Warnf(ctx, "killed orphan VMM pid=%d for VM %s", pid, id)
+		// Catches workers/siblings the pidfile-based stop didn't see; fail-closed on scan error so we never wipe rundir while VMM state is unknown.
+		scanned, scanErr := utils.FindVMMByCmdline(b.Conf.BinaryName(), sockPath)
+		if scanErr != nil {
+			return fmt.Errorf("refuse delete: VM %s /proc scan errored: %w (resolve host issue and retry)", id, scanErr)
+		}
+		for _, pid := range scanned {
+			if termErr := utils.TerminateProcess(ctx, pid, b.Conf.BinaryName(), sockPath, b.Conf.TerminateGracePeriod()); termErr != nil {
+				return fmt.Errorf("terminate orphan VMM pid=%d for VM %s: %w", pid, id, termErr)
 			}
+			log.WithFunc(b.Typ+".Delete").Warnf(ctx, "killed orphan VMM pid=%d for VM %s", pid, id)
 		}
 		if rmErr := RemoveVMDirs(rec.RunDir, rec.LogDir); rmErr != nil {
 			return fmt.Errorf("cleanup VM dirs: %w", rmErr)

utils/process_linux.go

@@ -6,11 +6,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 )
 
-// FindVMMByCmdline returns pids whose argv[0] basename matches binaryName and args contain expectArg.
+// FindVMMByCmdline returns pids whose argv[0] basename matches binaryName and args contain expectArg, sorted numerically.
 func FindVMMByCmdline(binaryName, expectArg string) ([]int, error) {
 	entries, err := os.ReadDir("/proc")
 	if err != nil {
@@ -26,6 +27,7 @@ func FindVMMByCmdline(binaryName, expectArg string) ([]int, error) {
 			pids = append(pids, pid)
 		}
 	}
+	slices.Sort(pids)
 	return pids, nil
 }
 

utils/process_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"testing"
 	"time"
@@ -246,12 +247,12 @@ func TestTerminateProcess_SIGTERMIgnored_FallsBackToKill(t *testing.T) {
 }
 
 func TestFindVMMByCmdline(t *testing.T) {
-	if _, err := os.Stat("/proc/self/cmdline"); err != nil {
-		t.Skip("/proc not available")
+	if runtime.GOOS != "linux" {
+		t.Skip("FindVMMByCmdline scans /proc — linux only")
 	}
 	marker := "cocoon-find-marker-" + strconv.Itoa(os.Getpid())
-	cmd := exec.Command("sleep", "60")
-	cmd.Args = []string{"sleep", marker, "60"}
+	// "sleep 60 && :" is a compound command so sh can't tail-exec into sleep and lose the marker arg.
+	cmd := exec.Command("sh", "-c", "sleep 60 && :", marker)
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("start: %v", err)
 	}
@@ -263,7 +264,7 @@ func TestFindVMMByCmdline(t *testing.T) {
 	// Poll briefly: cmdline is written by execve, so the parent may scan before /proc/<pid>/cmdline reflects argv.
 	var pids []int
 	for range 50 {
-		got, err := FindVMMByCmdline("sleep", marker)
+		got, err := FindVMMByCmdline("sh", marker)
 		if err != nil {
 			t.Fatalf("FindVMMByCmdline: %v", err)
 		}
@@ -280,7 +281,7 @@ func TestFindVMMByCmdline(t *testing.T) {
 	if got, _ := FindVMMByCmdline("definitely-no-such-binary", marker); len(got) != 0 {
 		t.Errorf("wrong-binary scan matched: %v", got)
 	}
-	if got, _ := FindVMMByCmdline("sleep", "no-such-marker"); len(got) != 0 {
+	if got, _ := FindVMMByCmdline("sh", "no-such-marker-"+marker); len(got) != 0 {
 		t.Errorf("wrong-marker scan matched: %v", got)
 	}
 }