fix: give vm.snapshot / vm.restore a 10-minute heavy-op timeout

The CH REST client utils.NewSocketHTTPClient sets http.Client.Timeout
to 30 seconds, which fires regardless of any context.WithTimeout at
the call site. vm.snapshot and vm.restore write/read the entire VM
memory image; on multi-GiB Windows snapshots over GCE standard
persistent disk, or for mid-boot snapshots still dirty with Windows
init activity, 30s is marginal and occasionally exceeded.

DoWithRetry masks the symptom by retrying 4x, but each attempt hits
the same 30s ceiling, so the user-visible effect is a ~2-minute wait
followed by "Client.Timeout exceeded while awaiting headers".

Fix: add heavyClient(hc) that clones the caller's http.Client by
value and raises Timeout to 10 minutes. The Transport (socket dialer
and connection pool) stays shared, so light endpoints like vm.pause
and vm.resume still use the original 30s client unchanged. Only
vm.snapshot and vm.restore route through heavyClient.

10 minutes was chosen to give multi-GiB cold-start restores real
headroom on slow disks while still bounding a genuine CH hang. The
global HTTPTimeout stays 30s to keep fast endpoints responsive.

Observed: epoch snapshot round-trip test against cocoonset-node-2
(2 GiB memory, freshly-booted Windows 11) — vm.restore timed out
with the default 30s client during cocoon vm clone even though the
memory-ranges file, disk, and network were all healthy.

Author: CMGS

hypervisor/backend.go

@@ -28,6 +28,16 @@ const (
 	// CreatingStateGCGrace is how long a VM can stay in "creating" state
 	// before GC treats it as a crash remnant and cleans it up.
 	CreatingStateGCGrace = 24 * time.Hour
+
+	// VMMemTransferTimeout is the per-call HTTP timeout for hypervisor API
+	// endpoints that move the entire guest memory (snapshot/restore). The
+	// default utils.HTTPTimeout (30s) is marginal for multi-GiB transfers
+	// on slow storage, and these operations have no useful retry semantics:
+	// they are non-idempotent (a partial snapshot overwrites files; a
+	// partial restore leaves the hypervisor in an undefined state), so
+	// callers must bypass the per-backend retry wrapper and issue a single
+	// call with this deadline.
+	VMMemTransferTimeout = 10 * time.Minute
 )
 
 // BackendConfig provides backend-specific values needed by shared Backend methods.

hypervisor/cloudhypervisor/clone.go

@@ -206,10 +206,10 @@ func (ch *CloudHypervisor) restoreAndResumeClone(
 		}
 	}()
 
-	hc := utils.NewSocketHTTPClient(sockPath)
-	if err = restoreVM(ctx, hc, runDir); err != nil {
+	if err = restoreVM(ctx, sockPath, runDir); err != nil {
 		return fmt.Errorf("vm.restore: %w", err)
 	}
+	hc := utils.NewSocketHTTPClient(sockPath)
 
 	// Hot-swap NICs while paused: remove snapshot's virtio-net devices (which carry
 	// the old MAC baked in binary device state), then add fresh ones with correct MAC.

hypervisor/cloudhypervisor/helper.go

@@ -66,24 +66,31 @@ func resumeVM(ctx context.Context, hc *http.Client) error {
 	return vmAPI(ctx, hc, "vm.resume", nil)
 }
 
-func snapshotVM(ctx context.Context, hc *http.Client, destDir string) error {
+// snapshotVM and restoreVM bypass vmAPI's retry layer — see hypervisor.VMMemTransferTimeout.
+func snapshotVM(ctx context.Context, sockPath, destDir string) error {
 	body, err := json.Marshal(map[string]string{
 		"destination_url": "file://" + destDir,
 	})
 	if err != nil {
 		return fmt.Errorf("marshal snapshot request: %w", err)
 	}
-	return vmAPI(ctx, hc, "vm.snapshot", body)
+	hc := utils.NewSocketHTTPClientWithTimeout(sockPath, hypervisor.VMMemTransferTimeout)
+	_, err = utils.DoAPI(ctx, hc, http.MethodPut,
+		"http://localhost/api/v1/vm.snapshot", body, http.StatusNoContent)
+	return err
 }
 
-func restoreVM(ctx context.Context, hc *http.Client, sourceDir string) error {
+func restoreVM(ctx context.Context, sockPath, sourceDir string) error {
 	body, err := json.Marshal(map[string]string{
 		"source_url": "file://" + sourceDir,
 	})
 	if err != nil {
 		return fmt.Errorf("marshal restore request: %w", err)
 	}
-	return vmAPI(ctx, hc, "vm.restore", body)
+	hc := utils.NewSocketHTTPClientWithTimeout(sockPath, hypervisor.VMMemTransferTimeout)
+	_, err = utils.DoAPI(ctx, hc, http.MethodPut,
+		"http://localhost/api/v1/vm.restore", body, http.StatusNoContent)
+	return err
 }
 
 func addDiskVM(ctx context.Context, hc *http.Client, disk chDisk) error {

hypervisor/cloudhypervisor/restore.go

@@ -113,10 +113,10 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string,
 		}
 	}()
 
-	hc := utils.NewSocketHTTPClient(sockPath)
-	if err = restoreVM(ctx, hc, rec.RunDir); err != nil {
+	if err = restoreVM(ctx, sockPath, rec.RunDir); err != nil {
 		return nil, fmt.Errorf("vm.restore: %w", err)
 	}
+	hc := utils.NewSocketHTTPClient(sockPath)
 	if err = resumeVM(ctx, hc); err != nil {
 		return nil, fmt.Errorf("vm.resume: %w", err)
 	}

hypervisor/cloudhypervisor/snapshot.go

@@ -69,7 +69,7 @@ func (ch *CloudHypervisor) Snapshot(ctx context.Context, ref string) (*types.Sna
 		}
 		defer doResume()
 
-		if err := snapshotVM(ctx, hc, tmpDir); err != nil {
+		if err := snapshotVM(ctx, sockPath, tmpDir); err != nil {
 			return fmt.Errorf("snapshot: %w", err)
 		}
 

hypervisor/firecracker/api.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"slices"
 
+	"github.com/cocoonstack/cocoon/hypervisor"
 	"github.com/cocoonstack/cocoon/utils"
 )
 
@@ -196,20 +197,25 @@ type fcSnapshotMemBE struct {
 }
 
 // createSnapshotFC creates a full VM snapshot (vmstate + memory file) in destDir.
-func createSnapshotFC(ctx context.Context, hc *http.Client, destDir string) error {
+// Bypasses fcAPI's retry layer — see hypervisor.VMMemTransferTimeout.
+func createSnapshotFC(ctx context.Context, sockPath, destDir string) error {
 	body, err := json.Marshal(fcSnapshotCreate{
 		SnapshotPath: filepath.Join(destDir, snapshotVMStateFile),
 		MemFilePath:  filepath.Join(destDir, snapshotMemFile),
 	})
 	if err != nil {
 		return fmt.Errorf("marshal snapshot/create request: %w", err)
 	}
-	return fcAPI(ctx, hc, http.MethodPut, "/snapshot/create", body)
+	hc := utils.NewSocketHTTPClientWithTimeout(sockPath, hypervisor.VMMemTransferTimeout)
+	_, err = utils.DoAPI(ctx, hc, http.MethodPut,
+		"http://localhost/snapshot/create", body, http.StatusNoContent)
+	return err
 }
 
 // loadSnapshotFC loads a VM snapshot from sourceDir into a freshly started FC process.
 // networkOverrides replaces TAP devices from the snapshot with new ones.
-func loadSnapshotFC(ctx context.Context, hc *http.Client, sourceDir string, networkOverrides []fcNetworkOverride) error {
+// Bypasses fcAPI's retry layer — see hypervisor.VMMemTransferTimeout.
+func loadSnapshotFC(ctx context.Context, sockPath, sourceDir string, networkOverrides []fcNetworkOverride) error {
 	body, err := json.Marshal(fcSnapshotLoad{
 		SnapshotPath: filepath.Join(sourceDir, snapshotVMStateFile),
 		MemBackend: fcSnapshotMemBE{
@@ -221,5 +227,8 @@ func loadSnapshotFC(ctx context.Context, hc *http.Client, sourceDir string, netw
 	if err != nil {
 		return fmt.Errorf("marshal snapshot/load request: %w", err)
 	}
-	return fcAPI(ctx, hc, http.MethodPut, "/snapshot/load", body)
+	hc := utils.NewSocketHTTPClientWithTimeout(sockPath, hypervisor.VMMemTransferTimeout)
+	_, err = utils.DoAPI(ctx, hc, http.MethodPut,
+		"http://localhost/snapshot/load", body, http.StatusNoContent)
+	return err
 }

hypervisor/firecracker/clone.go

@@ -177,17 +177,17 @@ func (fc *Firecracker) restoreAndResumeClone(
 		}
 	}()
 
-	hc := utils.NewSocketHTTPClient(sockPath)
 	// Use network_overrides to provide clone's TAP devices during snapshot/load.
 	// FC re-creates network devices from vmstate — overrides replace the
 	// source TAP with the clone's TAP so FC opens the right device.
 	netOverrides := buildNetworkOverrides(networkConfigs)
 	// FC opens drive files during snapshot/load and holds the fds.
 	// The symlink redirect ensures FC opens the clone's COW, not the source's.
 	// No drive reconfiguration needed — fds survive symlink cleanup.
-	if err = loadSnapshotFC(ctx, hc, runDir, netOverrides); err != nil {
+	if err = loadSnapshotFC(ctx, sockPath, runDir, netOverrides); err != nil {
 		return fmt.Errorf("snapshot/load: %w", err)
 	}
+	hc := utils.NewSocketHTTPClient(sockPath)
 	if err = resumeVM(ctx, hc); err != nil {
 		return fmt.Errorf("resume: %w", err)
 	}

hypervisor/firecracker/restore.go

@@ -120,13 +120,13 @@ func (fc *Firecracker) restoreAfterExtract(ctx context.Context, vmID string, vmC
 		}
 	}()
 
-	hc := utils.NewSocketHTTPClient(sockPath)
 	// Restore uses the same VM — TAP and drives are unchanged.
 	// No network_overrides or drive reconfiguration needed.
-	if err = loadSnapshotFC(ctx, hc, rec.RunDir, nil); err != nil {
+	if err = loadSnapshotFC(ctx, sockPath, rec.RunDir, nil); err != nil {
 		return nil, fmt.Errorf("snapshot/load: %w", err)
 	}
 
+	hc := utils.NewSocketHTTPClient(sockPath)
 	if err = resumeVM(ctx, hc); err != nil {
 		return nil, fmt.Errorf("resume: %w", err)
 	}

hypervisor/firecracker/snapshot.go

@@ -69,7 +69,7 @@ func (fc *Firecracker) Snapshot(ctx context.Context, ref string) (*types.Snapsho
 			}
 			defer doResume()
 
-			if err := createSnapshotFC(ctx, hc, tmpDir); err != nil {
+			if err := createSnapshotFC(ctx, sockPath, tmpDir); err != nil {
 				return fmt.Errorf("snapshot: %w", err)
 			}
 			if err := utils.ReflinkCopy(filepath.Join(tmpDir, cowFileName), cowPath); err != nil {

utils/http.go

@@ -27,8 +27,16 @@ func (e *APIError) Error() string { return e.Message }
 
 // NewSocketHTTPClient creates an HTTP client that dials a Unix socket.
 func NewSocketHTTPClient(socketPath string) *http.Client {
+	return NewSocketHTTPClientWithTimeout(socketPath, HTTPTimeout)
+}
+
+// NewSocketHTTPClientWithTimeout is like NewSocketHTTPClient but with a custom
+// per-call Timeout. Use this for endpoints where the default HTTPTimeout is too
+// tight (e.g. hypervisor snapshot/restore which transfer the entire guest memory
+// synchronously and can take minutes on multi-GiB VMs or slow storage).
+func NewSocketHTTPClientWithTimeout(socketPath string, timeout time.Duration) *http.Client {
 	return &http.Client{
-		Timeout: HTTPTimeout,
+		Timeout: timeout,
 		Transport: &http.Transport{
 			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
 				var d net.Dialer