diff --git a/.air.darwin.toml b/.air.darwin.toml new file mode 100644 index 00000000..ded73f54 --- /dev/null +++ b/.air.darwin.toml @@ -0,0 +1,48 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] + args_bin = [] + bin = "./tmp/main" + # Build for macOS with vz support, then sign with entitlements + # Also builds and signs vz-shim (subprocess that hosts vz VMs) + cmd = "make build-embedded && go build -o ./tmp/vz-shim ./cmd/vz-shim && codesign --sign - --entitlements vz.entitlements --force ./tmp/vz-shim && go build -tags containers_image_openpgp -o ./tmp/main ./cmd/api && codesign --sign - --entitlements vz.entitlements --force ./tmp/main" + delay = 1000 + exclude_dir = ["assets", "tmp", "vendor", "testdata", "bin", "scripts", "data", "kernel"] + exclude_file = [] + exclude_regex = ["_test.go"] + exclude_unchanged = false + follow_symlink = false + # No sudo needed on macOS - vz doesn't require root + full_bin = "./tmp/main" + include_dir = [] + include_ext = ["go", "tpl", "tmpl", "html", "yaml"] + include_file = [] + log = "build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + kill_delay = '1s' + rerun = false + rerun_delay = 500 + send_interrupt = true + stop_on_error = false + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[log] + main_only = false + time = false + +[misc] + clean_on_exit = false + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/.env.darwin.example b/.env.darwin.example new file mode 100644 index 00000000..f714f06e --- /dev/null +++ b/.env.darwin.example @@ -0,0 +1,122 @@ +# ============================================================================= +# macOS (Darwin) Configuration for Hypeman +# ============================================================================= +# Copy this file to .env and customize for your environment. +# +# Key differences from Linux (.env.example): +# - DEFAULT_HYPERVISOR: Use "vz" (Virtualization.framework) instead of cloud-hypervisor/qemu +# - DATA_DIR: Uses macOS conventions (~/Library/Application Support) +# - Network settings: BRIDGE_NAME, SUBNET_CIDR, etc. are IGNORED (vz uses NAT) +# - Rate limiting: Not supported on macOS (no tc/HTB equivalent) +# - GPU passthrough: Not supported on macOS +# ============================================================================= + +# Required +JWT_SECRET=dev-secret-change-me + +# Data directory - use macOS conventions +# Note: ~ expands to $HOME at runtime +DATA_DIR=~/Library/Application Support/hypeman + +# Server configuration +PORT=8080 + +# Logging +LOG_LEVEL=debug + +# ============================================================================= +# Hypervisor Configuration (IMPORTANT FOR MACOS) +# ============================================================================= +# On macOS, use "vz" (Virtualization.framework) +# - "cloud-hypervisor" and "qemu" are NOT supported on macOS +DEFAULT_HYPERVISOR=vz + +# ============================================================================= +# Network Configuration (DIFFERENT ON MACOS) +# ============================================================================= +# On macOS with vz, network is handled automatically via NAT: +# - VMs get IP addresses from 192.168.64.0/24 via DHCP +# - No TAP devices, bridges, or iptables needed +# - The following settings are IGNORED on macOS: +# BRIDGE_NAME, SUBNET_CIDR, SUBNET_GATEWAY, UPLINK_INTERFACE + +# DNS Server for VMs (used by guest for resolution) +DNS_SERVER=8.8.8.8 + +# ============================================================================= +# Caddy / Ingress Configuration +# ============================================================================= +CADDY_LISTEN_ADDRESS=0.0.0.0 +CADDY_ADMIN_ADDRESS=127.0.0.1 +CADDY_ADMIN_PORT=2019 +# Note: 5353 is used by mDNSResponder (Bonjour) on macOS, using 5354 instead +INTERNAL_DNS_PORT=5354 +CADDY_STOP_ON_SHUTDOWN=false + +# ============================================================================= +# Build System Configuration +# ============================================================================= +# For builds on macOS with vz, the registry URL needs to be accessible from +# NAT VMs. Since vz uses 192.168.64.0/24 for NAT, the host is at 192.168.64.1. +# +# IMPORTANT: "host.docker.internal" does NOT work in vz VMs - that's a Docker +# Desktop-specific hostname. Use the NAT gateway IP instead. +# +# Registry URL (the host's hypeman API, accessible from VMs) +REGISTRY_URL=192.168.64.1:8080 +# Use HTTP (not HTTPS) since hypeman's internal registry uses plaintext +REGISTRY_INSECURE=true + +BUILDER_IMAGE=hypeman/builder:latest +MAX_CONCURRENT_SOURCE_BUILDS=2 +BUILD_TIMEOUT=600 + +# ============================================================================= +# Resource Limits (same as Linux) +# ============================================================================= +# Per-instance limits +MAX_VCPUS_PER_INSTANCE=4 +MAX_MEMORY_PER_INSTANCE=8GB + +# Aggregate limits (0 or empty = unlimited) +# MAX_TOTAL_VOLUME_STORAGE= + +# ============================================================================= +# OpenTelemetry (optional, same as Linux) +# ============================================================================= +# OTEL_ENABLED=false +# OTEL_ENDPOINT=127.0.0.1:4317 +# OTEL_SERVICE_NAME=hypeman +# OTEL_INSECURE=true +# ENV=dev + +# ============================================================================= +# TLS / ACME Configuration (same as Linux) +# ============================================================================= +# ACME_EMAIL=admin@example.com +# ACME_DNS_PROVIDER=cloudflare +# TLS_ALLOWED_DOMAINS=*.example.com +# CLOUDFLARE_API_TOKEN= + +# ============================================================================= +# macOS Limitations +# ============================================================================= +# The following features are NOT AVAILABLE on macOS: +# +# 1. GPU Passthrough (VFIO, mdev) +# - GPU_PROFILE_CACHE_TTL is ignored +# - Device registration/binding will fail +# +# 2. Network Rate Limiting +# - UPLOAD_BURST_MULTIPLIER, DOWNLOAD_BURST_MULTIPLIER are ignored +# - No tc/HTB equivalent on macOS +# +# 3. CPU/Memory Hotplug +# - Resize operations not supported +# +# 4. Disk I/O Limiting +# - DISK_IO_LIMIT, OVERSUB_DISK_IO are ignored +# +# 5. Snapshots (requires macOS 14+ on Apple Silicon) +# - SaveMachineStateToPath/RestoreMachineStateFromURL require macOS 14+ +# - Only supported on ARM64 (Apple Silicon) Macs diff --git a/.gitignore b/.gitignore index b2b815d4..14a5d0e2 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,9 @@ dist/** # UTM VM - downloaded ISO files scripts/utm/images/ + +# IDE and editor +.cursor/ + +# Build artifacts +api diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 85a14f8b..d7b32e46 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,7 +4,17 @@ This document covers development setup, configuration, and contributing to Hypem ## Prerequisites -> **macOS Users:** Hypeman requires KVM, which is only available on Linux. See [scripts/utm/README.md](scripts/utm/README.md) for instructions on setting up a Linux VM with nested virtualization on Apple Silicon Macs. +### Linux (Default) + +**Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq** + +### macOS (Experimental) + +See [macOS Development](#macos-development) below for native macOS development using Virtualization.framework. + +--- + +**Linux Prerequisites:** **Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq** @@ -314,3 +324,176 @@ Or generate everything at once: ```bash make generate-all ``` + +## macOS Development + +Hypeman supports native macOS development using Apple's Virtualization.framework (via the `vz` hypervisor). + +### Requirements + +- **macOS 11.0+** (Big Sur or later) +- **Apple Silicon** (M1/M2/M3) recommended +- **macOS 14.0+** (Sonoma) required for snapshot/restore (ARM64 only) +- **Go 1.25.4+** +- **Caddy** (for ingress): `brew install caddy` +- **e2fsprogs** (for ext4 disk images): `brew install e2fsprogs` + +### Quick Start + +```bash +# 1. Install dependencies +brew install caddy e2fsprogs + +# 2. Add e2fsprogs to PATH (it's keg-only) +export PATH="/opt/homebrew/opt/e2fsprogs/bin:/opt/homebrew/opt/e2fsprogs/sbin:$PATH" +# Add to ~/.zshrc for persistence + +# 3. Configure environment +cp .env.darwin.example .env +# Edit .env as needed (defaults work for local development) + +# 4. Create data directory +mkdir -p ~/Library/Application\ Support/hypeman + +# 5. Run in development mode (auto-detects macOS, builds, signs, and runs with hot reload) +make dev +``` + +The `make dev` command automatically detects macOS and: +- Builds with vz support +- Signs with required entitlements +- Runs with hot reload (no sudo required) + +### Alternative Commands + +```bash +# Build and sign only (no hot reload) +make sign-darwin + +# Verify entitlements are correct +make verify-entitlements + +# Run manually after signing +./bin/hypeman +``` + +### Key Differences from Linux Development + +| Aspect | Linux | macOS | +|--------|-------|-------| +| Hypervisor | Cloud Hypervisor, QEMU | vz (Virtualization.framework) | +| Binary signing | Not required | Automatic via `make dev` or `make sign-darwin` | +| Networking | TAP + bridge + iptables | Automatic NAT (no setup needed) | +| Root/sudo | Required for networking | Not required | +| Caddy | Embedded binary | Install via `brew install caddy` | +| DNS port | 5353 | 5354 (avoids mDNSResponder conflict) | + +### macOS-Specific Configuration + +The following environment variables work differently on macOS (see `.env.darwin.example`): + +| Variable | Linux | macOS | +|----------|-------|-------| +| `DEFAULT_HYPERVISOR` | `cloud-hypervisor` | `vz` | +| `DATA_DIR` | `/var/lib/hypeman` | `~/Library/Application Support/hypeman` | +| `INTERNAL_DNS_PORT` | `5353` | `5354` (5353 is used by mDNSResponder) | +| `BRIDGE_NAME` | Used | Ignored (NAT) | +| `SUBNET_CIDR` | Used | Ignored (NAT) | +| `UPLINK_INTERFACE` | Used | Ignored (NAT) | +| Network rate limiting | Supported | Not supported | +| GPU passthrough | Supported (VFIO) | Not supported | + +### Code Organization + +Platform-specific code uses Go build tags: + +``` +lib/network/ +├── bridge_linux.go # Linux networking (TAP, bridges, iptables) +├── bridge_darwin.go # macOS stubs (uses NAT) +└── ip.go # Shared utilities + +lib/devices/ +├── discovery_linux.go # Linux PCI device discovery +├── discovery_darwin.go # macOS stubs (no passthrough) +├── mdev_linux.go # Linux vGPU (mdev) +├── mdev_darwin.go # macOS stubs +├── vfio_linux.go # Linux VFIO binding +├── vfio_darwin.go # macOS stubs +└── types.go # Shared types + +lib/hypervisor/ +├── cloudhypervisor/ # Cloud Hypervisor (Linux) +├── qemu/ # QEMU (Linux, vsock_linux.go) +└── vz/ # Virtualization.framework (macOS only) + ├── starter.go # VMStarter implementation + ├── hypervisor.go # Hypervisor interface + └── vsock.go # VsockDialer via VirtioSocketDevice +``` + +### Testing on macOS + +```bash +# Verify vz package compiles correctly +make test-vz-compile + +# Run unit tests (Linux-specific tests like networking will be skipped) +go test ./lib/hypervisor/vz/... +go test ./lib/resources/... +go test ./lib/images/... +``` + +Note: Full integration tests require Linux. On macOS, focus on unit tests and manual API testing. + +### Known Limitations + +1. **Disk Format**: vz only supports raw disk images (not qcow2). Convert images: + ```bash + qemu-img convert -f qcow2 -O raw disk.qcow2 disk.raw + ``` + +2. **Snapshots**: Only available on macOS 14+ (Sonoma) on Apple Silicon: + ```go + // Check support at runtime + valid, err := vmConfig.ValidateSaveRestoreSupport() + ``` + +3. **Network Ingress**: VMs get DHCP addresses from macOS NAT. To access a VM's services: + - Query the VM's IP via guest agent + - Use vsock for internal communication (no NAT traversal needed) + +4. **In-Process VMM**: Unlike CH/QEMU which run as separate processes, vz VMs run in the hypeman process. If hypeman crashes, all VMs stop. + +### Troubleshooting + +**"binary needs to be signed with entitlements"** +```bash +make sign-darwin +# Or just use: make dev (handles signing automatically) +``` + +**"caddy binary is not embedded on macOS"** +```bash +brew install caddy +``` + +**"address already in use" on port 5353** +- Port 5353 is used by mDNSResponder (Bonjour) on macOS +- Use port 5354 instead: `INTERNAL_DNS_PORT=5354` in `.env` +- The `.env.darwin.example` already has this configured correctly + +**"Virtualization.framework is not available"** +- Ensure you're on macOS 11.0+ +- Check if virtualization is enabled in Recovery Mode settings + +**"snapshot not supported"** +- Requires macOS 14.0+ on Apple Silicon +- Check: `sw_vers` and `uname -m` (should be arm64) + +**VM fails to start** +- Check serial log: `$DATA_DIR/instances//serial.log` +- Ensure kernel and initrd paths are correct in config + +**IOMMU/VFIO warnings at startup** +- These are expected on macOS and can be ignored +- GPU passthrough is not supported on macOS diff --git a/Makefile b/Makefile index 88eab9c9..34127c67 100644 --- a/Makefile +++ b/Makefile @@ -174,14 +174,16 @@ ensure-caddy-binaries: fi # Build guest-agent (guest binary) into its own directory for embedding +# Cross-compile for Linux since it runs inside the VM lib/system/guest_agent/guest-agent: lib/system/guest_agent/*.go - @echo "Building guest-agent..." - cd lib/system/guest_agent && CGO_ENABLED=0 go build -ldflags="-s -w" -o guest-agent . + @echo "Building guest-agent for Linux..." + cd lib/system/guest_agent && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o guest-agent . # Build init binary (runs as PID 1 in guest VM) for embedding +# Cross-compile for Linux since it runs inside the VM lib/system/init/init: lib/system/init/*.go - @echo "Building init binary..." - cd lib/system/init && CGO_ENABLED=0 go build -ldflags="-s -w" -o init . + @echo "Building init binary for Linux..." + cd lib/system/init && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o init . build-embedded: lib/system/guest_agent/guest-agent lib/system/init/init @@ -193,7 +195,16 @@ build: ensure-ch-binaries ensure-caddy-binaries build-embedded | $(BIN_DIR) build-all: build # Run in development mode with hot reload -dev: ensure-ch-binaries ensure-caddy-binaries build-embedded $(AIR) +# On macOS, redirects to dev-darwin which uses vz instead of cloud-hypervisor +dev: + @if [ "$$(uname)" = "Darwin" ]; then \ + $(MAKE) dev-darwin; \ + else \ + $(MAKE) dev-linux; \ + fi + +# Linux development mode with hot reload +dev-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded $(AIR) @rm -f ./tmp/main $(AIR) -c .air.toml @@ -238,3 +249,94 @@ clean: # Downloads all embedded binaries and builds embedded components release-prep: download-ch-binaries build-caddy-binaries build-embedded go mod tidy + +# ============================================================================= +# macOS (vz/Virtualization.framework) targets +# ============================================================================= + +# Entitlements file for macOS codesigning +ENTITLEMENTS_FILE ?= vz.entitlements + +# Build vz-shim (subprocess that hosts vz VMs) +.PHONY: build-vz-shim +build-vz-shim: | $(BIN_DIR) + @echo "Building vz-shim for macOS..." + go build -o $(BIN_DIR)/vz-shim ./cmd/vz-shim + @echo "Build complete: $(BIN_DIR)/vz-shim" + +# Sign vz-shim with entitlements +.PHONY: sign-vz-shim +sign-vz-shim: build-vz-shim + @echo "Signing $(BIN_DIR)/vz-shim with entitlements..." + codesign --sign - --entitlements $(ENTITLEMENTS_FILE) --force $(BIN_DIR)/vz-shim + @echo "Signed: $(BIN_DIR)/vz-shim" + +# Build for macOS with vz support +# Note: This builds without embedded CH/Caddy binaries since vz doesn't need them +# Guest-agent and init are cross-compiled for Linux (they run inside the VM) +.PHONY: build-darwin +build-darwin: build-embedded build-vz-shim | $(BIN_DIR) + @echo "Building hypeman for macOS with vz support..." + go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api + @echo "Build complete: $(BIN_DIR)/hypeman" + +# Sign the binary with entitlements (required for Virtualization.framework) +# Usage: make sign-darwin +.PHONY: sign-darwin +sign-darwin: build-darwin sign-vz-shim + @echo "Signing $(BIN_DIR)/hypeman with entitlements..." + codesign --sign - --entitlements $(ENTITLEMENTS_FILE) --force $(BIN_DIR)/hypeman + @echo "Verifying signature..." + codesign --display --entitlements - $(BIN_DIR)/hypeman + +# Sign with a specific identity (for distribution) +# Usage: make sign-darwin-identity IDENTITY="Developer ID Application: Your Name" +.PHONY: sign-darwin-identity +sign-darwin-identity: build-darwin + @if [ -z "$(IDENTITY)" ]; then \ + echo "Error: IDENTITY not set. Usage: make sign-darwin-identity IDENTITY='Developer ID Application: ...'"; \ + exit 1; \ + fi + @echo "Signing $(BIN_DIR)/hypeman with identity: $(IDENTITY)" + codesign --sign "$(IDENTITY)" --entitlements $(ENTITLEMENTS_FILE) --force --options runtime $(BIN_DIR)/hypeman + @echo "Verifying signature..." + codesign --verify --verbose $(BIN_DIR)/hypeman + +# Run on macOS with vz support (development mode) +# Automatically signs the binary before running +.PHONY: dev-darwin +# macOS development mode with hot reload (uses vz, no sudo needed) +dev-darwin: build-embedded $(AIR) + @rm -f ./tmp/main + $(AIR) -c .air.darwin.toml + +# Run without hot reload (for testing) +run: + @if [ "$$(uname)" = "Darwin" ]; then \ + $(MAKE) run-darwin; \ + else \ + $(MAKE) run-linux; \ + fi + +run-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded build + ./bin/hypeman + +run-darwin: sign-darwin + ./bin/hypeman + +# Quick test of vz package compilation +.PHONY: test-vz-compile +test-vz-compile: + @echo "Testing vz package compilation..." + go build ./lib/hypervisor/vz/... + @echo "vz package compiles successfully" + +# Verify entitlements on a signed binary +.PHONY: verify-entitlements +verify-entitlements: + @if [ ! -f $(BIN_DIR)/hypeman ]; then \ + echo "Error: $(BIN_DIR)/hypeman not found. Run 'make sign-darwin' first."; \ + exit 1; \ + fi + @echo "Entitlements on $(BIN_DIR)/hypeman:" + codesign --display --entitlements - $(BIN_DIR)/hypeman diff --git a/README.md b/README.md index 69a5f18f..ac4a2573 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,13 @@ ## Requirements -Hypeman server runs on **Linux** with **KVM** virtualization support. The CLI can run locally on the server or connect remotely from any machine. +### Linux (Production) +Hypeman server runs on **Linux** with **KVM** virtualization support. Supports Cloud Hypervisor and QEMU as hypervisors. + +### macOS (Experimental) +Hypeman also supports **macOS** (11.0+) using Apple's **Virtualization.framework** via the `vz` hypervisor. See [macOS Support](#macos-support) below. + +The CLI can run locally on the server or connect remotely from any machine. ## Quick Start @@ -153,6 +159,59 @@ hypeman logs --source hypeman my-app For all available commands, run `hypeman --help`. +## macOS Support + +Hypeman supports macOS using Apple's Virtualization.framework through the `vz` hypervisor. This provides native virtualization on Apple Silicon and Intel Macs. + +### Requirements + +- macOS 11.0+ (macOS 14.0+ required for snapshot/restore on ARM64) +- Apple Silicon (M1/M2/M3) recommended +- Caddy: `brew install caddy` +- e2fsprogs: `brew install e2fsprogs` (for ext4 disk images) + +### Quick Start (macOS) + +```bash +# Install dependencies +brew install caddy e2fsprogs + +# Add e2fsprogs to PATH (it's keg-only) +export PATH="/opt/homebrew/opt/e2fsprogs/bin:/opt/homebrew/opt/e2fsprogs/sbin:$PATH" + +# Configure environment +cp .env.darwin.example .env + +# Create data directory +mkdir -p ~/Library/Application\ Support/hypeman + +# Run with hot reload (auto-detects macOS, builds, signs, and runs) +make dev +``` + +The `make dev` command automatically detects macOS and handles building with vz support and signing with required entitlements. + +### Key Differences from Linux + +| Feature | Linux | macOS | +|---------|-------|-------| +| Hypervisors | Cloud Hypervisor, QEMU | vz (Virtualization.framework) | +| Networking | TAP devices, bridges, iptables | NAT (built-in, automatic) | +| Rate Limiting | HTB/tc | Not supported | +| GPU Passthrough | VFIO | Not supported | +| Disk Format | qcow2, raw | raw only | +| Snapshots | Always available | macOS 14+ ARM64 only | + +### Limitations + +- **Networking**: macOS uses NAT networking automatically. No manual bridge/TAP configuration needed, but ingress requires discovering the VM's NAT IP. +- **Rate Limiting**: Network and disk I/O rate limiting is not available on macOS. +- **GPU**: PCI device passthrough is not supported on macOS. +- **Disk Images**: qcow2 format is not directly supported; use raw disk images. +- **Snapshots**: Requires macOS 14.0+ on Apple Silicon (ARM64). + +For detailed development setup, see [DEVELOPMENT.md](DEVELOPMENT.md). + ## Development See [DEVELOPMENT.md](DEVELOPMENT.md) for build instructions, configuration options, and contributing guidelines. diff --git a/cmd/api/api/cp.go b/cmd/api/api/cp.go index 3b060d39..6bae53ed 100644 --- a/cmd/api/api/cp.go +++ b/cmd/api/api/cp.go @@ -11,7 +11,6 @@ import ( "github.com/gorilla/websocket" "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" mw "github.com/kernel/hypeman/lib/middleware" @@ -219,10 +218,9 @@ func (s *ApiService) CpHandler(w http.ResponseWriter, r *http.Request) { // handleCopyTo handles copying files from client to guest // Returns the number of bytes transferred and any error. func (s *ApiService) handleCopyTo(ctx context.Context, ws *websocket.Conn, inst *instances.Instance, req CpRequest) (int64, error) { - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - return 0, fmt.Errorf("create vsock dialer: %w", err) + return 0, fmt.Errorf("get vsock dialer: %w", err) } grpcConn, err := guest.GetOrCreateConn(ctx, dialer) @@ -329,10 +327,9 @@ func (s *ApiService) handleCopyTo(ctx context.Context, ws *websocket.Conn, inst // handleCopyFrom handles copying files from guest to client // Returns the number of bytes transferred and any error. func (s *ApiService) handleCopyFrom(ctx context.Context, ws *websocket.Conn, inst *instances.Instance, req CpRequest) (int64, error) { - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - return 0, fmt.Errorf("create vsock dialer: %w", err) + return 0, fmt.Errorf("get vsock dialer: %w", err) } grpcConn, err := guest.GetOrCreateConn(ctx, dialer) diff --git a/cmd/api/api/exec.go b/cmd/api/api/exec.go index b9f5f3b3..b1e13c2c 100644 --- a/cmd/api/api/exec.go +++ b/cmd/api/api/exec.go @@ -12,7 +12,6 @@ import ( "github.com/gorilla/websocket" "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" mw "github.com/kernel/hypeman/lib/middleware" @@ -132,10 +131,9 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) { // Create WebSocket read/writer wrapper that handles resize messages wsConn := &wsReadWriter{ws: ws, ctx: ctx, resizeChan: resizeChan} - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(hypervisor.Type(inst.HypervisorType), inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - log.ErrorContext(ctx, "failed to create vsock dialer", "error", err) + log.ErrorContext(ctx, "failed to get vsock dialer", "error", err) ws.WriteMessage(websocket.BinaryMessage, []byte(fmt.Sprintf("Error: %v\r\n", err))) ws.WriteMessage(websocket.TextMessage, []byte(`{"exitCode":127}`)) return diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 62621dec..5907af10 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -592,13 +592,12 @@ func (s *ApiService) StatInstancePath(ctx context.Context, request oapi.StatInst }, nil } - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - log.ErrorContext(ctx, "failed to create vsock dialer", "error", err) + log.ErrorContext(ctx, "failed to get vsock dialer", "error", err) return oapi.StatInstancePath500JSONResponse{ Code: "internal_error", - Message: "failed to create vsock dialer", + Message: "failed to get vsock dialer", }, nil } diff --git a/cmd/api/hypervisor_check_darwin.go b/cmd/api/hypervisor_check_darwin.go new file mode 100644 index 00000000..c1a7eda8 --- /dev/null +++ b/cmd/api/hypervisor_check_darwin.go @@ -0,0 +1,32 @@ +//go:build darwin + +package main + +import ( + "fmt" + "runtime" + + "github.com/Code-Hex/vz/v3" +) + +// checkHypervisorAccess verifies Virtualization.framework is available on macOS +func checkHypervisorAccess() error { + // Check if we're on ARM64 (Apple Silicon) - required for best support + if runtime.GOARCH != "arm64" { + return fmt.Errorf("Virtualization.framework on macOS requires Apple Silicon (arm64), got %s", runtime.GOARCH) + } + + // Validate virtualization is usable by attempting to get max CPU count + // This will fail if entitlements are missing or virtualization is not available + maxCPU := vz.VirtualMachineConfigurationMaximumAllowedCPUCount() + if maxCPU < 1 { + return fmt.Errorf("Virtualization.framework reports 0 max CPUs - check entitlements") + } + + return nil +} + +// hypervisorAccessCheckName returns the name of the hypervisor access check for logging +func hypervisorAccessCheckName() string { + return "Virtualization.framework" +} diff --git a/cmd/api/hypervisor_check_linux.go b/cmd/api/hypervisor_check_linux.go new file mode 100644 index 00000000..042e70ca --- /dev/null +++ b/cmd/api/hypervisor_check_linux.go @@ -0,0 +1,29 @@ +//go:build linux + +package main + +import ( + "fmt" + "os" +) + +// checkHypervisorAccess verifies KVM is available and the user has permission to use it +func checkHypervisorAccess() error { + f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("/dev/kvm not found - KVM not enabled or not supported") + } + if os.IsPermission(err) { + return fmt.Errorf("permission denied accessing /dev/kvm - user not in 'kvm' group") + } + return fmt.Errorf("cannot access /dev/kvm: %w", err) + } + f.Close() + return nil +} + +// hypervisorAccessCheckName returns the name of the hypervisor access check for logging +func hypervisorAccessCheckName() string { + return "KVM" +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 7f5e4265..127a1d22 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -130,11 +130,11 @@ func run() error { logger.Warn("JWT_SECRET not configured - API authentication will fail") } - // Verify KVM access (required for VM creation) - if err := checkKVMAccess(); err != nil { - return fmt.Errorf("KVM access check failed: %w\n\nEnsure:\n 1. KVM is enabled (check /dev/kvm exists)\n 2. User is in 'kvm' group: sudo usermod -aG kvm $USER\n 3. Log out and back in, or use: newgrp kvm", err) + // Verify hypervisor access (KVM on Linux, Virtualization.framework on macOS) + if err := checkHypervisorAccess(); err != nil { + return fmt.Errorf("hypervisor access check failed: %w", err) } - logger.Info("KVM access verified") + logger.Info("Hypervisor access verified", "type", hypervisorAccessCheckName()) // Check if QEMU is available (optional - only warn if not present) if _, err := (&qemu.Starter{}).GetBinaryPath(nil, ""); err != nil { @@ -465,18 +465,5 @@ func run() error { return err } -// checkKVMAccess verifies KVM is available and the user has permission to use it -func checkKVMAccess() error { - f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("/dev/kvm not found - KVM not enabled or not supported") - } - if os.IsPermission(err) { - return fmt.Errorf("permission denied accessing /dev/kvm - user not in 'kvm' group") - } - return fmt.Errorf("cannot access /dev/kvm: %w", err) - } - f.Close() - return nil -} +// checkHypervisorAccess and hypervisorAccessCheckName are defined in +// hypervisor_check_linux.go and hypervisor_check_darwin.go diff --git a/cmd/vz-shim/main.go b/cmd/vz-shim/main.go new file mode 100644 index 00000000..04bd9ce9 --- /dev/null +++ b/cmd/vz-shim/main.go @@ -0,0 +1,218 @@ +//go:build darwin + +// Package main implements hypeman-vz-shim, a subprocess that hosts vz VMs. +// This allows VMs to survive hypeman restarts by running in a separate process. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/Code-Hex/vz/v3" +) + +// ShimConfig is the configuration passed from hypeman to the shim. +type ShimConfig struct { + // Compute resources + VCPUs int `json:"vcpus"` + MemoryBytes int64 `json:"memory_bytes"` + + // Storage + Disks []DiskConfig `json:"disks"` + + // Network + Networks []NetworkConfig `json:"networks"` + + // Console + SerialLogPath string `json:"serial_log_path"` + + // Boot configuration + KernelPath string `json:"kernel_path"` + InitrdPath string `json:"initrd_path"` + KernelArgs string `json:"kernel_args"` + + // Socket paths (where shim should listen) + ControlSocket string `json:"control_socket"` + VsockSocket string `json:"vsock_socket"` + + // Logging + LogPath string `json:"log_path"` + + // Restore from snapshot (optional) + RestoreStatePath string `json:"restore_state_path,omitempty"` +} + +// DiskConfig represents a disk attached to the VM. +type DiskConfig struct { + Path string `json:"path"` + Readonly bool `json:"readonly"` +} + +// NetworkConfig represents a network interface. +type NetworkConfig struct { + MAC string `json:"mac"` +} + +func main() { + configJSON := flag.String("config", "", "VM configuration as JSON") + flag.Parse() + + if *configJSON == "" { + fmt.Fprintln(os.Stderr, "error: -config is required") + os.Exit(1) + } + + var config ShimConfig + if err := json.Unmarshal([]byte(*configJSON), &config); err != nil { + fmt.Fprintf(os.Stderr, "error: invalid config JSON: %v\n", err) + os.Exit(1) + } + + // Setup logging to file + if err := setupLogging(config.LogPath); err != nil { + fmt.Fprintf(os.Stderr, "error: setup logging: %v\n", err) + os.Exit(1) + } + + slog.Info("vz-shim starting", "control_socket", config.ControlSocket, "vsock_socket", config.VsockSocket) + + // Create the VM + vm, vmConfig, err := createVM(config) + if err != nil { + slog.Error("failed to create VM", "error", err) + os.Exit(1) + } + + // Either restore from snapshot or start fresh + // NOTE: Linux VM restore is NOT supported by Virtualization.framework + // This code path exists for potential future macOS guest support + if config.RestoreStatePath != "" { + slog.Info("restoring VM from snapshot", "path", config.RestoreStatePath) + if err := vm.RestoreMachineStateFromURL(config.RestoreStatePath); err != nil { + slog.Error("failed to restore VM from snapshot", "error", err) + os.Exit(1) + } + // After restore, VM is in paused state - resume it + if err := vm.Resume(); err != nil { + slog.Error("failed to resume VM after restore", "error", err) + os.Exit(1) + } + slog.Info("VM restored and resumed", "vcpus", config.VCPUs, "memory_mb", config.MemoryBytes/1024/1024) + } else { + if err := vm.Start(); err != nil { + slog.Error("failed to start VM", "error", err) + os.Exit(1) + } + slog.Info("VM started", "vcpus", config.VCPUs, "memory_mb", config.MemoryBytes/1024/1024) + } + + // Create the shim server + server := NewShimServer(vm, vmConfig) + + // Start control socket listener + controlListener, err := net.Listen("unix", config.ControlSocket) + if err != nil { + slog.Error("failed to listen on control socket", "error", err, "path", config.ControlSocket) + os.Exit(1) + } + defer controlListener.Close() + + // Start vsock proxy listener + vsockListener, err := net.Listen("unix", config.VsockSocket) + if err != nil { + slog.Error("failed to listen on vsock socket", "error", err, "path", config.VsockSocket) + os.Exit(1) + } + defer vsockListener.Close() + + // Start HTTP server for control API + httpServer := &http.Server{Handler: server.Handler()} + go func() { + slog.Info("control API listening", "socket", config.ControlSocket) + if err := httpServer.Serve(controlListener); err != nil && err != http.ErrServerClosed { + slog.Error("control API server error", "error", err) + } + }() + + // Start vsock proxy + go func() { + slog.Info("vsock proxy listening", "socket", config.VsockSocket) + server.ServeVsock(vsockListener) + }() + + // Wait for shutdown signal or VM stop + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) + + // Monitor VM state + go func() { + for { + select { + case <-ctx.Done(): + return + case newState := <-vm.StateChangedNotify(): + slog.Info("VM state changed", "state", newState) + if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError { + slog.Info("VM stopped, shutting down shim") + cancel() + return + } + } + } + }() + + select { + case sig := <-sigChan: + slog.Info("received signal, shutting down", "signal", sig) + case <-ctx.Done(): + slog.Info("context cancelled, shutting down") + } + + // Graceful shutdown + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + + httpServer.Shutdown(shutdownCtx) + + if vm.State() == vz.VirtualMachineStateRunning { + slog.Info("stopping VM") + if vm.CanStop() { + vm.Stop() + } + } + + slog.Info("vz-shim shutdown complete") +} + +func setupLogging(logPath string) error { + if logPath == "" { + // Log to stderr if no path specified + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + return nil + } + + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + return fmt.Errorf("create log directory: %w", err) + } + + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return fmt.Errorf("open log file: %w", err) + } + + slog.SetDefault(slog.New(slog.NewJSONHandler(file, &slog.HandlerOptions{Level: slog.LevelDebug}))) + return nil +} diff --git a/cmd/vz-shim/server.go b/cmd/vz-shim/server.go new file mode 100644 index 00000000..21fce0ac --- /dev/null +++ b/cmd/vz-shim/server.go @@ -0,0 +1,320 @@ +//go:build darwin + +package main + +import ( + "encoding/json" + "fmt" + "log/slog" + "net" + "net/http" + "sync" + + "github.com/Code-Hex/vz/v3" +) + +// ShimServer handles control API and vsock proxy for a vz VM. +type ShimServer struct { + vm *vz.VirtualMachine + vmConfig *vz.VirtualMachineConfiguration + mu sync.RWMutex +} + +// NewShimServer creates a new shim server. +func NewShimServer(vm *vz.VirtualMachine, vmConfig *vz.VirtualMachineConfiguration) *ShimServer { + return &ShimServer{ + vm: vm, + vmConfig: vmConfig, + } +} + +// VMInfoResponse matches the cloud-hypervisor VmInfo structure. +type VMInfoResponse struct { + State string `json:"state"` +} + +// SnapshotRequest is the request body for vm.snapshot. +type SnapshotRequest struct { + DestinationURL string `json:"destination_url"` +} + +// Handler returns the HTTP handler for the control API. +func (s *ShimServer) Handler() http.Handler { + mux := http.NewServeMux() + + // Match cloud-hypervisor API patterns + mux.HandleFunc("GET /api/v1/vm.info", s.handleVMInfo) + mux.HandleFunc("PUT /api/v1/vm.pause", s.handlePause) + mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume) + mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown) + mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton) + mux.HandleFunc("PUT /api/v1/vm.snapshot", s.handleSnapshot) + mux.HandleFunc("GET /api/v1/vmm.ping", s.handlePing) + mux.HandleFunc("PUT /api/v1/vmm.shutdown", s.handleVMMShutdown) + + return mux +} + +func (s *ShimServer) handleVMInfo(w http.ResponseWriter, r *http.Request) { + s.mu.RLock() + defer s.mu.RUnlock() + + state := vzStateToString(s.vm.State()) + resp := VMInfoResponse{State: state} + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} + +func (s *ShimServer) handlePause(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.vm.CanPause() { + http.Error(w, "cannot pause VM", http.StatusBadRequest) + return + } + + if err := s.vm.Pause(); err != nil { + slog.Error("failed to pause VM", "error", err) + http.Error(w, fmt.Sprintf("pause failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("VM paused") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handleResume(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.vm.CanResume() { + http.Error(w, "cannot resume VM", http.StatusBadRequest) + return + } + + if err := s.vm.Resume(); err != nil { + slog.Error("failed to resume VM", "error", err) + http.Error(w, fmt.Sprintf("resume failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("VM resumed") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + // Request graceful shutdown via guest + success, err := s.vm.RequestStop() + if err != nil || !success { + slog.Warn("RequestStop failed, trying Stop", "error", err) + if s.vm.CanStop() { + if err := s.vm.Stop(); err != nil { + slog.Error("failed to stop VM", "error", err) + http.Error(w, fmt.Sprintf("shutdown failed: %v", err), http.StatusInternalServerError) + return + } + } + } + + slog.Info("VM shutdown requested") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handlePowerButton(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + // RequestStop sends an ACPI power button event + success, err := s.vm.RequestStop() + if err != nil || !success { + slog.Error("failed to send power button", "error", err, "success", success) + http.Error(w, fmt.Sprintf("power button failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("power button sent") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handleSnapshot(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + var req SnapshotRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("invalid request: %v", err), http.StatusBadRequest) + return + } + + if req.DestinationURL == "" { + http.Error(w, "destination_url is required", http.StatusBadRequest) + return + } + + // Check if save/restore is supported by this configuration + supported, err := s.vmConfig.ValidateSaveRestoreSupport() + if err != nil || !supported { + slog.Error("snapshot not supported", "error", err, "supported", supported) + http.Error(w, fmt.Sprintf("snapshot not supported: %v", err), http.StatusBadRequest) + return + } + + // VM must be paused to save state + if s.vm.State() != vz.VirtualMachineStatePaused { + http.Error(w, "VM must be paused before snapshot", http.StatusBadRequest) + return + } + + slog.Info("saving VM state", "path", req.DestinationURL) + if err := s.vm.SaveMachineStateToPath(req.DestinationURL); err != nil { + slog.Error("failed to save VM state", "error", err) + http.Error(w, fmt.Sprintf("snapshot failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("VM state saved", "path", req.DestinationURL) + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handlePing(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) +} + +func (s *ShimServer) handleVMMShutdown(w http.ResponseWriter, r *http.Request) { + slog.Info("VMM shutdown requested") + w.WriteHeader(http.StatusNoContent) + + // Stop the VM and exit + go func() { + s.mu.Lock() + defer s.mu.Unlock() + + if s.vm.CanStop() { + s.vm.Stop() + } + // Process will exit when VM stops (monitored in main) + }() +} + +func vzStateToString(state vz.VirtualMachineState) string { + switch state { + case vz.VirtualMachineStateStopped: + return "Shutdown" + case vz.VirtualMachineStateRunning: + return "Running" + case vz.VirtualMachineStatePaused: + return "Paused" + case vz.VirtualMachineStateError: + return "Error" + case vz.VirtualMachineStateStarting: + return "Starting" + case vz.VirtualMachineStatePausing: + return "Pausing" + case vz.VirtualMachineStateResuming: + return "Resuming" + case vz.VirtualMachineStateStopping: + return "Stopping" + default: + return "Unknown" + } +} + +// ServeVsock handles vsock proxy connections using the Cloud Hypervisor protocol. +// Protocol: Client sends "CONNECT {port}\n", server responds "OK {port}\n", then proxies. +func (s *ShimServer) ServeVsock(listener net.Listener) { + for { + conn, err := listener.Accept() + if err != nil { + slog.Debug("vsock listener closed", "error", err) + return + } + go s.handleVsockConnection(conn) + } +} + +func (s *ShimServer) handleVsockConnection(conn net.Conn) { + defer conn.Close() + + // Read the CONNECT command + buf := make([]byte, 256) + n, err := conn.Read(buf) + if err != nil { + slog.Error("failed to read vsock handshake", "error", err) + return + } + + // Parse "CONNECT {port}\n" + var port uint32 + cmd := string(buf[:n]) + if _, err := fmt.Sscanf(cmd, "CONNECT %d\n", &port); err != nil { + slog.Error("invalid vsock handshake", "cmd", cmd, "error", err) + conn.Write([]byte(fmt.Sprintf("ERR invalid command: %s", cmd))) + return + } + + slog.Debug("vsock connect request", "port", port) + + // Get vsock device and connect to guest + s.mu.RLock() + socketDevices := s.vm.SocketDevices() + s.mu.RUnlock() + + if len(socketDevices) == 0 { + slog.Error("no vsock device configured") + conn.Write([]byte("ERR no vsock device\n")) + return + } + + guestConn, err := socketDevices[0].Connect(port) + if err != nil { + slog.Error("failed to connect to guest vsock", "port", port, "error", err) + conn.Write([]byte(fmt.Sprintf("ERR connect failed: %v\n", err))) + return + } + defer guestConn.Close() + + // Send OK response (matching CH protocol) + if _, err := conn.Write([]byte(fmt.Sprintf("OK %d\n", port))); err != nil { + slog.Error("failed to send OK response", "error", err) + return + } + + slog.Debug("vsock connection established", "port", port) + + // Proxy data bidirectionally + done := make(chan struct{}, 2) + + go func() { + copyData(guestConn, conn) + done <- struct{}{} + }() + + go func() { + copyData(conn, guestConn) + done <- struct{}{} + }() + + // Wait for one direction to close + <-done +} + +func copyData(dst, src net.Conn) { + buf := make([]byte, 32*1024) + for { + n, err := src.Read(buf) + if n > 0 { + if _, werr := dst.Write(buf[:n]); werr != nil { + return + } + } + if err != nil { + return + } + } +} diff --git a/cmd/vz-shim/vm.go b/cmd/vz-shim/vm.go new file mode 100644 index 00000000..bb628f60 --- /dev/null +++ b/cmd/vz-shim/vm.go @@ -0,0 +1,264 @@ +//go:build darwin + +package main + +import ( + "fmt" + "log/slog" + "net" + "os" + "runtime" + "strings" + + "github.com/Code-Hex/vz/v3" +) + +// createVM creates and configures a vz.VirtualMachine from ShimConfig. +func createVM(config ShimConfig) (*vz.VirtualMachine, *vz.VirtualMachineConfiguration, error) { + // Prepare kernel command line (vz uses hvc0 for serial console) + kernelArgs := config.KernelArgs + if kernelArgs == "" { + kernelArgs = "console=hvc0 root=/dev/vda" + } else { + kernelArgs = strings.ReplaceAll(kernelArgs, "console=ttyS0", "console=hvc0") + } + + bootLoader, err := vz.NewLinuxBootLoader( + config.KernelPath, + vz.WithCommandLine(kernelArgs), + vz.WithInitrd(config.InitrdPath), + ) + if err != nil { + return nil, nil, fmt.Errorf("create boot loader: %w", err) + } + + vcpus := computeCPUCount(config.VCPUs) + memoryBytes := computeMemorySize(uint64(config.MemoryBytes)) + + slog.Debug("VM config", "vcpus", vcpus, "memory_bytes", memoryBytes, "kernel", config.KernelPath, "initrd", config.InitrdPath) + + vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, vcpus, memoryBytes) + if err != nil { + return nil, nil, fmt.Errorf("create vm configuration: %w", err) + } + + if err := configureSerialConsole(vmConfig, config.SerialLogPath); err != nil { + return nil, nil, fmt.Errorf("configure serial: %w", err) + } + + if err := configureNetwork(vmConfig, config.Networks); err != nil { + return nil, nil, fmt.Errorf("configure network: %w", err) + } + + entropyConfig, err := vz.NewVirtioEntropyDeviceConfiguration() + if err != nil { + return nil, nil, fmt.Errorf("create entropy device: %w", err) + } + vmConfig.SetEntropyDevicesVirtualMachineConfiguration([]*vz.VirtioEntropyDeviceConfiguration{entropyConfig}) + + if err := configureStorage(vmConfig, config.Disks); err != nil { + return nil, nil, fmt.Errorf("configure storage: %w", err) + } + + vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() + if err != nil { + return nil, nil, fmt.Errorf("create vsock device: %w", err) + } + vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig}) + + if balloonConfig, err := vz.NewVirtioTraditionalMemoryBalloonDeviceConfiguration(); err == nil { + vmConfig.SetMemoryBalloonDevicesVirtualMachineConfiguration([]vz.MemoryBalloonDeviceConfiguration{balloonConfig}) + } + + if validated, err := vmConfig.Validate(); !validated || err != nil { + return nil, nil, fmt.Errorf("invalid vm configuration: %w", err) + } + + // Note: ValidateSaveRestoreSupport() returns true but Linux VM restore + // still fails with "invalid argument". This is an undocumented limitation + // of Virtualization.framework - only macOS guests support save/restore. + + vm, err := vz.NewVirtualMachine(vmConfig) + if err != nil { + return nil, nil, fmt.Errorf("create virtual machine: %w", err) + } + + return vm, vmConfig, nil +} + +func configureSerialConsole(vmConfig *vz.VirtualMachineConfiguration, logPath string) error { + var serialAttachment *vz.FileHandleSerialPortAttachment + + nullRead, err := os.OpenFile("/dev/null", os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("open /dev/null for reading: %w", err) + } + + if logPath != "" { + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + nullRead.Close() + return fmt.Errorf("open serial log file: %w", err) + } + serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, file) + if err != nil { + nullRead.Close() + file.Close() + return fmt.Errorf("create serial attachment: %w", err) + } + } else { + nullWrite, err := os.OpenFile("/dev/null", os.O_WRONLY, 0) + if err != nil { + nullRead.Close() + return fmt.Errorf("open /dev/null for writing: %w", err) + } + serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, nullWrite) + if err != nil { + nullRead.Close() + nullWrite.Close() + return fmt.Errorf("create serial attachment: %w", err) + } + } + + consoleConfig, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(serialAttachment) + if err != nil { + return fmt.Errorf("create console config: %w", err) + } + vmConfig.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{ + consoleConfig, + }) + + return nil +} + +func configureNetwork(vmConfig *vz.VirtualMachineConfiguration, networks []NetworkConfig) error { + if len(networks) == 0 { + return addNATNetwork(vmConfig, "") + } + for _, netConfig := range networks { + if err := addNATNetwork(vmConfig, netConfig.MAC); err != nil { + return err + } + } + return nil +} + +func addNATNetwork(vmConfig *vz.VirtualMachineConfiguration, macAddr string) error { + natAttachment, err := vz.NewNATNetworkDeviceAttachment() + if err != nil { + return fmt.Errorf("create NAT attachment: %w", err) + } + + networkConfig, err := vz.NewVirtioNetworkDeviceConfiguration(natAttachment) + if err != nil { + return fmt.Errorf("create network config: %w", err) + } + + var mac *vz.MACAddress + if macAddr != "" { + hwAddr, parseErr := net.ParseMAC(macAddr) + if parseErr == nil { + mac, err = vz.NewMACAddress(hwAddr) + if err != nil { + slog.Warn("failed to create MAC from parsed address, generating random", "mac", macAddr, "error", err) + mac, err = vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return fmt.Errorf("generate MAC address: %w", err) + } + } else { + slog.Info("using specified MAC address", "mac", macAddr) + } + } else { + slog.Warn("failed to parse MAC address, generating random", "mac", macAddr, "error", parseErr) + mac, err = vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return fmt.Errorf("generate MAC address: %w", err) + } + } + } else { + mac, err = vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return fmt.Errorf("generate MAC address: %w", err) + } + slog.Info("generated random MAC address", "mac", mac.String()) + } + networkConfig.SetMACAddress(mac) + + vmConfig.SetNetworkDevicesVirtualMachineConfiguration([]*vz.VirtioNetworkDeviceConfiguration{ + networkConfig, + }) + + return nil +} + +func configureStorage(vmConfig *vz.VirtualMachineConfiguration, disks []DiskConfig) error { + var storageDevices []vz.StorageDeviceConfiguration + + for _, disk := range disks { + if _, err := os.Stat(disk.Path); os.IsNotExist(err) { + return fmt.Errorf("disk image not found: %s", disk.Path) + } + + if strings.HasSuffix(disk.Path, ".qcow2") { + return fmt.Errorf("qcow2 not supported by vz, use raw format: %s", disk.Path) + } + + attachment, err := vz.NewDiskImageStorageDeviceAttachment(disk.Path, disk.Readonly) + if err != nil { + return fmt.Errorf("create disk attachment for %s: %w", disk.Path, err) + } + + blockConfig, err := vz.NewVirtioBlockDeviceConfiguration(attachment) + if err != nil { + return fmt.Errorf("create block device config: %w", err) + } + + storageDevices = append(storageDevices, blockConfig) + } + + if len(storageDevices) > 0 { + vmConfig.SetStorageDevicesVirtualMachineConfiguration(storageDevices) + } + + return nil +} + +func computeCPUCount(requested int) uint { + virtualCPUCount := uint(requested) + if virtualCPUCount == 0 { + virtualCPUCount = uint(runtime.NumCPU() - 1) + if virtualCPUCount < 1 { + virtualCPUCount = 1 + } + } + + maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedCPUCount() + minAllowed := vz.VirtualMachineConfigurationMinimumAllowedCPUCount() + + if virtualCPUCount > maxAllowed { + virtualCPUCount = maxAllowed + } + if virtualCPUCount < minAllowed { + virtualCPUCount = minAllowed + } + + return virtualCPUCount +} + +func computeMemorySize(requested uint64) uint64 { + if requested == 0 { + requested = 2 * 1024 * 1024 * 1024 // 2GB default + } + + maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedMemorySize() + minAllowed := vz.VirtualMachineConfigurationMinimumAllowedMemorySize() + + if requested > maxAllowed { + requested = maxAllowed + } + if requested < minAllowed { + requested = minAllowed + } + + return requested +} diff --git a/go.mod b/go.mod index 16a40d39..5102f359 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,8 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect + github.com/Code-Hex/vz/v3 v3.7.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect diff --git a/go.sum b/go.sum index 6fd5278f..1b933d0b 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Code-Hex/go-infinity-channel v1.0.0 h1:M8BWlfDOxq9or9yvF9+YkceoTkDI1pFAqvnP87Zh0Nw= +github.com/Code-Hex/go-infinity-channel v1.0.0/go.mod h1:5yUVg/Fqao9dAjcpzoQ33WwfdMWmISOrQloDRn3bsvY= +github.com/Code-Hex/vz/v3 v3.7.1 h1:EN1yNiyrbPq+dl388nne2NySo8I94EnPppvqypA65XM= +github.com/Code-Hex/vz/v3 v3.7.1/go.mod h1:1LsW0jqW0r0cQ+IeR4hHbjdqOtSidNCVMWhStMHGho8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 3a612baa..336600b6 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -504,9 +504,14 @@ func (m *manager) waitForResult(ctx context.Context, inst *instances.Instance) ( default: } - conn, err = m.dialBuilderVsock(inst.VsockSocket) - if err == nil { - break + dialer, dialerErr := m.instanceManager.GetVsockDialer(ctx, inst.Id) + if dialerErr == nil { + conn, err = dialer.DialVsock(ctx, BuildAgentVsockPort) + if err == nil { + break + } + } else { + err = dialerErr } m.logger.Debug("waiting for builder agent", "attempt", attempt+1, "error", err) diff --git a/lib/devices/discovery_darwin.go b/lib/devices/discovery_darwin.go new file mode 100644 index 00000000..7541c7bd --- /dev/null +++ b/lib/devices/discovery_darwin.go @@ -0,0 +1,57 @@ +//go:build darwin + +package devices + +import ( + "fmt" +) + +const ( + sysfsDevicesPath = "/sys/bus/pci/devices" // Not used on macOS + sysfsIOMMUPath = "/sys/kernel/iommu_groups" +) + +// ErrNotSupportedOnMacOS is returned for operations not supported on macOS +var ErrNotSupportedOnMacOS = fmt.Errorf("PCI device passthrough is not supported on macOS") + +// ValidatePCIAddress validates that a string is a valid PCI address format. +// On macOS, this always returns false as PCI passthrough is not supported. +func ValidatePCIAddress(addr string) bool { + return false +} + +// DiscoverAvailableDevices returns an empty list on macOS. +// PCI device passthrough is not supported on macOS. +func DiscoverAvailableDevices() ([]AvailableDevice, error) { + return []AvailableDevice{}, nil +} + +// GetDeviceInfo returns an error on macOS as PCI passthrough is not supported. +func GetDeviceInfo(pciAddress string) (*AvailableDevice, error) { + return nil, ErrNotSupportedOnMacOS +} + +// GetIOMMUGroupDevices returns an error on macOS as IOMMU is not available. +func GetIOMMUGroupDevices(iommuGroup int) ([]string, error) { + return nil, ErrNotSupportedOnMacOS +} + +// DetermineDeviceType returns DeviceTypeGeneric on macOS. +func DetermineDeviceType(device *AvailableDevice) DeviceType { + return DeviceTypeGeneric +} + +// readSysfsFile is not available on macOS. +func readSysfsFile(path string) (string, error) { + return "", ErrNotSupportedOnMacOS +} + +// readIOMMUGroup is not available on macOS. +func readIOMMUGroup(pciAddress string) (int, error) { + return -1, ErrNotSupportedOnMacOS +} + +// readCurrentDriver is not available on macOS. +func readCurrentDriver(pciAddress string) *string { + return nil +} diff --git a/lib/devices/discovery.go b/lib/devices/discovery_linux.go similarity index 99% rename from lib/devices/discovery.go rename to lib/devices/discovery_linux.go index b04213c0..33798292 100644 --- a/lib/devices/discovery.go +++ b/lib/devices/discovery_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package devices import ( diff --git a/lib/devices/manager.go b/lib/devices/manager.go index d93a7572..6c0d84b6 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "runtime" "strings" "sync" "time" @@ -552,6 +553,11 @@ func (m *manager) ReconcileDevices(ctx context.Context) error { func (m *manager) validatePrerequisites(ctx context.Context) { log := logger.FromContext(ctx) + // Skip GPU passthrough checks on macOS - not supported + if runtime.GOOS == "darwin" { + return + } + // Check IOMMU availability iommuGroupsDir := "/sys/kernel/iommu_groups" entries, err := os.ReadDir(iommuGroupsDir) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go new file mode 100644 index 00000000..dacca12f --- /dev/null +++ b/lib/devices/mdev_darwin.go @@ -0,0 +1,57 @@ +//go:build darwin + +package devices + +import ( + "context" + "fmt" +) + +// ErrVGPUNotSupportedOnMacOS is returned for vGPU operations on macOS +var ErrVGPUNotSupportedOnMacOS = fmt.Errorf("vGPU (mdev) is not supported on macOS") + +// SetGPUProfileCacheTTL is a no-op on macOS. +func SetGPUProfileCacheTTL(ttl string) { + // No-op on macOS +} + +// DiscoverVFs returns an empty list on macOS. +// SR-IOV Virtual Functions are not available on macOS. +func DiscoverVFs() ([]VirtualFunction, error) { + return []VirtualFunction{}, nil +} + +// ListGPUProfiles returns an empty list on macOS. +func ListGPUProfiles() ([]GPUProfile, error) { + return []GPUProfile{}, nil +} + +// ListGPUProfilesWithVFs returns an empty list on macOS. +func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { + return []GPUProfile{}, nil +} + +// ListMdevDevices returns an empty list on macOS. +func ListMdevDevices() ([]MdevDevice, error) { + return []MdevDevice{}, nil +} + +// CreateMdev returns an error on macOS as mdev is not supported. +func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevice, error) { + return nil, ErrVGPUNotSupportedOnMacOS +} + +// DestroyMdev is a no-op on macOS. +func DestroyMdev(ctx context.Context, mdevUUID string) error { + return nil +} + +// IsMdevInUse returns false on macOS. +func IsMdevInUse(mdevUUID string) bool { + return false +} + +// ReconcileMdevs is a no-op on macOS. +func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { + return nil +} diff --git a/lib/devices/mdev.go b/lib/devices/mdev_linux.go similarity index 98% rename from lib/devices/mdev.go rename to lib/devices/mdev_linux.go index de648e05..2e5bab44 100644 --- a/lib/devices/mdev.go +++ b/lib/devices/mdev_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package devices import ( @@ -604,13 +606,6 @@ func IsMdevInUse(mdevUUID string) bool { return err == nil // Has a driver = in use } -// MdevReconcileInfo contains information needed to reconcile mdevs for an instance -type MdevReconcileInfo struct { - InstanceID string - MdevUUID string - IsRunning bool // true if instance's VMM is running or state is unknown -} - // ReconcileMdevs destroys orphaned mdevs that belong to hypeman but are no longer in use. // This is called on server startup to clean up stale mdevs from previous runs. // diff --git a/lib/devices/types.go b/lib/devices/types.go index bd66fa86..d436ca1d 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -94,3 +94,10 @@ type PassthroughDevice struct { Name string `json:"name"` // GPU name, e.g., "NVIDIA L40S" Available bool `json:"available"` // true if not attached to an instance } + +// MdevReconcileInfo contains information needed to reconcile mdevs for an instance +type MdevReconcileInfo struct { + InstanceID string + MdevUUID string + IsRunning bool // true if instance's VMM is running or state is unknown +} diff --git a/lib/devices/vfio_darwin.go b/lib/devices/vfio_darwin.go new file mode 100644 index 00000000..ae47cbcd --- /dev/null +++ b/lib/devices/vfio_darwin.go @@ -0,0 +1,74 @@ +//go:build darwin + +package devices + +import ( + "fmt" +) + +// ErrVFIONotSupportedOnMacOS is returned for VFIO operations on macOS +var ErrVFIONotSupportedOnMacOS = fmt.Errorf("VFIO device passthrough is not supported on macOS") + +// VFIOBinder handles binding and unbinding devices to/from VFIO. +// On macOS, this is a stub that returns errors for all operations. +type VFIOBinder struct{} + +// NewVFIOBinder creates a new VFIOBinder +func NewVFIOBinder() *VFIOBinder { + return &VFIOBinder{} +} + +// IsVFIOAvailable returns false on macOS as VFIO is not available. +func (v *VFIOBinder) IsVFIOAvailable() bool { + return false +} + +// IsDeviceBoundToVFIO returns false on macOS. +func (v *VFIOBinder) IsDeviceBoundToVFIO(pciAddress string) bool { + return false +} + +// BindToVFIO returns an error on macOS as VFIO is not supported. +func (v *VFIOBinder) BindToVFIO(pciAddress string) error { + return ErrVFIONotSupportedOnMacOS +} + +// UnbindFromVFIO returns an error on macOS as VFIO is not supported. +func (v *VFIOBinder) UnbindFromVFIO(pciAddress string) error { + return ErrVFIONotSupportedOnMacOS +} + +// GetVFIOGroupPath returns an error on macOS as VFIO is not supported. +func (v *VFIOBinder) GetVFIOGroupPath(pciAddress string) (string, error) { + return "", ErrVFIONotSupportedOnMacOS +} + +// CheckIOMMUGroupSafe returns an error on macOS as IOMMU is not available. +func (v *VFIOBinder) CheckIOMMUGroupSafe(pciAddress string, allowedDevices []string) error { + return ErrVFIONotSupportedOnMacOS +} + +// GetDeviceSysfsPath returns an empty string on macOS. +func GetDeviceSysfsPath(pciAddress string) string { + return "" +} + +// unbindFromDriver is not available on macOS. +func (v *VFIOBinder) unbindFromDriver(pciAddress, driver string) error { + return ErrVFIONotSupportedOnMacOS +} + +// setDriverOverride is not available on macOS. +func (v *VFIOBinder) setDriverOverride(pciAddress, driver string) error { + return ErrVFIONotSupportedOnMacOS +} + +// triggerDriverProbe is not available on macOS. +func (v *VFIOBinder) triggerDriverProbe(pciAddress string) error { + return ErrVFIONotSupportedOnMacOS +} + +// startNvidiaPersistenced is not available on macOS. +func (v *VFIOBinder) startNvidiaPersistenced() error { + return nil // No-op, not an error +} diff --git a/lib/devices/vfio.go b/lib/devices/vfio_linux.go similarity index 99% rename from lib/devices/vfio.go rename to lib/devices/vfio_linux.go index 38606f5b..65be8104 100644 --- a/lib/devices/vfio.go +++ b/lib/devices/vfio_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package devices import ( diff --git a/lib/hypervisor/README.md b/lib/hypervisor/README.md index 2bab53d9..3eafd673 100644 --- a/lib/hypervisor/README.md +++ b/lib/hypervisor/README.md @@ -4,20 +4,29 @@ Provides a common interface for VM management across different hypervisors. ## Purpose -Hypeman originally supported only Cloud Hypervisor. This abstraction layer allows supporting multiple hypervisors (e.g., QEMU) through a unified interface, enabling: +Hypeman originally supported only Cloud Hypervisor. This abstraction layer allows supporting multiple hypervisors through a unified interface, enabling: - **Hypervisor choice per instance** - Different instances can use different hypervisors +- **Platform support** - Linux uses Cloud Hypervisor/QEMU, macOS uses Virtualization.framework - **Feature parity where possible** - Common operations work the same way - **Graceful degradation** - Features unsupported by a hypervisor can be detected and handled +## Implementations + +| Hypervisor | Platform | Process Model | Control Interface | +|------------|----------|---------------|-------------------| +| Cloud Hypervisor | Linux | External process | HTTP API over Unix socket | +| QEMU | Linux | External process | QMP over Unix socket | +| vz | macOS | In-process | Direct API calls | + ## How It Works The abstraction defines two key interfaces: 1. **Hypervisor** - VM lifecycle operations (create, boot, pause, resume, snapshot, restore, shutdown) -2. **ProcessManager** - Hypervisor process lifecycle (start binary, get binary path) +2. **VMStarter** - VM startup and configuration (start binary, get binary path) -Each hypervisor implementation translates the generic configuration and operations to its native format. For example, Cloud Hypervisor uses an HTTP API over a Unix socket, while QEMU would use QMP. +Each implementation translates generic configuration to its native format. Cloud Hypervisor and QEMU run as external processes with socket-based control. The vz implementation runs VMs in-process using Apple's Virtualization.framework. Before using optional features, callers check capabilities: @@ -27,6 +36,19 @@ if hv.Capabilities().SupportsSnapshot { } ``` +## Platform Differences + +### Linux (Cloud Hypervisor, QEMU) +- VMs run as separate processes with PIDs +- State persists across hypeman restarts (reconnect via socket) +- TAP devices and Linux bridges for networking + +### macOS (vz) +- VMs run in-process (no separate PID) +- VMs stop if hypeman stops (cannot reconnect) +- NAT networking via Virtualization.framework +- Requires code signing with virtualization entitlement + ## Hypervisor Switching Instances store their hypervisor type in metadata. An instance can switch hypervisors only when stopped (no running VM, no snapshot), since: diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index b81b72d4..c30b6c3d 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -15,6 +15,9 @@ import ( func init() { hypervisor.RegisterSocketName(hypervisor.TypeCloudHypervisor, "ch.sock") + hypervisor.RegisterClientFactory(hypervisor.TypeCloudHypervisor, func(socketPath string) (hypervisor.Hypervisor, error) { + return New(socketPath) + }) } // Starter implements hypervisor.VMStarter for Cloud Hypervisor. diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 197a6ac7..b4287a79 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -5,6 +5,7 @@ package hypervisor import ( "context" + "errors" "fmt" "net" "time" @@ -12,6 +13,16 @@ import ( "github.com/kernel/hypeman/lib/paths" ) +// Common errors +var ( + // ErrHypervisorNotRunning is returned when trying to connect to a hypervisor + // that is not currently running or cannot be reconnected to. + ErrHypervisorNotRunning = errors.New("hypervisor is not running") + + // ErrNotSupported is returned when an operation is not supported by the hypervisor. + ErrNotSupported = errors.New("operation not supported by this hypervisor") +) + // Type identifies the hypervisor implementation type Type string @@ -20,6 +31,8 @@ const ( TypeCloudHypervisor Type = "cloud-hypervisor" // TypeQEMU is the QEMU VMM TypeQEMU Type = "qemu" + // TypeVZ is the Virtualization.framework VMM (macOS only) + TypeVZ Type = "vz" ) // socketNames maps hypervisor types to their socket filenames. @@ -164,3 +177,23 @@ func NewVsockDialer(hvType Type, vsockSocket string, vsockCID int64) (VsockDiale } return factory(vsockSocket, vsockCID), nil } + +// ClientFactory creates Hypervisor client instances for a hypervisor type. +type ClientFactory func(socketPath string) (Hypervisor, error) + +// clientFactories maps hypervisor types to their client factories. +var clientFactories = make(map[Type]ClientFactory) + +// RegisterClientFactory registers a Hypervisor client factory. +func RegisterClientFactory(t Type, factory ClientFactory) { + clientFactories[t] = factory +} + +// NewClient creates a Hypervisor client for the given type and socket. +func NewClient(hvType Type, socketPath string) (Hypervisor, error) { + factory, ok := clientFactories[hvType] + if !ok { + return nil, fmt.Errorf("no client factory registered for hypervisor type: %s", hvType) + } + return factory(socketPath) +} diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 459d94eb..e2e1d098 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -37,6 +37,9 @@ const ( func init() { hypervisor.RegisterSocketName(hypervisor.TypeQEMU, "qemu.sock") + hypervisor.RegisterClientFactory(hypervisor.TypeQEMU, func(socketPath string) (hypervisor.Hypervisor, error) { + return New(socketPath) + }) } // Starter implements hypervisor.VMStarter for QEMU. diff --git a/lib/hypervisor/qemu/vsock.go b/lib/hypervisor/qemu/vsock.go index 50c0791f..88be6cc5 100644 --- a/lib/hypervisor/qemu/vsock.go +++ b/lib/hypervisor/qemu/vsock.go @@ -1,3 +1,5 @@ +//go:build linux + package qemu import ( diff --git a/lib/hypervisor/vz/client.go b/lib/hypervisor/vz/client.go new file mode 100644 index 00000000..ae5603ed --- /dev/null +++ b/lib/hypervisor/vz/client.go @@ -0,0 +1,233 @@ +//go:build darwin + +package vz + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// Client implements hypervisor.Hypervisor via HTTP to the vz-shim process. +type Client struct { + socketPath string + httpClient *http.Client +} + +// NewClient creates a new vz shim client. +func NewClient(socketPath string) (*Client, error) { + transport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + } + httpClient := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + } + + // Verify connectivity + resp, err := httpClient.Get("http://vz-shim/api/v1/vmm.ping") + if err != nil { + return nil, fmt.Errorf("ping shim: %w", err) + } + resp.Body.Close() + + return &Client{ + socketPath: socketPath, + httpClient: httpClient, + }, nil +} + +// Verify Client implements the interface +var _ hypervisor.Hypervisor = (*Client)(nil) + +// vmInfoResponse matches the shim's VMInfoResponse structure. +type vmInfoResponse struct { + State string `json:"state"` +} + +// Capabilities returns the features supported by vz. +func (c *Client) Capabilities() hypervisor.Capabilities { + return hypervisor.Capabilities{ + // Snapshot NOT supported: Virtualization.framework does not support + // save/restore for Linux guest VMs - only macOS guests work. + // This is an undocumented limitation of the framework. + // See: https://github.com/cirruslabs/tart/issues/1177 + // See: https://github.com/cirruslabs/tart/issues/796 + // See: https://github.com/utmapp/UTM/issues/6654 + SupportsSnapshot: false, + SupportsHotplugMemory: false, + SupportsPause: true, + SupportsVsock: true, + SupportsGPUPassthrough: false, + SupportsDiskIOLimit: false, + } +} + +// DeleteVM requests a graceful shutdown of the guest. +func (c *Client) DeleteVM(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim/api/v1/vm.shutdown", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("shutdown request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return fmt.Errorf("shutdown failed with status %d", resp.StatusCode) + } + + return nil +} + +// Shutdown stops the VMM (shim) forcefully. +func (c *Client) Shutdown(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim/api/v1/vmm.shutdown", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + // Connection reset is expected when shim exits + return nil + } + defer resp.Body.Close() + + return nil +} + +// GetVMInfo returns current VM state information. +func (c *Client) GetVMInfo(ctx context.Context) (*hypervisor.VMInfo, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vz-shim/api/v1/vm.info", nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("get vm info: %w", err) + } + defer resp.Body.Close() + + var info vmInfoResponse + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return nil, fmt.Errorf("decode vm info: %w", err) + } + + var state hypervisor.VMState + switch info.State { + case "Running": + state = hypervisor.StateRunning + case "Paused": + state = hypervisor.StatePaused + case "Shutdown", "Stopped": + state = hypervisor.StateShutdown + default: + state = hypervisor.StateRunning + } + + return &hypervisor.VMInfo{State: state}, nil +} + +// Pause suspends VM execution. +func (c *Client) Pause(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim/api/v1/vm.pause", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("pause request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return fmt.Errorf("pause failed with status %d", resp.StatusCode) + } + + return nil +} + +// Resume continues VM execution after pause. +func (c *Client) Resume(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim/api/v1/vm.resume", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("resume request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return fmt.Errorf("resume failed with status %d", resp.StatusCode) + } + + return nil +} + +// snapshotRequest matches the shim's SnapshotRequest structure. +type snapshotRequest struct { + DestinationURL string `json:"destination_url"` +} + +// Snapshot saves the VM state to a file. VM must be paused first. +// Requires macOS 14+ on ARM64. +// Note: destPath is expected to be a directory (matching CH convention). +// vz expects a file path, so we append "vz-state" to the directory. +func (c *Client) Snapshot(ctx context.Context, destPath string) error { + // vz SaveMachineStateToPath expects a file path, not a directory + // Append a fixed filename to match the directory-based API of other hypervisors + statePath := destPath + "/vz-state" + + reqBody := snapshotRequest{DestinationURL: statePath} + body, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("marshal snapshot request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim/api/v1/vm.snapshot", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("snapshot request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("snapshot failed with status %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// ResizeMemory is not supported by vz. +func (c *Client) ResizeMemory(ctx context.Context, bytes int64) error { + return fmt.Errorf("memory resize not supported by vz") +} + +// ResizeMemoryAndWait is not supported by vz. +func (c *Client) ResizeMemoryAndWait(ctx context.Context, bytes int64, timeout time.Duration) error { + return fmt.Errorf("memory resize not supported by vz") +} diff --git a/lib/hypervisor/vz/client_test.go b/lib/hypervisor/vz/client_test.go new file mode 100644 index 00000000..414f7662 --- /dev/null +++ b/lib/hypervisor/vz/client_test.go @@ -0,0 +1,49 @@ +//go:build darwin + +package vz + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestClientCapabilities(t *testing.T) { + // Create a mock client (without actual socket connection) + // We can't create a real client without a running shim + c := &Client{ + socketPath: "/nonexistent/socket", + httpClient: nil, // Will fail if actually used + } + + caps := c.Capabilities() + + // Verify expected capabilities + assert.False(t, caps.SupportsSnapshot, "Snapshot not supported: Virtualization.framework limitation for Linux guests") + assert.True(t, caps.SupportsPause, "vz supports pause") + assert.True(t, caps.SupportsVsock, "vz supports vsock") + assert.False(t, caps.SupportsHotplugMemory, "vz does not support memory hotplug") + assert.False(t, caps.SupportsGPUPassthrough, "vz does not support GPU passthrough") + assert.False(t, caps.SupportsDiskIOLimit, "vz does not support disk I/O limits") +} + +func TestVzMetadataStructure(t *testing.T) { + // Test that vzMetadata can be unmarshaled from stored instance metadata + metadataJSON := `{ + "Image": "alpine:3.20", + "Vcpus": 2, + "Size": 1073741824, + "KernelVersion": "ch-6.12.8-kernel-1.3-202601152", + "MAC": "02:00:00:34:49:ae" + }` + + var metadata vzMetadata + err := json.Unmarshal([]byte(metadataJSON), &metadata) + assert.NoError(t, err) + assert.Equal(t, "alpine:3.20", metadata.Image) + assert.Equal(t, 2, metadata.VCPUs) + assert.Equal(t, int64(1073741824), metadata.Size) + assert.Equal(t, "ch-6.12.8-kernel-1.3-202601152", metadata.KernelVersion) + assert.Equal(t, "02:00:00:34:49:ae", metadata.MAC) +} diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go new file mode 100644 index 00000000..638dd0b4 --- /dev/null +++ b/lib/hypervisor/vz/starter.go @@ -0,0 +1,408 @@ +//go:build darwin + +// Package vz implements the hypervisor.Hypervisor interface for +// Apple's Virtualization.framework on macOS via the vz-shim subprocess. +package vz + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/system" +) + +func init() { + hypervisor.RegisterSocketName(hypervisor.TypeVZ, "vz.sock") + hypervisor.RegisterVsockDialerFactory(hypervisor.TypeVZ, NewVsockDialer) + hypervisor.RegisterClientFactory(hypervisor.TypeVZ, func(socketPath string) (hypervisor.Hypervisor, error) { + return NewClient(socketPath) + }) +} + +// ShimConfig is the configuration passed to the vz-shim process. +type ShimConfig struct { + VCPUs int `json:"vcpus"` + MemoryBytes int64 `json:"memory_bytes"` + Disks []DiskConfig `json:"disks"` + Networks []NetworkConfig `json:"networks"` + SerialLogPath string `json:"serial_log_path"` + KernelPath string `json:"kernel_path"` + InitrdPath string `json:"initrd_path"` + KernelArgs string `json:"kernel_args"` + ControlSocket string `json:"control_socket"` + VsockSocket string `json:"vsock_socket"` + LogPath string `json:"log_path"` + RestoreStatePath string `json:"restore_state_path,omitempty"` +} + +// DiskConfig for shim. +type DiskConfig struct { + Path string `json:"path"` + Readonly bool `json:"readonly"` +} + +// NetworkConfig for shim. +type NetworkConfig struct { + MAC string `json:"mac"` +} + +// Starter implements hypervisor.VMStarter for Virtualization.framework. +type Starter struct{} + +// NewStarter creates a new vz starter. +func NewStarter() *Starter { + return &Starter{} +} + +// Verify Starter implements the interface +var _ hypervisor.VMStarter = (*Starter)(nil) + +// SocketName returns the socket filename for vz. +func (s *Starter) SocketName() string { + return "vz.sock" +} + +// GetBinaryPath returns empty - vz uses system Virtualization.framework. +func (s *Starter) GetBinaryPath(p *paths.Paths, version string) (string, error) { + return "", nil +} + +// GetVersion returns the macOS version as the "hypervisor version". +func (s *Starter) GetVersion(p *paths.Paths) (string, error) { + // Return a version indicating vz availability + return "vz-macos", nil +} + +// StartVM spawns a vz-shim subprocess to host the VM. +// Returns the shim PID and a client to control the VM. +func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { + log := logger.FromContext(ctx) + + // Derive socket paths from the control socket path + instanceDir := filepath.Dir(socketPath) + controlSocket := socketPath + vsockSocket := filepath.Join(instanceDir, "vz.vsock") + logPath := filepath.Join(instanceDir, "logs", "vz-shim.log") + + // Build shim config + shimConfig := ShimConfig{ + VCPUs: config.VCPUs, + MemoryBytes: config.MemoryBytes, + SerialLogPath: config.SerialLogPath, + KernelPath: config.KernelPath, + InitrdPath: config.InitrdPath, + KernelArgs: config.KernelArgs, + ControlSocket: controlSocket, + VsockSocket: vsockSocket, + LogPath: logPath, + } + + // Convert disks + for _, disk := range config.Disks { + shimConfig.Disks = append(shimConfig.Disks, DiskConfig{ + Path: disk.Path, + Readonly: disk.Readonly, + }) + } + + // Convert networks + for _, net := range config.Networks { + shimConfig.Networks = append(shimConfig.Networks, NetworkConfig{ + MAC: net.MAC, + }) + } + + configJSON, err := json.Marshal(shimConfig) + if err != nil { + return 0, nil, fmt.Errorf("marshal shim config: %w", err) + } + + log.DebugContext(ctx, "spawning vz-shim", "config", string(configJSON)) + + // Find the vz-shim binary (same directory as hypeman or in PATH) + shimPath, err := s.findShimBinary() + if err != nil { + return 0, nil, fmt.Errorf("find vz-shim binary: %w", err) + } + + // Spawn the shim process + cmd := exec.CommandContext(ctx, shimPath, "-config", string(configJSON)) + cmd.Stdout = nil // Shim logs to file + cmd.Stderr = nil + cmd.Stdin = nil + + if err := cmd.Start(); err != nil { + return 0, nil, fmt.Errorf("start vz-shim: %w", err) + } + + pid := cmd.Process.Pid + log.InfoContext(ctx, "vz-shim started", "pid", pid, "control_socket", controlSocket) + + // Wait for the control socket to be ready + client, err := s.waitForShim(ctx, controlSocket, 30*time.Second) + if err != nil { + // Kill the shim if we can't connect + cmd.Process.Kill() + return 0, nil, fmt.Errorf("connect to vz-shim: %w", err) + } + + // Release the process so it's not killed when cmd goes out of scope + cmd.Process.Release() + + return pid, client, nil +} + +// findShimBinary locates the vz-shim binary. +func (s *Starter) findShimBinary() (string, error) { + // First, check next to the current executable + exe, err := os.Executable() + if err == nil { + exeDir := filepath.Dir(exe) + shimPath := filepath.Join(exeDir, "vz-shim") + if _, err := os.Stat(shimPath); err == nil { + return shimPath, nil + } + // Also check parent's tmp dir (for air hot-reload development) + // When running ./tmp/main, check ./tmp/vz-shim + if filepath.Base(exeDir) == "tmp" { + shimPath = filepath.Join(exeDir, "vz-shim") + if _, err := os.Stat(shimPath); err == nil { + return shimPath, nil + } + } + } + + // Check in PATH + shimPath, err := exec.LookPath("vz-shim") + if err == nil { + return shimPath, nil + } + + // Check common locations + commonPaths := []string{ + "/usr/local/bin/vz-shim", + filepath.Join(os.Getenv("HOME"), "bin", "vz-shim"), + } + for _, p := range commonPaths { + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + + return "", fmt.Errorf("vz-shim binary not found") +} + +// waitForShim waits for the shim's control socket to be ready. +func (s *Starter) waitForShim(ctx context.Context, socketPath string, timeout time.Duration) (*Client, error) { + deadline := time.Now().Add(timeout) + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + client, err := NewClient(socketPath) + if err == nil { + return client, nil + } + + time.Sleep(100 * time.Millisecond) + } + + return nil, fmt.Errorf("timeout waiting for shim socket: %s", socketPath) +} + +// vzMetadata is the subset of instance metadata needed for restore. +type vzMetadata struct { + Image string `json:"Image"` + VCPUs int `json:"Vcpus"` + Size int64 `json:"Size"` // memory in bytes + KernelVersion string `json:"KernelVersion"` + MAC string `json:"MAC"` +} + +// RestoreVM restores a VM from a snapshot. +// Unlike Cloud Hypervisor, vz snapshots only contain CPU/memory state, not VM config. +// We load the VM config from metadata.json in the instance directory. +func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, socketPath string, snapshotPath string) (int, hypervisor.Hypervisor, error) { + log := logger.FromContext(ctx) + + // Derive paths from socketPath (which is in the instance directory) + instanceDir := filepath.Dir(socketPath) + controlSocket := socketPath + vsockSocket := filepath.Join(instanceDir, "vz.vsock") + logPath := filepath.Join(instanceDir, "logs", "vz-shim.log") + + // The snapshot file is inside snapshotPath directory + restoreStatePath := filepath.Join(snapshotPath, "vz-state") + if _, err := os.Stat(restoreStatePath); err != nil { + return 0, nil, fmt.Errorf("snapshot file not found: %s", restoreStatePath) + } + + // Load metadata to get VM config + metadataPath := filepath.Join(instanceDir, "metadata.json") + metadataBytes, err := os.ReadFile(metadataPath) + if err != nil { + return 0, nil, fmt.Errorf("read metadata: %w", err) + } + + var metadata vzMetadata + if err := json.Unmarshal(metadataBytes, &metadata); err != nil { + return 0, nil, fmt.Errorf("parse metadata: %w", err) + } + + // Build disk list - order matters for vz (vda, vdb, vdc...) + var disks []DiskConfig + + // 1. Get rootfs disk path from image info + // Parse image name to get digest path + // The Image field is like "alpine:3.20" - we need to find the digest + // For now, scan the images directory for this image + imagesDir := p.ImagesDir() + imageRootfs, err := findImageRootfs(imagesDir, metadata.Image) + if err != nil { + return 0, nil, fmt.Errorf("find image rootfs: %w", err) + } + disks = append(disks, DiskConfig{Path: imageRootfs, Readonly: true}) + + // 2. Overlay disk + overlayDisk := filepath.Join(instanceDir, "overlay.raw") + if _, err := os.Stat(overlayDisk); err == nil { + disks = append(disks, DiskConfig{Path: overlayDisk, Readonly: false}) + } + + // 3. Config disk + configDisk := filepath.Join(instanceDir, "config.ext4") + if _, err := os.Stat(configDisk); err == nil { + disks = append(disks, DiskConfig{Path: configDisk, Readonly: true}) + } + + // Get kernel and initrd paths + arch := system.GetArch() + kernelPath := p.SystemKernel(metadata.KernelVersion, arch) + + // Resolve the initrd symlink to get the same path as original start + // This is critical for vz snapshot restore which requires identical paths + initrdLatest := p.SystemInitrdLatest(arch) + initrdTimestamp, err := os.Readlink(initrdLatest) + if err != nil { + return 0, nil, fmt.Errorf("read initrd symlink: %w", err) + } + initrdPath := p.SystemInitrdTimestamp(initrdTimestamp, arch) + + // Build shim config + shimConfig := ShimConfig{ + VCPUs: metadata.VCPUs, + MemoryBytes: metadata.Size, + Disks: disks, + Networks: []NetworkConfig{{MAC: metadata.MAC}}, + SerialLogPath: filepath.Join(instanceDir, "logs", "app.log"), + KernelPath: kernelPath, + InitrdPath: initrdPath, + KernelArgs: "console=hvc0", + ControlSocket: controlSocket, + VsockSocket: vsockSocket, + LogPath: logPath, + RestoreStatePath: restoreStatePath, + } + + configJSON, err := json.Marshal(shimConfig) + if err != nil { + return 0, nil, fmt.Errorf("marshal shim config: %w", err) + } + + log.DebugContext(ctx, "spawning vz-shim for restore", "config", string(configJSON), "snapshot", restoreStatePath) + + // Find the vz-shim binary + shimPath, err := s.findShimBinary() + if err != nil { + return 0, nil, fmt.Errorf("find vz-shim binary: %w", err) + } + + // Spawn the shim process + cmd := exec.CommandContext(ctx, shimPath, "-config", string(configJSON)) + cmd.Stdout = nil + cmd.Stderr = nil + cmd.Stdin = nil + + if err := cmd.Start(); err != nil { + return 0, nil, fmt.Errorf("start vz-shim: %w", err) + } + + pid := cmd.Process.Pid + log.InfoContext(ctx, "vz-shim started for restore", "pid", pid, "control_socket", controlSocket, "snapshot", restoreStatePath) + + // Wait for the control socket to be ready + client, err := s.waitForShim(ctx, controlSocket, 30*time.Second) + if err != nil { + cmd.Process.Kill() + return 0, nil, fmt.Errorf("connect to vz-shim: %w", err) + } + + // Release the process so it's not killed when cmd goes out of scope + cmd.Process.Release() + + return pid, client, nil +} + +// findImageRootfs locates the rootfs.ext4 for an image by name. +// This scans the images directory structure to find the latest version. +func findImageRootfs(imagesDir, imageName string) (string, error) { + // Images are stored as: {imagesDir}/{registry}/{repo}/{digest}/rootfs.ext4 + // For "alpine:3.20" -> docker.io/library/alpine/{sha256:xxx}/rootfs.ext4 + + // Normalize image name to path components + // Simple approach: walk the images directory looking for matching image + var foundPath string + err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil // Skip errors + } + if info.Name() == "rootfs.ext4" { + // Check if parent directory structure matches image name + // For now, just find any rootfs.ext4 that might match + dir := filepath.Dir(path) + // The path structure is like: .../alpine/{digest}/rootfs.ext4 + if containsImageName(dir, imageName) { + foundPath = path + return filepath.SkipAll + } + } + return nil + }) + if err != nil { + return "", err + } + if foundPath == "" { + return "", fmt.Errorf("rootfs not found for image: %s", imageName) + } + return foundPath, nil +} + +// containsImageName checks if a path contains the image name components. +func containsImageName(path, imageName string) bool { + // Extract just the image name without tag (e.g., "alpine" from "alpine:3.20") + parts := filepath.SplitList(imageName) + name := imageName + if idx := len(name) - 1; idx >= 0 { + for i := len(name) - 1; i >= 0; i-- { + if name[i] == ':' { + name = name[:i] + break + } + } + } + _ = parts + return filepath.Base(filepath.Dir(filepath.Dir(path))) == name || + filepath.Base(filepath.Dir(path)) == name +} diff --git a/lib/hypervisor/vz/vsock.go b/lib/hypervisor/vz/vsock.go new file mode 100644 index 00000000..456ff967 --- /dev/null +++ b/lib/hypervisor/vz/vsock.go @@ -0,0 +1,115 @@ +//go:build darwin + +package vz + +import ( + "bufio" + "context" + "fmt" + "log/slog" + "net" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +const ( + vsockDialTimeout = 5 * time.Second + vsockHandshakeTimeout = 5 * time.Second +) + +// VsockDialer implements hypervisor.VsockDialer for vz via the shim's Unix socket proxy. +// Uses the same protocol as Cloud Hypervisor: CONNECT {port}\n -> OK {port}\n +type VsockDialer struct { + socketPath string // path to vz.vsock Unix socket +} + +// NewVsockDialer creates a new VsockDialer for vz. +// The vsockSocket parameter should be the control socket path (vz.sock). +// We derive the vsock proxy socket path from it (vz.vsock). +func NewVsockDialer(vsockSocket string, vsockCID int64) hypervisor.VsockDialer { + // Derive vsock proxy socket path from control socket + dir := filepath.Dir(vsockSocket) + vsockProxySocket := filepath.Join(dir, "vz.vsock") + return &VsockDialer{ + socketPath: vsockProxySocket, + } +} + +// Key returns a unique identifier for this dialer, used for connection pooling. +func (d *VsockDialer) Key() string { + return "vz:" + d.socketPath +} + +// DialVsock connects to the guest on the specified port via the shim's vsock proxy. +func (d *VsockDialer) DialVsock(ctx context.Context, port int) (net.Conn, error) { + slog.DebugContext(ctx, "connecting to vsock via shim proxy", "socket", d.socketPath, "port", port) + + // Use dial timeout, respecting context deadline if shorter + dialTimeout := vsockDialTimeout + if deadline, ok := ctx.Deadline(); ok { + if remaining := time.Until(deadline); remaining < dialTimeout { + dialTimeout = remaining + } + } + + // Connect to the shim's vsock proxy Unix socket + dialer := net.Dialer{Timeout: dialTimeout} + conn, err := dialer.DialContext(ctx, "unix", d.socketPath) + if err != nil { + return nil, fmt.Errorf("dial vsock proxy socket %s: %w", d.socketPath, err) + } + + slog.DebugContext(ctx, "connected to vsock proxy, performing handshake", "port", port) + + // Set deadline for handshake + if err := conn.SetDeadline(time.Now().Add(vsockHandshakeTimeout)); err != nil { + conn.Close() + return nil, fmt.Errorf("set handshake deadline: %w", err) + } + + // Perform handshake (same protocol as Cloud Hypervisor) + handshakeCmd := fmt.Sprintf("CONNECT %d\n", port) + if _, err := conn.Write([]byte(handshakeCmd)); err != nil { + conn.Close() + return nil, fmt.Errorf("send vsock handshake: %w", err) + } + + // Read handshake response + reader := bufio.NewReader(conn) + response, err := reader.ReadString('\n') + if err != nil { + conn.Close() + return nil, fmt.Errorf("read vsock handshake response (is guest-agent running?): %w", err) + } + + // Clear deadline after successful handshake + if err := conn.SetDeadline(time.Time{}); err != nil { + conn.Close() + return nil, fmt.Errorf("clear deadline: %w", err) + } + + response = strings.TrimSpace(response) + if !strings.HasPrefix(response, "OK ") { + conn.Close() + return nil, fmt.Errorf("vsock handshake failed: %s", response) + } + + slog.DebugContext(ctx, "vsock handshake successful", "response", response) + + // Return wrapped connection that uses the bufio.Reader + return &bufferedConn{Conn: conn, reader: reader}, nil +} + +// bufferedConn wraps a net.Conn with a bufio.Reader to ensure any buffered +// data from the handshake is properly drained before reading from the connection. +type bufferedConn struct { + net.Conn + reader *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { + return c.reader.Read(p) +} diff --git a/lib/images/disk.go b/lib/images/disk.go index 53378b49..c76660d6 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -108,6 +108,17 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { return stat.Size(), nil } +// sectorSize is the block size for disk images (required by Virtualization.framework) +const sectorSize = 4096 + +// alignToSector rounds size up to the nearest sector boundary +func alignToSector(size int64) int64 { + if size%sectorSize == 0 { + return size + } + return ((size / sectorSize) + 1) * sectorSize +} + // convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 func convertToExt4(rootfsDir, diskPath string) (int64, error) { // Calculate size of rootfs directory @@ -125,6 +136,9 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { diskSizeBytes = minSize } + // Align to sector boundary (required by macOS Virtualization.framework) + diskSizeBytes = alignToSector(diskSizeBytes) + // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) @@ -142,7 +156,7 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { f.Close() // Format as ext4 with rootfs contents using mkfs.ext4 - // -b 4096: 4KB blocks (standard, matches VM page size) + // -b 4096: 4KB blocks (standard, matches VM page size and sector alignment) // -O ^has_journal: Disable journal (not needed for read-only VM mounts) // -d: Copy directory contents into filesystem // -F: Force creation (file not block device) @@ -152,12 +166,21 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) } - // Get actual disk size + // Verify final size is sector-aligned (mkfs.ext4 should preserve our truncated size) stat, err := os.Stat(diskPath) if err != nil { return 0, fmt.Errorf("stat disk: %w", err) } + // Re-align if mkfs.ext4 changed the size (shouldn't happen with -F on a regular file) + if stat.Size()%sectorSize != 0 { + alignedSize := alignToSector(stat.Size()) + if err := os.Truncate(diskPath, alignedSize); err != nil { + return 0, fmt.Errorf("align disk to sector boundary: %w", err) + } + return alignedSize, nil + } + return stat.Size(), nil } @@ -204,6 +227,9 @@ func dirSize(path string) (int64, error) { // CreateEmptyExt4Disk creates a sparse disk file and formats it as ext4. // Used for volumes and instance overlays that need empty writable filesystems. func CreateEmptyExt4Disk(diskPath string, sizeBytes int64) error { + // Align to sector boundary (required by macOS Virtualization.framework) + sizeBytes = alignToSector(sizeBytes) + // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return fmt.Errorf("create disk parent dir: %w", err) @@ -221,8 +247,8 @@ func CreateEmptyExt4Disk(diskPath string, sizeBytes int64) error { return fmt.Errorf("truncate disk file: %w", err) } - // Format as ext4 - cmd := exec.Command("mkfs.ext4", "-F", diskPath) + // Format as ext4 with 4KB blocks (matches sector alignment) + cmd := exec.Command("mkfs.ext4", "-b", "4096", "-F", diskPath) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) diff --git a/lib/images/oci.go b/lib/images/oci.go index 31962d88..1d07758d 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -64,11 +64,13 @@ func newOCIClient(cacheDir string) (*ociClient, error) { return &ociClient{cacheDir: cacheDir}, nil } -// currentPlatform returns the platform for the current host -func currentPlatform() gcr.Platform { +// vmPlatform returns the target platform for VM images. +// Always returns Linux since hypeman VMs are always Linux guests, +// regardless of the host OS (Linux or macOS). +func vmPlatform() gcr.Platform { return gcr.Platform{ Architecture: runtime.GOARCH, - OS: runtime.GOOS, + OS: "linux", } } @@ -77,6 +79,12 @@ func currentPlatform() gcr.Platform { // For multi-arch images, it returns the platform-specific manifest digest // (matching the current host platform) rather than the manifest index digest. func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (string, error) { + return c.inspectManifestWithPlatform(ctx, imageRef, vmPlatform()) +} + +// inspectManifestWithPlatform synchronously inspects a remote image to get its digest +// for a specific platform. +func (c *ociClient) inspectManifestWithPlatform(ctx context.Context, imageRef string, platform gcr.Platform) (string, error) { ref, err := name.ParseReference(imageRef) if err != nil { return "", fmt.Errorf("parse image reference: %w", err) @@ -89,7 +97,7 @@ func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (strin img, err := remote.Image(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain), - remote.WithPlatform(currentPlatform())) + remote.WithPlatform(platform)) if err != nil { return "", fmt.Errorf("fetch manifest: %w", wrapRegistryError(err)) } @@ -109,6 +117,10 @@ type pullResult struct { } func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportDir string) (*pullResult, error) { + return c.pullAndExportWithPlatform(ctx, imageRef, digest, exportDir, vmPlatform()) +} + +func (c *ociClient) pullAndExportWithPlatform(ctx context.Context, imageRef, digest, exportDir string, platform gcr.Platform) (*pullResult, error) { // Use a shared OCI layout for all images to enable automatic layer caching // The cacheDir itself is the OCI layout root with shared blobs/sha256/ directory // The digest is ALWAYS known at this point (from inspectManifest or digest reference) @@ -117,7 +129,7 @@ func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportD // Check if this digest is already cached if !c.existsInLayout(layoutTag) { // Not cached, pull it using digest-based tag - if err := c.pullToOCILayout(ctx, imageRef, layoutTag); err != nil { + if err := c.pullToOCILayoutWithPlatform(ctx, imageRef, layoutTag, platform); err != nil { return nil, fmt.Errorf("pull to oci layout: %w", err) } } @@ -141,6 +153,10 @@ func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportD } func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag string) error { + return c.pullToOCILayoutWithPlatform(ctx, imageRef, layoutTag, vmPlatform()) +} + +func (c *ociClient) pullToOCILayoutWithPlatform(ctx context.Context, imageRef, layoutTag string, platform gcr.Platform) error { ref, err := name.ParseReference(imageRef) if err != nil { return fmt.Errorf("parse image reference: %w", err) @@ -152,7 +168,7 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag str img, err := remote.Image(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain), - remote.WithPlatform(currentPlatform())) + remote.WithPlatform(platform)) if err != nil { // Rate limits fail here immediately (429 is not retried by default) return fmt.Errorf("fetch image manifest: %w", wrapRegistryError(err)) diff --git a/lib/images/oci_public.go b/lib/images/oci_public.go index 5d20835e..66643b97 100644 --- a/lib/images/oci_public.go +++ b/lib/images/oci_public.go @@ -20,11 +20,18 @@ func NewOCIClient(cacheDir string) (*OCIClient, error) { } // InspectManifest inspects a remote image to get its digest (public for system manager) +// Always targets Linux platform since hypeman VMs are Linux guests. func (c *OCIClient) InspectManifest(ctx context.Context, imageRef string) (string, error) { return c.client.inspectManifest(ctx, imageRef) } +// InspectManifestForLinux is an alias for InspectManifest (all images target Linux) +func (c *OCIClient) InspectManifestForLinux(ctx context.Context, imageRef string) (string, error) { + return c.InspectManifest(ctx, imageRef) +} + // PullAndUnpack pulls an OCI image and unpacks it to a directory (public for system manager) +// Always targets Linux platform since hypeman VMs are Linux guests. func (c *OCIClient) PullAndUnpack(ctx context.Context, imageRef, digest, exportDir string) error { _, err := c.client.pullAndExport(ctx, imageRef, digest, exportDir) if err != nil { @@ -33,3 +40,7 @@ func (c *OCIClient) PullAndUnpack(ctx context.Context, imageRef, digest, exportD return nil } +// PullAndUnpackForLinux is an alias for PullAndUnpack (all images target Linux) +func (c *OCIClient) PullAndUnpackForLinux(ctx context.Context, imageRef, digest, exportDir string) error { + return c.PullAndUnpack(ctx, imageRef, digest, exportDir) +} diff --git a/lib/ingress/binaries_amd64.go b/lib/ingress/binaries_amd64.go index 309da631..551e12fb 100644 --- a/lib/ingress/binaries_amd64.go +++ b/lib/ingress/binaries_amd64.go @@ -1,4 +1,4 @@ -//go:build amd64 +//go:build amd64 && linux package ingress diff --git a/lib/ingress/binaries_arm64.go b/lib/ingress/binaries_arm64.go index 8fb413ce..995578a8 100644 --- a/lib/ingress/binaries_arm64.go +++ b/lib/ingress/binaries_arm64.go @@ -1,4 +1,4 @@ -//go:build arm64 +//go:build arm64 && linux package ingress diff --git a/lib/ingress/binaries_darwin.go b/lib/ingress/binaries_darwin.go new file mode 100644 index 00000000..1a2ba408 --- /dev/null +++ b/lib/ingress/binaries_darwin.go @@ -0,0 +1,33 @@ +//go:build darwin + +package ingress + +import ( + "fmt" + "os/exec" + + "github.com/kernel/hypeman/lib/paths" +) + +// CaddyVersion is the version of Caddy to use. +const CaddyVersion = "v2.10.2" + +// ErrCaddyNotEmbedded indicates Caddy is not embedded on macOS. +// Users should install Caddy via Homebrew or download from caddyserver.com. +var ErrCaddyNotEmbedded = fmt.Errorf("caddy binary is not embedded on macOS; install via: brew install caddy") + +// ExtractCaddyBinary on macOS attempts to find Caddy in PATH. +// Unlike Linux, we don't embed the binary on macOS. +func ExtractCaddyBinary(p *paths.Paths) (string, error) { + // Try to find caddy in PATH + path, err := exec.LookPath("caddy") + if err != nil { + return "", ErrCaddyNotEmbedded + } + return path, nil +} + +// GetCaddyBinaryPath returns path to Caddy, looking in PATH on macOS. +func GetCaddyBinaryPath(p *paths.Paths) (string, error) { + return ExtractCaddyBinary(p) +} diff --git a/lib/ingress/binaries.go b/lib/ingress/binaries_linux.go similarity index 99% rename from lib/ingress/binaries.go rename to lib/ingress/binaries_linux.go index 79143506..2b2a6a87 100644 --- a/lib/ingress/binaries.go +++ b/lib/ingress/binaries_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package ingress import ( diff --git a/lib/instances/README.md b/lib/instances/README.md index a2d42172..51a245ef 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -1,12 +1,12 @@ # Instance Manager -Manages VM instance lifecycle using Cloud Hypervisor. +Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEMU on Linux; vz on macOS). ## Design Decisions ### Why State Machine? (state.go) -**What:** Single-hop state transitions matching Cloud Hypervisor's actual states +**What:** Single-hop state transitions matching hypervisor states **Why:** - Validates transitions before execution (prevents invalid operations) @@ -132,6 +132,6 @@ TestStorageOperations - metadata persistence, directory cleanup - `lib/images` - Image manager for OCI image validation - `lib/system` - System manager for kernel/initrd files -- `lib/vmm` - Cloud Hypervisor client for VM operations -- System tools: `mkfs.erofs`, `cpio`, `gzip` +- `lib/hypervisor` - Hypervisor abstraction for VM operations +- System tools: `mkfs.erofs`, `cpio`, `gzip` (Linux); `mkfs.ext4` (macOS) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index a3fd4387..c21b902a 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -121,6 +121,7 @@ func (m *manager) deleteInstance( func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log := logger.FromContext(ctx) + // All hypervisors (cloud-hypervisor, QEMU, vz-shim) now run as external processes. // If we have a PID, kill the process immediately if inst.HypervisorPID != nil { pid := *inst.HypervisorPID diff --git a/lib/instances/hypervisor_darwin.go b/lib/instances/hypervisor_darwin.go new file mode 100644 index 00000000..70b9589a --- /dev/null +++ b/lib/instances/hypervisor_darwin.go @@ -0,0 +1,12 @@ +//go:build darwin + +package instances + +import ( + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz" +) + +func init() { + additionalStarters[hypervisor.TypeVZ] = vz.NewStarter() +} diff --git a/lib/instances/hypervisor_linux.go b/lib/instances/hypervisor_linux.go new file mode 100644 index 00000000..15124340 --- /dev/null +++ b/lib/instances/hypervisor_linux.go @@ -0,0 +1,7 @@ +//go:build linux + +package instances + +func init() { + // No additional starters on Linux - CH and QEMU are in the base set +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 8411d193..b72110e4 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -44,6 +44,8 @@ type Manager interface { // SetResourceValidator sets the validator for aggregate resource limit checking. // Called after initialization to avoid circular dependencies. SetResourceValidator(v ResourceValidator) + // GetVsockDialer returns a VsockDialer for the specified instance. + GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) } // ResourceLimits contains configurable resource limits for instances @@ -79,6 +81,9 @@ type manager struct { defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request } +// additionalStarters is populated by platform-specific init functions. +var additionalStarters = make(map[hypervisor.Type]hypervisor.VMStarter) + // NewManager creates a new instances manager. // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. @@ -88,20 +93,28 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste defaultHypervisor = hypervisor.TypeCloudHypervisor } + // Initialize base VM starters (CH and QEMU available on all platforms) + vmStarters := map[hypervisor.Type]hypervisor.VMStarter{ + hypervisor.TypeCloudHypervisor: cloudhypervisor.NewStarter(), + hypervisor.TypeQEMU: qemu.NewStarter(), + } + + // Add platform-specific starters (e.g., vz on macOS) + for hvType, starter := range additionalStarters { + vmStarters[hvType] = starter + } + m := &manager{ - paths: p, - imageManager: imageManager, - systemManager: systemManager, - networkManager: networkManager, - deviceManager: deviceManager, - volumeManager: volumeManager, - limits: limits, - instanceLocks: sync.Map{}, - hostTopology: detectHostTopology(), // Detect and cache host topology - vmStarters: map[hypervisor.Type]hypervisor.VMStarter{ - hypervisor.TypeCloudHypervisor: cloudhypervisor.NewStarter(), - hypervisor.TypeQEMU: qemu.NewStarter(), - }, + paths: p, + imageManager: imageManager, + systemManager: systemManager, + networkManager: networkManager, + deviceManager: deviceManager, + volumeManager: volumeManager, + limits: limits, + instanceLocks: sync.Map{}, + hostTopology: detectHostTopology(), // Detect and cache host topology + vmStarters: vmStarters, defaultHypervisor: defaultHypervisor, } @@ -125,14 +138,7 @@ func (m *manager) SetResourceValidator(v ResourceValidator) { // getHypervisor creates a hypervisor client for the given socket and type. // Used for connecting to already-running VMs (e.g., for state queries). func (m *manager) getHypervisor(socketPath string, hvType hypervisor.Type) (hypervisor.Hypervisor, error) { - switch hvType { - case hypervisor.TypeCloudHypervisor: - return cloudhypervisor.New(socketPath) - case hypervisor.TypeQEMU: - return qemu.New(socketPath) - default: - return nil, fmt.Errorf("unsupported hypervisor type: %s", hvType) - } + return hypervisor.NewClient(hvType, socketPath) } // getVMStarter returns the VM starter for the given hypervisor type. diff --git a/lib/instances/query.go b/lib/instances/query.go index 1bc26fc2..6c7a8709 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -21,7 +21,8 @@ type stateResult struct { func (m *manager) deriveState(ctx context.Context, stored *StoredMetadata) stateResult { log := logger.FromContext(ctx) - // 1. Check if socket exists + // All hypervisors (cloud-hypervisor, QEMU, vz-shim) are socket-based. + // Check if socket exists if _, err := os.Stat(stored.SocketPath); err != nil { // No socket - check for snapshot to distinguish Stopped vs Standby if m.hasSnapshot(stored.DataDir) { @@ -30,7 +31,7 @@ func (m *manager) deriveState(ctx context.Context, stored *StoredMetadata) state return stateResult{State: StateStopped} } - // 2. Socket exists - query hypervisor for actual state + // 3. Socket exists - query hypervisor for actual state hv, err := m.getHypervisor(stored.SocketPath, stored.HypervisorType) if err != nil { // Failed to create client - this is unexpected if socket exists @@ -43,19 +44,24 @@ func (m *manager) deriveState(ctx context.Context, stored *StoredMetadata) state return stateResult{State: StateUnknown, Error: &errMsg} } + return m.queryHypervisorState(ctx, stored, hv) +} + +// queryHypervisorState queries a hypervisor instance for VM state. +func (m *manager) queryHypervisorState(ctx context.Context, stored *StoredMetadata, hv hypervisor.Hypervisor) stateResult { + log := logger.FromContext(ctx) + info, err := hv.GetVMInfo(ctx) if err != nil { - // Socket exists but hypervisor is unreachable - this is unexpected errMsg := fmt.Sprintf("failed to query hypervisor: %v", err) log.WarnContext(ctx, "failed to query hypervisor state", "instance_id", stored.Id, - "socket", stored.SocketPath, "error", err, ) return stateResult{State: StateUnknown, Error: &errMsg} } - // 3. Map hypervisor state to our state + // Map hypervisor state to our state switch info.State { case hypervisor.StateCreated: return stateResult{State: StateCreated} diff --git a/lib/instances/vsock_darwin.go b/lib/instances/vsock_darwin.go new file mode 100644 index 00000000..a9c4ff91 --- /dev/null +++ b/lib/instances/vsock_darwin.go @@ -0,0 +1,20 @@ +//go:build darwin + +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// GetVsockDialer returns a VsockDialer for the specified instance. +// On macOS, all hypervisors (including vz via shim) use socket-based vsock. +func (m *manager) GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) { + inst, err := m.GetInstance(ctx, instanceID) + if err != nil { + return nil, err + } + + return hypervisor.NewVsockDialer(hypervisor.Type(inst.HypervisorType), inst.VsockSocket, inst.VsockCID) +} diff --git a/lib/instances/vsock_linux.go b/lib/instances/vsock_linux.go new file mode 100644 index 00000000..c5612be8 --- /dev/null +++ b/lib/instances/vsock_linux.go @@ -0,0 +1,19 @@ +//go:build linux + +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// GetVsockDialer returns a VsockDialer for the specified instance. +func (m *manager) GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) { + inst, err := m.GetInstance(ctx, instanceID) + if err != nil { + return nil, err + } + + return hypervisor.NewVsockDialer(hypervisor.Type(inst.HypervisorType), inst.VsockSocket, inst.VsockCID) +} diff --git a/lib/network/README.md b/lib/network/README.md index 1e771532..c54e66a8 100644 --- a/lib/network/README.md +++ b/lib/network/README.md @@ -1,6 +1,21 @@ # Network Manager -Manages the default virtual network for instances using a Linux bridge and TAP devices. +Manages the default virtual network for instances. + +## Platform Support + +| Platform | Network Model | Implementation | +|----------|---------------|----------------| +| Linux | Bridge + TAP | Linux bridge with TAP devices per VM, iptables NAT | +| macOS | NAT | Virtualization.framework built-in NAT (192.168.64.0/24) | + +On macOS, the network manager skips bridge/TAP creation since vz provides NAT networking automatically. + +--- + +## Linux Networking + +On Linux, hypeman manages a virtual network using a Linux bridge and TAP devices. ## How Linux VM Networking Works diff --git a/lib/network/bridge_darwin.go b/lib/network/bridge_darwin.go new file mode 100644 index 00000000..43b195db --- /dev/null +++ b/lib/network/bridge_darwin.go @@ -0,0 +1,98 @@ +//go:build darwin + +package network + +import ( + "context" + "fmt" + + "github.com/kernel/hypeman/lib/logger" +) + +// checkSubnetConflicts is a no-op on macOS as we use NAT networking. +func (m *manager) checkSubnetConflicts(ctx context.Context, subnet string) error { + // NAT networking doesn't conflict with host routes + return nil +} + +// createBridge is a no-op on macOS as we use NAT networking. +// Virtualization.framework provides built-in NAT with NATNetworkDeviceAttachment. +func (m *manager) createBridge(ctx context.Context, name, gateway, subnet string) error { + log := logger.FromContext(ctx) + log.InfoContext(ctx, "macOS: skipping bridge creation (using NAT networking)") + return nil +} + +// setupIPTablesRules is a no-op on macOS as we use NAT networking. +func (m *manager) setupIPTablesRules(ctx context.Context, subnet, bridgeName string) error { + return nil +} + +// setupBridgeHTB is a no-op on macOS as we use NAT networking. +// macOS doesn't use traffic control qdiscs. +func (m *manager) setupBridgeHTB(ctx context.Context, bridgeName string, capacityBps int64) error { + return nil +} + +// createTAPDevice is a no-op on macOS as we use NAT networking. +// Virtualization.framework creates virtual network interfaces internally. +func (m *manager) createTAPDevice(tapName, bridgeName string, isolated bool, downloadBps, uploadBps, uploadCeilBps int64) error { + // On macOS with vz, network devices are created by the VMM itself + return nil +} + +// deleteTAPDevice is a no-op on macOS as we use NAT networking. +func (m *manager) deleteTAPDevice(tapName string) error { + return nil +} + +// queryNetworkState returns a stub network state for macOS. +// On macOS, we use NAT which doesn't have a physical bridge. +func (m *manager) queryNetworkState(bridgeName string) (*Network, error) { + // Return a virtual network representing macOS NAT + // The actual IP will be assigned by Virtualization.framework's DHCP + return &Network{ + Bridge: "nat", + Gateway: "192.168.64.1", // Default macOS vz NAT gateway + Subnet: "192.168.64.0/24", + }, nil +} + +// CleanupOrphanedTAPs is a no-op on macOS as we don't create TAP devices. +func (m *manager) CleanupOrphanedTAPs(ctx context.Context, runningInstanceIDs []string) int { + return 0 +} + +// CleanupOrphanedClasses is a no-op on macOS as we don't use traffic control. +func (m *manager) CleanupOrphanedClasses(ctx context.Context) int { + return 0 +} + +// Note: On macOS with vz, network configuration is different: +// - VMs get IP addresses via DHCP from macOS's NAT (192.168.64.x) +// - No TAP devices are created - vz handles network internally +// - No iptables/pf rules needed - NAT is built-in +// - Rate limiting is not supported (no tc equivalent) +// +// The CreateAllocation and ReleaseAllocation methods in allocate.go +// will need platform-specific handling for the TAP-related calls. + +// macOSNetworkConfig holds macOS-specific network configuration +type macOSNetworkConfig struct { + UseNAT bool // Always true for macOS +} + +// GetMacOSNetworkConfig returns the macOS network configuration +func GetMacOSNetworkConfig() *macOSNetworkConfig { + return &macOSNetworkConfig{ + UseNAT: true, + } +} + +// IsMacOS returns true on macOS builds +func IsMacOS() bool { + return true +} + +// ErrRateLimitNotSupported indicates rate limiting is not supported on macOS +var ErrRateLimitNotSupported = fmt.Errorf("network rate limiting is not supported on macOS") diff --git a/lib/network/bridge.go b/lib/network/bridge_linux.go similarity index 98% rename from lib/network/bridge.go rename to lib/network/bridge_linux.go index a979c111..952d7dbb 100644 --- a/lib/network/bridge.go +++ b/lib/network/bridge_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package network import ( @@ -15,22 +17,6 @@ import ( "golang.org/x/sys/unix" ) -// DeriveGateway returns the first usable IP in a subnet (used as gateway). -// e.g., 10.100.0.0/16 -> 10.100.0.1 -func DeriveGateway(cidr string) (string, error) { - _, ipNet, err := net.ParseCIDR(cidr) - if err != nil { - return "", fmt.Errorf("parse CIDR: %w", err) - } - - // Gateway is network address + 1 - gateway := make(net.IP, len(ipNet.IP)) - copy(gateway, ipNet.IP) - gateway[len(gateway)-1]++ // Increment last octet - - return gateway.String(), nil -} - // checkSubnetConflicts checks if the configured subnet conflicts with existing routes. // Returns an error if a conflict is detected, with guidance on how to resolve it. func (m *manager) checkSubnetConflicts(ctx context.Context, subnet string) error { diff --git a/lib/network/ip.go b/lib/network/ip.go new file mode 100644 index 00000000..555ad579 --- /dev/null +++ b/lib/network/ip.go @@ -0,0 +1,22 @@ +package network + +import ( + "fmt" + "net" +) + +// DeriveGateway returns the first usable IP in a subnet (used as gateway). +// e.g., 10.100.0.0/16 -> 10.100.0.1 +func DeriveGateway(cidr string) (string, error) { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return "", fmt.Errorf("parse CIDR: %w", err) + } + + // Gateway is network address + 1 + gateway := make(net.IP, len(ipNet.IP)) + copy(gateway, ipNet.IP) + gateway[len(gateway)-1]++ // Increment last octet + + return gateway.String(), nil +} diff --git a/lib/resources/cpu.go b/lib/resources/cpu.go index 883cbff7..edac6e50 100644 --- a/lib/resources/cpu.go +++ b/lib/resources/cpu.go @@ -1,12 +1,7 @@ package resources import ( - "bufio" "context" - "fmt" - "os" - "strconv" - "strings" ) // CPUResource implements Resource for CPU discovery and tracking. @@ -15,7 +10,7 @@ type CPUResource struct { instanceLister InstanceLister } -// NewCPUResource discovers host CPU capacity from /proc/cpuinfo. +// NewCPUResource discovers host CPU capacity. func NewCPUResource() (*CPUResource, error) { capacity, err := detectCPUCapacity() if err != nil { @@ -59,78 +54,6 @@ func (c *CPUResource) Allocated(ctx context.Context) (int64, error) { return total, nil } -// detectCPUCapacity reads /proc/cpuinfo to determine total vCPU count. -// Returns threads × cores × sockets. -func detectCPUCapacity() (int64, error) { - file, err := os.Open("/proc/cpuinfo") - if err != nil { - return 0, fmt.Errorf("open /proc/cpuinfo: %w", err) - } - defer file.Close() - - var ( - siblings int - physicalIDs = make(map[int]bool) - hasSiblings bool - hasPhysicalID bool - ) - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - - parts := strings.SplitN(line, ":", 2) - if len(parts) != 2 { - continue - } - - key := strings.TrimSpace(parts[0]) - value := strings.TrimSpace(parts[1]) - - switch key { - case "siblings": - if !hasSiblings { - siblings, _ = strconv.Atoi(value) - hasSiblings = true - } - case "physical id": - physicalID, _ := strconv.Atoi(value) - physicalIDs[physicalID] = true - hasPhysicalID = true - } - } - - if err := scanner.Err(); err != nil { - return 0, err - } - - // Calculate total vCPUs - if hasSiblings && hasPhysicalID { - // siblings = threads per socket, physicalIDs = number of sockets - sockets := len(physicalIDs) - if sockets < 1 { - sockets = 1 - } - return int64(siblings * sockets), nil - } - - // Fallback: count processor entries - file.Seek(0, 0) - scanner = bufio.NewScanner(file) - count := 0 - for scanner.Scan() { - if strings.HasPrefix(scanner.Text(), "processor") { - count++ - } - } - if count > 0 { - return int64(count), nil - } - - // Ultimate fallback - return 1, nil -} - // isActiveState returns true if the instance state indicates it's consuming resources. func isActiveState(state string) bool { switch state { diff --git a/lib/resources/cpu_darwin.go b/lib/resources/cpu_darwin.go new file mode 100644 index 00000000..8931af85 --- /dev/null +++ b/lib/resources/cpu_darwin.go @@ -0,0 +1,13 @@ +//go:build darwin + +package resources + +import ( + "runtime" +) + +// detectCPUCapacity returns the number of logical CPUs on macOS. +// Uses runtime.NumCPU() which calls sysctl on macOS. +func detectCPUCapacity() (int64, error) { + return int64(runtime.NumCPU()), nil +} diff --git a/lib/resources/cpu_linux.go b/lib/resources/cpu_linux.go new file mode 100644 index 00000000..606cd718 --- /dev/null +++ b/lib/resources/cpu_linux.go @@ -0,0 +1,83 @@ +//go:build linux + +package resources + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// detectCPUCapacity reads /proc/cpuinfo to determine total vCPU count. +// Returns threads × cores × sockets. +func detectCPUCapacity() (int64, error) { + file, err := os.Open("/proc/cpuinfo") + if err != nil { + return 0, fmt.Errorf("open /proc/cpuinfo: %w", err) + } + defer file.Close() + + var ( + siblings int + physicalIDs = make(map[int]bool) + hasSiblings bool + hasPhysicalID bool + ) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + switch key { + case "siblings": + if !hasSiblings { + siblings, _ = strconv.Atoi(value) + hasSiblings = true + } + case "physical id": + physicalID, _ := strconv.Atoi(value) + physicalIDs[physicalID] = true + hasPhysicalID = true + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + // Calculate total vCPUs + if hasSiblings && hasPhysicalID { + // siblings = threads per socket, physicalIDs = number of sockets + sockets := len(physicalIDs) + if sockets < 1 { + sockets = 1 + } + return int64(siblings * sockets), nil + } + + // Fallback: count processor entries + file.Seek(0, 0) + scanner = bufio.NewScanner(file) + count := 0 + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "processor") { + count++ + } + } + if count > 0 { + return int64(count), nil + } + + // Ultimate fallback + return 1, nil +} diff --git a/lib/resources/disk_darwin.go b/lib/resources/disk_darwin.go new file mode 100644 index 00000000..3eb38537 --- /dev/null +++ b/lib/resources/disk_darwin.go @@ -0,0 +1,176 @@ +//go:build darwin + +package resources + +import ( + "context" + "os" + "strings" + + "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/paths" + "golang.org/x/sys/unix" +) + +// DiskResource implements Resource for disk space discovery and tracking. +type DiskResource struct { + capacity int64 // bytes + dataDir string + instanceLister InstanceLister + imageLister ImageLister + volumeLister VolumeLister +} + +// NewDiskResource discovers disk capacity on macOS. +func NewDiskResource(cfg *config.Config, p *paths.Paths, instLister InstanceLister, imgLister ImageLister, volLister VolumeLister) (*DiskResource, error) { + var capacity int64 + + if cfg.DiskLimit != "" { + // Parse configured limit + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(cfg.DiskLimit)); err != nil { + return nil, err + } + capacity = int64(ds.Bytes()) + } else { + // Auto-detect from filesystem using statfs + var stat unix.Statfs_t + dataDir := cfg.DataDir + if err := unix.Statfs(dataDir, &stat); err != nil { + // Fallback: try to stat the root if data dir doesn't exist yet + if os.IsNotExist(err) { + if err := unix.Statfs("/", &stat); err != nil { + return nil, err + } + } else { + return nil, err + } + } + capacity = int64(stat.Blocks) * int64(stat.Bsize) + } + + return &DiskResource{ + capacity: capacity, + dataDir: cfg.DataDir, + instanceLister: instLister, + imageLister: imgLister, + volumeLister: volLister, + }, nil +} + +// Type returns the resource type. +func (d *DiskResource) Type() ResourceType { + return ResourceDisk +} + +// Capacity returns the disk capacity in bytes. +func (d *DiskResource) Capacity() int64 { + return d.capacity +} + +// Allocated returns currently allocated disk space. +func (d *DiskResource) Allocated(ctx context.Context) (int64, error) { + breakdown, err := d.GetBreakdown(ctx) + if err != nil { + return 0, err + } + return breakdown.Images + breakdown.OCICache + breakdown.Volumes + breakdown.Overlays, nil +} + +// GetBreakdown returns disk usage broken down by category. +func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) { + var breakdown DiskBreakdown + + // Get image sizes + if d.imageLister != nil { + imageBytes, err := d.imageLister.TotalImageBytes(ctx) + if err == nil { + breakdown.Images = imageBytes + } + ociCacheBytes, err := d.imageLister.TotalOCICacheBytes(ctx) + if err == nil { + breakdown.OCICache = ociCacheBytes + } + } + + // Get volume sizes + if d.volumeLister != nil { + volumeBytes, err := d.volumeLister.TotalVolumeBytes(ctx) + if err == nil { + breakdown.Volumes = volumeBytes + } + } + + // Get overlay sizes from instances + if d.instanceLister != nil { + instances, err := d.instanceLister.ListInstanceAllocations(ctx) + if err == nil { + for _, inst := range instances { + if isActiveState(inst.State) { + breakdown.Overlays += inst.OverlayBytes + inst.VolumeOverlayBytes + } + } + } + } + + return &breakdown, nil +} + +// parseDiskIOLimit parses a disk I/O limit string like "500MB/s", "1GB/s". +// Returns bytes per second. +func parseDiskIOLimit(limit string) (int64, error) { + limit = strings.TrimSpace(limit) + limit = strings.ToLower(limit) + + // Remove "/s" or "ps" suffix if present + limit = strings.TrimSuffix(limit, "/s") + limit = strings.TrimSuffix(limit, "ps") + + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(limit)); err != nil { + return 0, err + } + + return int64(ds.Bytes()), nil +} + +// DiskIOResource implements Resource for disk I/O bandwidth tracking. +type DiskIOResource struct { + capacity int64 // bytes per second + instanceLister InstanceLister +} + +// NewDiskIOResource creates a disk I/O resource with the given capacity. +func NewDiskIOResource(capacity int64, instLister InstanceLister) *DiskIOResource { + return &DiskIOResource{capacity: capacity, instanceLister: instLister} +} + +// Type returns the resource type. +func (d *DiskIOResource) Type() ResourceType { + return ResourceDiskIO +} + +// Capacity returns the total disk I/O capacity in bytes per second. +func (d *DiskIOResource) Capacity() int64 { + return d.capacity +} + +// Allocated returns total disk I/O allocated across all active instances. +// On macOS, disk I/O rate limiting is not enforced. +func (d *DiskIOResource) Allocated(ctx context.Context) (int64, error) { + if d.instanceLister == nil { + return 0, nil + } + instances, err := d.instanceLister.ListInstanceAllocations(ctx) + if err != nil { + return 0, err + } + var total int64 + for _, inst := range instances { + if isActiveState(inst.State) { + total += inst.DiskIOBps + } + } + return total, nil +} diff --git a/lib/resources/disk.go b/lib/resources/disk_linux.go similarity index 99% rename from lib/resources/disk.go rename to lib/resources/disk_linux.go index 2b6bf76d..ec65b4d9 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package resources import ( diff --git a/lib/resources/memory.go b/lib/resources/memory.go index 52cebd78..0e334cff 100644 --- a/lib/resources/memory.go +++ b/lib/resources/memory.go @@ -1,12 +1,7 @@ package resources import ( - "bufio" "context" - "fmt" - "os" - "strconv" - "strings" ) // MemoryResource implements Resource for memory discovery and tracking. @@ -15,7 +10,7 @@ type MemoryResource struct { instanceLister InstanceLister } -// NewMemoryResource discovers host memory capacity from /proc/meminfo. +// NewMemoryResource discovers host memory capacity. func NewMemoryResource() (*MemoryResource, error) { capacity, err := detectMemoryCapacity() if err != nil { @@ -58,34 +53,3 @@ func (m *MemoryResource) Allocated(ctx context.Context) (int64, error) { } return total, nil } - -// detectMemoryCapacity reads /proc/meminfo to determine total memory. -func detectMemoryCapacity() (int64, error) { - file, err := os.Open("/proc/meminfo") - if err != nil { - return 0, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "MemTotal:") { - // Format: "MemTotal: 16384000 kB" - fields := strings.Fields(line) - if len(fields) >= 2 { - kb, err := strconv.ParseInt(fields[1], 10, 64) - if err != nil { - return 0, fmt.Errorf("parse MemTotal: %w", err) - } - return kb * 1024, nil // Convert KB to bytes - } - } - } - - if err := scanner.Err(); err != nil { - return 0, err - } - - return 0, fmt.Errorf("MemTotal not found in /proc/meminfo") -} diff --git a/lib/resources/memory_darwin.go b/lib/resources/memory_darwin.go new file mode 100644 index 00000000..01989aa9 --- /dev/null +++ b/lib/resources/memory_darwin.go @@ -0,0 +1,17 @@ +//go:build darwin + +package resources + +import ( + "golang.org/x/sys/unix" +) + +// detectMemoryCapacity returns total physical memory on macOS using sysctl. +func detectMemoryCapacity() (int64, error) { + // Use sysctl to get hw.memsize + memsize, err := unix.SysctlUint64("hw.memsize") + if err != nil { + return 0, err + } + return int64(memsize), nil +} diff --git a/lib/resources/memory_linux.go b/lib/resources/memory_linux.go new file mode 100644 index 00000000..1ed59d26 --- /dev/null +++ b/lib/resources/memory_linux.go @@ -0,0 +1,42 @@ +//go:build linux + +package resources + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// detectMemoryCapacity reads /proc/meminfo to determine total memory. +func detectMemoryCapacity() (int64, error) { + file, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "MemTotal:") { + // Format: "MemTotal: 16384000 kB" + fields := strings.Fields(line) + if len(fields) >= 2 { + kb, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse MemTotal: %w", err) + } + return kb * 1024, nil // Convert KB to bytes + } + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + return 0, fmt.Errorf("MemTotal not found in /proc/meminfo") +} diff --git a/lib/resources/network_darwin.go b/lib/resources/network_darwin.go new file mode 100644 index 00000000..4e662975 --- /dev/null +++ b/lib/resources/network_darwin.go @@ -0,0 +1,49 @@ +//go:build darwin + +package resources + +import ( + "context" + + "github.com/kernel/hypeman/cmd/api/config" +) + +// NetworkResource implements Resource for network bandwidth discovery and tracking. +// On macOS, network rate limiting is not supported. +type NetworkResource struct { + capacity int64 // bytes per second (set to high value on macOS) + instanceLister InstanceLister +} + +// NewNetworkResource creates a network resource on macOS. +// Network capacity detection and rate limiting are not supported on macOS. +func NewNetworkResource(ctx context.Context, cfg *config.Config, instLister InstanceLister) (*NetworkResource, error) { + // Default to 10 Gbps as a reasonable high limit on macOS + // Network rate limiting is not enforced on macOS + return &NetworkResource{ + capacity: 10 * 1024 * 1024 * 1024 / 8, // 10 Gbps in bytes/sec + instanceLister: instLister, + }, nil +} + +// Type returns the resource type. +func (n *NetworkResource) Type() ResourceType { + return ResourceNetwork +} + +// Capacity returns the network capacity in bytes per second. +func (n *NetworkResource) Capacity() int64 { + return n.capacity +} + +// Allocated returns currently allocated network bandwidth. +// On macOS, this is always 0 as rate limiting is not supported. +func (n *NetworkResource) Allocated(ctx context.Context) (int64, error) { + return 0, nil +} + +// AvailableFor returns available network bandwidth. +// On macOS, this always returns the full capacity. +func (n *NetworkResource) AvailableFor(ctx context.Context, requested int64) (int64, error) { + return n.capacity, nil +} diff --git a/lib/resources/network.go b/lib/resources/network_linux.go similarity index 73% rename from lib/resources/network.go rename to lib/resources/network_linux.go index 41ba3d8e..cf02aa30 100644 --- a/lib/resources/network.go +++ b/lib/resources/network_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package resources import ( @@ -139,50 +141,3 @@ func getInterfaceSpeed(iface string) (int64, error) { return speed, nil } - -// ParseBandwidth parses a bandwidth string like "10Gbps", "1GB/s", "125MB/s". -// Handles both bit-based (bps) and byte-based (/s) formats. -// Returns bytes per second. -func ParseBandwidth(limit string) (int64, error) { - limit = strings.TrimSpace(limit) - limit = strings.ToLower(limit) - - // Handle bps variants (bits per second) - if strings.HasSuffix(limit, "bps") { - // Remove "bps" suffix - numPart := strings.TrimSuffix(limit, "bps") - numPart = strings.TrimSpace(numPart) - - // Check for multiplier prefix - var multiplier int64 = 1 - if strings.HasSuffix(numPart, "g") { - multiplier = 1000 * 1000 * 1000 - numPart = strings.TrimSuffix(numPart, "g") - } else if strings.HasSuffix(numPart, "m") { - multiplier = 1000 * 1000 - numPart = strings.TrimSuffix(numPart, "m") - } else if strings.HasSuffix(numPart, "k") { - multiplier = 1000 - numPart = strings.TrimSuffix(numPart, "k") - } - - bits, err := strconv.ParseInt(strings.TrimSpace(numPart), 10, 64) - if err != nil { - return 0, fmt.Errorf("invalid number: %s", numPart) - } - - // Convert bits to bytes - return (bits * multiplier) / 8, nil - } - - // Handle byte-based variants (e.g., "125MB/s", "1GB") - limit = strings.TrimSuffix(limit, "/s") - limit = strings.TrimSuffix(limit, "ps") - - var ds datasize.ByteSize - if err := ds.UnmarshalText([]byte(limit)); err != nil { - return 0, fmt.Errorf("parse as bytes: %w", err) - } - - return int64(ds.Bytes()), nil -} diff --git a/lib/resources/util.go b/lib/resources/util.go new file mode 100644 index 00000000..619037c8 --- /dev/null +++ b/lib/resources/util.go @@ -0,0 +1,56 @@ +package resources + +import ( + "fmt" + "strconv" + "strings" + + "github.com/c2h5oh/datasize" +) + +// ParseBandwidth parses a bandwidth string like "10Gbps", "1GB/s", "125MB/s". +// Handles both bit-based (bps) and byte-based (/s) formats. +// Returns bytes per second. +func ParseBandwidth(limit string) (int64, error) { + limit = strings.TrimSpace(limit) + limit = strings.ToLower(limit) + + // Handle bps variants (bits per second) + if strings.HasSuffix(limit, "bps") { + // Remove "bps" suffix + numPart := strings.TrimSuffix(limit, "bps") + numPart = strings.TrimSpace(numPart) + + // Check for multiplier prefix + var multiplier int64 = 1 + if strings.HasSuffix(numPart, "g") { + multiplier = 1000 * 1000 * 1000 + numPart = strings.TrimSuffix(numPart, "g") + } else if strings.HasSuffix(numPart, "m") { + multiplier = 1000 * 1000 + numPart = strings.TrimSuffix(numPart, "m") + } else if strings.HasSuffix(numPart, "k") { + multiplier = 1000 + numPart = strings.TrimSuffix(numPart, "k") + } + + bits, err := strconv.ParseInt(strings.TrimSpace(numPart), 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid number: %s", numPart) + } + + // Convert bits to bytes + return (bits * multiplier) / 8, nil + } + + // Handle byte-based variants (e.g., "125MB/s", "1GB") + limit = strings.TrimSuffix(limit, "/s") + limit = strings.TrimSuffix(limit, "ps") + + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(limit)); err != nil { + return 0, fmt.Errorf("parse as bytes: %w", err) + } + + return int64(ds.Bytes()), nil +} diff --git a/lib/system/guest_agent_binary.go b/lib/system/guest_agent_binary.go index 57d69722..2923477a 100644 --- a/lib/system/guest_agent_binary.go +++ b/lib/system/guest_agent_binary.go @@ -1,3 +1,5 @@ +//go:build linux + package system import _ "embed" diff --git a/lib/system/guest_agent_binary_darwin.go b/lib/system/guest_agent_binary_darwin.go new file mode 100644 index 00000000..76037e86 --- /dev/null +++ b/lib/system/guest_agent_binary_darwin.go @@ -0,0 +1,12 @@ +//go:build darwin + +package system + +import _ "embed" + +// GuestAgentBinary contains the cross-compiled Linux guest agent for guest VMs. +// This is built by the Makefile with GOOS=linux before the main binary is compiled. +// The guest agent handles exec, file operations, and other guest-side functionality. +// +//go:embed guest_agent/guest-agent +var GuestAgentBinary []byte diff --git a/lib/system/init/logger.go b/lib/system/init/logger.go index 6d0a5217..588c8bfb 100644 --- a/lib/system/init/logger.go +++ b/lib/system/init/logger.go @@ -17,12 +17,17 @@ func NewLogger() *Logger { l := &Logger{} // Open serial console for output - // ttyS0 for x86_64, ttyAMA0 for ARM64 (PL011 UART) - if f, err := os.OpenFile("/dev/ttyAMA0", os.O_WRONLY, 0); err == nil { - l.console = f - } else if f, err := os.OpenFile("/dev/ttyS0", os.O_WRONLY, 0); err == nil { - l.console = f - } else { + // hvc0 for Virtualization.framework (vz) on macOS + // ttyAMA0 for ARM64 PL011 UART (cloud-hypervisor) + // ttyS0 for x86_64 (QEMU, cloud-hypervisor) + consoles := []string{"/dev/hvc0", "/dev/ttyAMA0", "/dev/ttyS0"} + for _, console := range consoles { + if f, err := os.OpenFile(console, os.O_WRONLY, 0); err == nil { + l.console = f + break + } + } + if l.console == nil { // Fallback to stdout l.console = os.Stdout } diff --git a/lib/system/init/mount.go b/lib/system/init/mount.go index 50ebc079..07894d01 100644 --- a/lib/system/init/mount.go +++ b/lib/system/init/mount.go @@ -49,16 +49,20 @@ func mountEssentials(log *Logger) error { log.Info("mount", "mounted devpts/shm") // Set up serial console now that /dev is mounted - // ttyS0 for x86_64, ttyAMA0 for ARM64 (PL011 UART) - if _, err := os.Stat("/dev/ttyAMA0"); err == nil { - log.SetConsole("/dev/ttyAMA0") - redirectToConsole("/dev/ttyAMA0") - } else if _, err := os.Stat("/dev/ttyS0"); err == nil { - log.SetConsole("/dev/ttyS0") - redirectToConsole("/dev/ttyS0") + // hvc0 for Virtualization.framework (vz) on macOS + // ttyAMA0 for ARM64 PL011 UART (cloud-hypervisor) + // ttyS0 for x86_64 (QEMU, cloud-hypervisor) + consoles := []string{"/dev/hvc0", "/dev/ttyAMA0", "/dev/ttyS0"} + for _, console := range consoles { + if _, err := os.Stat(console); err == nil { + log.SetConsole(console) + redirectToConsole(console) + log.Info("mount", "using console "+console) + break + } } - log.Info("mount", "redirected to serial console") + log.Info("mount", "console setup complete") return nil } diff --git a/lib/system/init_binary.go b/lib/system/init_binary.go index ad378a67..85038ef3 100644 --- a/lib/system/init_binary.go +++ b/lib/system/init_binary.go @@ -1,3 +1,5 @@ +//go:build linux + package system import _ "embed" diff --git a/lib/system/init_binary_darwin.go b/lib/system/init_binary_darwin.go new file mode 100644 index 00000000..d0806ced --- /dev/null +++ b/lib/system/init_binary_darwin.go @@ -0,0 +1,12 @@ +//go:build darwin + +package system + +import _ "embed" + +// InitBinary contains the cross-compiled Linux init binary for guest VMs. +// This is built by the Makefile with GOOS=linux before the main binary is compiled. +// The init binary is a statically-linked Go program that runs as PID 1 in the guest VM. +// +//go:embed init/init +var InitBinary []byte diff --git a/lib/system/initrd.go b/lib/system/initrd.go index e3891bdf..168d90f2 100644 --- a/lib/system/initrd.go +++ b/lib/system/initrd.go @@ -35,14 +35,14 @@ func (m *manager) buildInitrd(ctx context.Context, arch string) (string, error) return "", fmt.Errorf("create oci client: %w", err) } - // Inspect Alpine base to get digest - digest, err := ociClient.InspectManifest(ctx, alpineBaseImage) + // Inspect Alpine base to get digest (always use Linux platform since this is for guest VMs) + digest, err := ociClient.InspectManifestForLinux(ctx, alpineBaseImage) if err != nil { return "", fmt.Errorf("inspect alpine manifest: %w", err) } - // Pull and unpack Alpine base - if err := ociClient.PullAndUnpack(ctx, alpineBaseImage, digest, rootfsDir); err != nil { + // Pull and unpack Alpine base (always use Linux platform since this is for guest VMs) + if err := ociClient.PullAndUnpackForLinux(ctx, alpineBaseImage, digest, rootfsDir); err != nil { return "", fmt.Errorf("pull alpine base: %w", err) } diff --git a/lib/vmm/binaries_darwin.go b/lib/vmm/binaries_darwin.go new file mode 100644 index 00000000..370c027c --- /dev/null +++ b/lib/vmm/binaries_darwin.go @@ -0,0 +1,34 @@ +//go:build darwin + +package vmm + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/paths" +) + +// CHVersion represents Cloud Hypervisor version +type CHVersion string + +const ( + V48_0 CHVersion = "v48.0" + V49_0 CHVersion = "v49.0" +) + +// SupportedVersions lists supported Cloud Hypervisor versions. +// On macOS, Cloud Hypervisor is not supported (use vz instead). +var SupportedVersions = []CHVersion{} + +// ErrNotSupportedOnMacOS indicates Cloud Hypervisor is not available on macOS +var ErrNotSupportedOnMacOS = fmt.Errorf("cloud-hypervisor is not supported on macOS; use vz hypervisor instead") + +// ExtractBinary is not supported on macOS +func ExtractBinary(p *paths.Paths, version CHVersion) (string, error) { + return "", ErrNotSupportedOnMacOS +} + +// GetBinaryPath is not supported on macOS +func GetBinaryPath(p *paths.Paths, version CHVersion) (string, error) { + return "", ErrNotSupportedOnMacOS +} diff --git a/lib/vmm/binaries.go b/lib/vmm/binaries_linux.go similarity index 98% rename from lib/vmm/binaries.go rename to lib/vmm/binaries_linux.go index 319884a2..73064a41 100644 --- a/lib/vmm/binaries.go +++ b/lib/vmm/binaries_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package vmm import ( diff --git a/vz.entitlements b/vz.entitlements new file mode 100644 index 00000000..41432913 --- /dev/null +++ b/vz.entitlements @@ -0,0 +1,14 @@ + + + + + + com.apple.security.virtualization + + + com.apple.security.network.server + + com.apple.security.network.client + + +