Skip to content
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions .air.darwin.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
root = "."
testdata_dir = "testdata"
tmp_dir = "tmp"

[build]
args_bin = []
bin = "./tmp/main"
# Build for macOS with vz support, then sign with entitlements
cmd = "make build-embedded && go build -tags containers_image_openpgp -o ./tmp/main ./cmd/api && codesign --sign - --entitlements vz.entitlements --force ./tmp/main"
delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata", "bin", "scripts", "data", "kernel"]
exclude_file = []
exclude_regex = ["_test.go"]
exclude_unchanged = false
follow_symlink = false
# No sudo needed on macOS - vz doesn't require root
full_bin = "./tmp/main"
include_dir = []
include_ext = ["go", "tpl", "tmpl", "html", "yaml"]
include_file = []
log = "build-errors.log"
poll = false
poll_interval = 0
post_cmd = []
kill_delay = '1s'
rerun = false
rerun_delay = 500
send_interrupt = true
stop_on_error = false

[color]
app = ""
build = "yellow"
main = "magenta"
runner = "green"
watcher = "cyan"

[log]
main_only = false
time = false

[misc]
clean_on_exit = false

[screen]
clear_on_rebuild = false
keep_scroll = true
122 changes: 122 additions & 0 deletions .env.darwin.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
# =============================================================================
# macOS (Darwin) Configuration for Hypeman
# =============================================================================
# Copy this file to .env and customize for your environment.
#
# Key differences from Linux (.env.example):
# - DEFAULT_HYPERVISOR: Use "vz" (Virtualization.framework) instead of cloud-hypervisor/qemu
# - DATA_DIR: Uses macOS conventions (~/Library/Application Support)
# - Network settings: BRIDGE_NAME, SUBNET_CIDR, etc. are IGNORED (vz uses NAT)
# - Rate limiting: Not supported on macOS (no tc/HTB equivalent)
# - GPU passthrough: Not supported on macOS
# =============================================================================

# Required
JWT_SECRET=dev-secret-change-me

# Data directory - use macOS conventions
# Note: ~ expands to $HOME at runtime
DATA_DIR=~/Library/Application Support/hypeman

# Server configuration
PORT=8080

# Logging
LOG_LEVEL=debug

# =============================================================================
# Hypervisor Configuration (IMPORTANT FOR MACOS)
# =============================================================================
# On macOS, use "vz" (Virtualization.framework)
# - "cloud-hypervisor" and "qemu" are NOT supported on macOS
DEFAULT_HYPERVISOR=vz

# =============================================================================
# Network Configuration (DIFFERENT ON MACOS)
# =============================================================================
# On macOS with vz, network is handled automatically via NAT:
# - VMs get IP addresses from 192.168.64.0/24 via DHCP
# - No TAP devices, bridges, or iptables needed
# - The following settings are IGNORED on macOS:
# BRIDGE_NAME, SUBNET_CIDR, SUBNET_GATEWAY, UPLINK_INTERFACE

# DNS Server for VMs (used by guest for resolution)
DNS_SERVER=8.8.8.8

# =============================================================================
# Caddy / Ingress Configuration
# =============================================================================
CADDY_LISTEN_ADDRESS=0.0.0.0
CADDY_ADMIN_ADDRESS=127.0.0.1
CADDY_ADMIN_PORT=2019
# Note: 5353 is used by mDNSResponder (Bonjour) on macOS, using 5354 instead
INTERNAL_DNS_PORT=5354
CADDY_STOP_ON_SHUTDOWN=false

# =============================================================================
# Build System Configuration
# =============================================================================
# For builds on macOS with vz, the registry URL needs to be accessible from
# NAT VMs. Since vz uses 192.168.64.0/24 for NAT, the host is at 192.168.64.1.
#
# IMPORTANT: "host.docker.internal" does NOT work in vz VMs - that's a Docker
# Desktop-specific hostname. Use the NAT gateway IP instead.
#
# Registry URL (the host's hypeman API, accessible from VMs)
REGISTRY_URL=192.168.64.1:8080
# Use HTTP (not HTTPS) since hypeman's internal registry uses plaintext
REGISTRY_INSECURE=true

BUILDER_IMAGE=hypeman/builder:latest
MAX_CONCURRENT_SOURCE_BUILDS=2
BUILD_TIMEOUT=600

# =============================================================================
# Resource Limits (same as Linux)
# =============================================================================
# Per-instance limits
MAX_VCPUS_PER_INSTANCE=4
MAX_MEMORY_PER_INSTANCE=8GB

# Aggregate limits (0 or empty = unlimited)
# MAX_TOTAL_VOLUME_STORAGE=

# =============================================================================
# OpenTelemetry (optional, same as Linux)
# =============================================================================
# OTEL_ENABLED=false
# OTEL_ENDPOINT=127.0.0.1:4317
# OTEL_SERVICE_NAME=hypeman
# OTEL_INSECURE=true
# ENV=dev

# =============================================================================
# TLS / ACME Configuration (same as Linux)
# =============================================================================
# ACME_EMAIL=admin@example.com
# ACME_DNS_PROVIDER=cloudflare
# TLS_ALLOWED_DOMAINS=*.example.com
# CLOUDFLARE_API_TOKEN=

# =============================================================================
# macOS Limitations
# =============================================================================
# The following features are NOT AVAILABLE on macOS:
#
# 1. GPU Passthrough (VFIO, mdev)
# - GPU_PROFILE_CACHE_TTL is ignored
# - Device registration/binding will fail
#
# 2. Network Rate Limiting
# - UPLOAD_BURST_MULTIPLIER, DOWNLOAD_BURST_MULTIPLIER are ignored
# - No tc/HTB equivalent on macOS
#
# 3. CPU/Memory Hotplug
# - Resize operations not supported
#
# 4. Disk I/O Limiting
# - DISK_IO_LIMIT, OVERSUB_DISK_IO are ignored
#
# 5. Snapshots (requires macOS 14+ on Apple Silicon)
# - SaveMachineStateToPath/RestoreMachineStateFromURL require macOS 14+
# - Only supported on ARM64 (Apple Silicon) Macs
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,9 @@ dist/**

# UTM VM - downloaded ISO files
scripts/utm/images/

# IDE and editor
.cursor/

# Build artifacts
api
185 changes: 184 additions & 1 deletion DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,17 @@ This document covers development setup, configuration, and contributing to Hypem

## Prerequisites

> **macOS Users:** Hypeman requires KVM, which is only available on Linux. See [scripts/utm/README.md](scripts/utm/README.md) for instructions on setting up a Linux VM with nested virtualization on Apple Silicon Macs.
### Linux (Default)

**Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq**

### macOS (Experimental)

See [macOS Development](#macos-development) below for native macOS development using Virtualization.framework.

---

**Linux Prerequisites:**

**Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq**

Expand Down Expand Up @@ -314,3 +324,176 @@ Or generate everything at once:
```bash
make generate-all
```

## macOS Development

Hypeman supports native macOS development using Apple's Virtualization.framework (via the `vz` hypervisor).

### Requirements

- **macOS 11.0+** (Big Sur or later)
- **Apple Silicon** (M1/M2/M3) recommended
- **macOS 14.0+** (Sonoma) required for snapshot/restore (ARM64 only)
- **Go 1.25.4+**
- **Caddy** (for ingress): `brew install caddy`
- **e2fsprogs** (for ext4 disk images): `brew install e2fsprogs`

### Quick Start

```bash
# 1. Install dependencies
brew install caddy e2fsprogs

# 2. Add e2fsprogs to PATH (it's keg-only)
export PATH="/opt/homebrew/opt/e2fsprogs/bin:/opt/homebrew/opt/e2fsprogs/sbin:$PATH"
# Add to ~/.zshrc for persistence

# 3. Configure environment
cp .env.darwin.example .env
# Edit .env as needed (defaults work for local development)

# 4. Create data directory
mkdir -p ~/Library/Application\ Support/hypeman

# 5. Run in development mode (auto-detects macOS, builds, signs, and runs with hot reload)
make dev
```

The `make dev` command automatically detects macOS and:
- Builds with vz support
- Signs with required entitlements
- Runs with hot reload (no sudo required)

### Alternative Commands

```bash
# Build and sign only (no hot reload)
make sign-darwin

# Verify entitlements are correct
make verify-entitlements

# Run manually after signing
./bin/hypeman
```

### Key Differences from Linux Development

| Aspect | Linux | macOS |
|--------|-------|-------|
| Hypervisor | Cloud Hypervisor, QEMU | vz (Virtualization.framework) |
| Binary signing | Not required | Automatic via `make dev` or `make sign-darwin` |
| Networking | TAP + bridge + iptables | Automatic NAT (no setup needed) |
| Root/sudo | Required for networking | Not required |
| Caddy | Embedded binary | Install via `brew install caddy` |
| DNS port | 5353 | 5354 (avoids mDNSResponder conflict) |

### macOS-Specific Configuration

The following environment variables work differently on macOS (see `.env.darwin.example`):

| Variable | Linux | macOS |
|----------|-------|-------|
| `DEFAULT_HYPERVISOR` | `cloud-hypervisor` | `vz` |
| `DATA_DIR` | `/var/lib/hypeman` | `~/Library/Application Support/hypeman` |
| `INTERNAL_DNS_PORT` | `5353` | `5354` (5353 is used by mDNSResponder) |
| `BRIDGE_NAME` | Used | Ignored (NAT) |
| `SUBNET_CIDR` | Used | Ignored (NAT) |
| `UPLINK_INTERFACE` | Used | Ignored (NAT) |
| Network rate limiting | Supported | Not supported |
| GPU passthrough | Supported (VFIO) | Not supported |

### Code Organization

Platform-specific code uses Go build tags:

```
lib/network/
├── bridge_linux.go # Linux networking (TAP, bridges, iptables)
├── bridge_darwin.go # macOS stubs (uses NAT)
└── ip.go # Shared utilities

lib/devices/
├── discovery_linux.go # Linux PCI device discovery
├── discovery_darwin.go # macOS stubs (no passthrough)
├── mdev_linux.go # Linux vGPU (mdev)
├── mdev_darwin.go # macOS stubs
├── vfio_linux.go # Linux VFIO binding
├── vfio_darwin.go # macOS stubs
└── types.go # Shared types

lib/hypervisor/
├── cloudhypervisor/ # Cloud Hypervisor (Linux)
├── qemu/ # QEMU (Linux, vsock_linux.go)
└── vz/ # Virtualization.framework (macOS only)
├── starter.go # VMStarter implementation
├── hypervisor.go # Hypervisor interface
└── vsock.go # VsockDialer via VirtioSocketDevice
```

### Testing on macOS

```bash
# Verify vz package compiles correctly
make test-vz-compile

# Run unit tests (Linux-specific tests like networking will be skipped)
go test ./lib/hypervisor/vz/...
go test ./lib/resources/...
go test ./lib/images/...
```

Note: Full integration tests require Linux. On macOS, focus on unit tests and manual API testing.

### Known Limitations

1. **Disk Format**: vz only supports raw disk images (not qcow2). Convert images:
```bash
qemu-img convert -f qcow2 -O raw disk.qcow2 disk.raw
```

2. **Snapshots**: Only available on macOS 14+ (Sonoma) on Apple Silicon:
```go
// Check support at runtime
valid, err := vmConfig.ValidateSaveRestoreSupport()
```

3. **Network Ingress**: VMs get DHCP addresses from macOS NAT. To access a VM's services:
- Query the VM's IP via guest agent
- Use vsock for internal communication (no NAT traversal needed)

4. **In-Process VMM**: Unlike CH/QEMU which run as separate processes, vz VMs run in the hypeman process. If hypeman crashes, all VMs stop.

### Troubleshooting

**"binary needs to be signed with entitlements"**
```bash
make sign-darwin
# Or just use: make dev (handles signing automatically)
```

**"caddy binary is not embedded on macOS"**
```bash
brew install caddy
```

**"address already in use" on port 5353**
- Port 5353 is used by mDNSResponder (Bonjour) on macOS
- Use port 5354 instead: `INTERNAL_DNS_PORT=5354` in `.env`
- The `.env.darwin.example` already has this configured correctly

**"Virtualization.framework is not available"**
- Ensure you're on macOS 11.0+
- Check if virtualization is enabled in Recovery Mode settings

**"snapshot not supported"**
- Requires macOS 14.0+ on Apple Silicon
- Check: `sw_vers` and `uname -m` (should be arm64)

**VM fails to start**
- Check serial log: `$DATA_DIR/instances/<id>/serial.log`
- Ensure kernel and initrd paths are correct in config

**IOMMU/VFIO warnings at startup**
- These are expected on macOS and can be ignored
- GPU passthrough is not supported on macOS
Loading
Loading