Merge pull request #565 from markdryan/master

ciao-launcher: Implement boot from volume

ciao-launcher/attachvolume_instance.go

@@ -24,7 +24,7 @@ import (
 
 func processAttachVolume(storageDriver storage.BlockDriver, monitorCh chan interface{}, cfg *vmConfig,
 	instance, instanceDir, volumeUUID string, conn serverConn) *attachVolumeError {
-	if _, found := cfg.Volumes[volumeUUID]; found {
+	if cfg.findVolume(volumeUUID) != nil {
 		attachErr := &attachVolumeError{nil, payloads.AttachVolumeAlreadyAttached}
 		glog.Errorf("%s is already attached to attach instance %s [%s]",
 			volumeUUID, instance, string(attachErr.code))
@@ -73,12 +73,12 @@ func processAttachVolume(storageDriver storage.BlockDriver, monitorCh chan inter
 		}
 	}
 
-	cfg.Volumes[volumeUUID] = struct{}{}
+	cfg.Volumes = append(cfg.Volumes, volumeConfig{UUID: volumeUUID})
 
 	err := cfg.save(instanceDir)
 	if err != nil {
 		// TODO: should we detach and unmap here?
-		delete(cfg.Volumes, volumeUUID)
+		cfg.removeVolume(volumeUUID)
 		attachErr := &attachVolumeError{err, payloads.AttachVolumeStateFailure}
 		glog.Errorf("Unable to persist instance %s state [%s]: %v",
 			instance, string(attachErr.code), err)

ciao-launcher/detachvolume_instance.go

@@ -23,13 +23,20 @@ import (
 )
 
 func processDetachVolume(storageDriver storage.BlockDriver, monitorCh chan interface{}, cfg *vmConfig, instance, instanceDir, volumeUUID string, conn serverConn) *detachVolumeError {
-	if _, found := cfg.Volumes[volumeUUID]; !found {
+	vol := cfg.findVolume(volumeUUID)
+	if vol == nil {
 		detachErr := &detachVolumeError{nil, payloads.DetachVolumeNotAttached}
 		glog.Errorf("%s not attached to attach instance %s [%s]",
 			volumeUUID, instance, string(detachErr.code))
 		return detachErr
 	}
 
+	if cfg.Image == "" && vol.Bootable {
+		glog.Errorf("Unable to detach volume %s from instance %s. Volume is bootable!", volumeUUID, instance)
+		attachErr := &detachVolumeError{nil, payloads.DetachVolumeDetachFailure}
+		return attachErr
+	}
+
 	if monitorCh != nil {
 		responseCh := make(chan error)
 		monitorCh <- virtualizerDetachCmd{
@@ -51,12 +58,13 @@ func processDetachVolume(storageDriver storage.BlockDriver, monitorCh chan inter
 	// but we might be able to get good error info out of rbd.
 	_ = storageDriver.UnmapVolumeFromNode(volumeUUID)
 
-	delete(cfg.Volumes, volumeUUID)
+	oldVols := cfg.Volumes
+	cfg.removeVolume(volumeUUID)
 
 	err := cfg.save(instanceDir)
 	if err != nil {
 		// TODO: What should I do here.  Try to re-attach?
-		cfg.Volumes[volumeUUID] = struct{}{}
+		cfg.Volumes = oldVols
 		detachErr := &detachVolumeError{err, payloads.DetachVolumeDetachFailure}
 		glog.Errorf("Unable to persist instance %s state [%s]: %v",
 			instance, string(detachErr.code), err)

ciao-launcher/instance.go

@@ -308,23 +308,23 @@ func (id *instanceData) instanceCommand(cmd interface{}) bool {
 
 func (id *instanceData) getVolumes() []string {
 	volumes := make([]string, 0, len(id.cfg.Volumes))
-	for k := range id.cfg.Volumes {
-		volumes = append(volumes, k)
+	for _, v := range id.cfg.Volumes {
+		volumes = append(volumes, v.UUID)
 	}
 	return volumes
 }
 
 func (id *instanceData) unmapVolumes() {
 	glog.Infof("Unmapping volumes for %s", id.instance)
 
-	for k := range id.cfg.Volumes {
+	for _, v := range id.cfg.Volumes {
 
 		// UnmapVolumeFromNode might fail if it's mapped to multiple
 		// instances on the same node.  We don't treat this as an
 		// error for now.
 
-		if err := id.storageDriver.UnmapVolumeFromNode(k); err == nil {
-			glog.Infof("Unmapping volume %s", k)
+		if err := id.storageDriver.UnmapVolumeFromNode(v.UUID); err == nil {
+			glog.Infof("Unmapping volume %s", v.UUID)
 		}
 	}
 }

ciao-launcher/instance_test.go

@@ -920,7 +920,6 @@ func TestStartRunningInstance(t *testing.T) {
 func TestAttachVolumeToInstance(t *testing.T) {
 	var wg sync.WaitGroup
 	cfg := standardCfg
-	cfg.Volumes = make(map[string]struct{})
 	state, ovsCh, cmdCh, doneCh := startVMWithCFG(t, &wg, &cfg, true, false)
 
 	select {
@@ -954,7 +953,6 @@ func TestAttachVolumeToInstance(t *testing.T) {
 func TestAttachExistingVolumeToInstance(t *testing.T) {
 	var wg sync.WaitGroup
 	cfg := standardCfg
-	cfg.Volumes = make(map[string]struct{})
 	state, ovsCh, cmdCh, doneCh := startVMWithCFG(t, &wg, &cfg, true, false)
 
 	select {
@@ -1007,7 +1005,6 @@ func TestAttachExistingVolumeToInstance(t *testing.T) {
 func TestDetachVolumeFromInstance(t *testing.T) {
 	var wg sync.WaitGroup
 	cfg := standardCfg
-	cfg.Volumes = make(map[string]struct{})
 	state, ovsCh, cmdCh, doneCh := startVMWithCFG(t, &wg, &cfg, true, false)
 
 	select {
@@ -1053,7 +1050,6 @@ func TestDetachVolumeFromInstance(t *testing.T) {
 func TestDetachNonexistingVolumeFromInstance(t *testing.T) {
 	var wg sync.WaitGroup
 	cfg := standardCfg
-	cfg.Volumes = make(map[string]struct{})
 	state, ovsCh, cmdCh, doneCh := startVMWithCFG(t, &wg, &cfg, true, false)
 
 	select {

ciao-launcher/payload.go

@@ -73,6 +73,11 @@ func printCloudinit(data *payloads.Start) {
 		glog.Infof("%8s:     %v", start.RequestedResources[i].Type,
 			start.RequestedResources[i].Value)
 	}
+
+	if start.Storage.ID != "" {
+		glog.Info("Volumes:")
+		glog.Infof("  %s Bootable=%t", start.Storage.ID, start.Storage.Bootable)
+	}
 }
 
 func computeSSHPort(networkNode bool, vnicIP string) int {
@@ -162,9 +167,12 @@ func parseStartPayload(data []byte) (*vmConfig, *payloadError) {
 	net := &start.Networking
 	vnicIP := strings.TrimSpace(net.PrivateIP)
 	sshPort := computeSSHPort(networkNode, vnicIP)
-	volumes := make(map[string]struct{})
+	var volumes []volumeConfig
 	if start.Storage.ID != "" {
-		volumes[start.Storage.ID] = struct{}{}
+		volumes = append(volumes, volumeConfig{
+			UUID:     start.Storage.ID,
+			Bootable: start.Storage.Bootable,
+		})
 	}
 
 	return &vmConfig{Cpus: cpus,

ciao-launcher/qemu.go

@@ -206,6 +206,9 @@ func createCloudInitISO(instanceDir, isoPath string, cfg *vmConfig, userData, me
 }
 
 func (q *qemuV) createRootfs() error {
+	if q.cfg.Image == "" {
+		return nil
+	}
 	vmImage := path.Join(q.instanceDir, "image.qcow2")
 	backingImage := path.Join(imagesPath, q.cfg.Image)
 	glog.Infof("Creating qcow image from %s backing %s", vmImage, backingImage)
@@ -421,15 +424,15 @@ func launchQemuWithSpice(params []string, fds []*os.File, ipAddress string) (int
 
 func generateQEMULaunchParams(cfg *vmConfig, isoPath, instanceDir string,
 	networkParams []string, cephID string) []string {
-	vmImage := path.Join(instanceDir, "image.qcow2")
-	qmpSocket := path.Join(instanceDir, "socket")
-	fileParam := fmt.Sprintf("file=%s,if=virtio,aio=threads,format=qcow2", vmImage)
-	isoParam := fmt.Sprintf("file=%s,if=virtio,media=cdrom", isoPath)
-	qmpParam := fmt.Sprintf("unix:%s,server,nowait", qmpSocket)
-
 	params := make([]string, 0, 32)
-	params = append(params, "-drive", fileParam)
-	params = append(params, "-drive", isoParam)
+
+	addr := 4
+	if cfg.Image != "" {
+		vmImage := path.Join(instanceDir, "image.qcow2")
+		fileParam := fmt.Sprintf("file=%s,if=virtio,aio=threads,format=qcow2", vmImage)
+		params = append(params, "-drive", fileParam)
+		addr++
+	}
 
 	// I know this is nasty but we have to specify a bus and address otherwise qemu
 	// hangs on startup.  I can't find a way to get qemu to pre-allocate the address.
@@ -438,24 +441,29 @@ func generateQEMULaunchParams(cfg *vmConfig, isoPath, instanceDir string,
 	// adds, i.e., the root address is assigned a slot of 3, with the current qemu
 	// parameters.
 
-	addr := 5
-	for v := range cfg.Volumes {
-		blockdevID := fmt.Sprintf("drive_%s", v)
+	for _, v := range cfg.Volumes {
+		blockdevID := fmt.Sprintf("drive_%s", v.UUID)
 		volDriveStr := fmt.Sprintf("file=rbd:rbd/%s:id=%s,if=none,id=%s,format=raw",
-			v, cephID, blockdevID)
+			v.UUID, cephID, blockdevID)
 		params = append(params, "-drive", volDriveStr)
 		volDeviceStr :=
 			fmt.Sprintf("virtio-blk-pci,scsi=off,bus=pci.0,addr=0x%x,id=device_%s,drive=%s",
-				addr, v, blockdevID)
+				addr, v.UUID, blockdevID)
 		params = append(params, "-device", volDeviceStr)
 		addr++
 	}
 
+	isoParam := fmt.Sprintf("file=%s,if=virtio,media=cdrom", isoPath)
+	params = append(params, "-drive", isoParam)
+
 	params = append(params, networkParams...)
 
 	params = append(params, "-enable-kvm")
 	params = append(params, "-cpu", "host")
 	params = append(params, "-daemonize")
+
+	qmpSocket := path.Join(instanceDir, "socket")
+	qmpParam := fmt.Sprintf("unix:%s,server,nowait", qmpSocket)
 	params = append(params, "-qmp", qmpParam)
 
 	if cfg.Mem > 0 {
@@ -646,7 +654,10 @@ func (q *qemuV) monitorVM(closedCh chan struct{}, connectedCh chan struct{},
 	return qmpChannel
 }
 
-func computeInstanceDiskspace(instanceDir string) int {
+func (q *qemuV) computeInstanceDiskspace(instanceDir string) int {
+	if q.cfg.Image == "" {
+		return 0
+	}
 	vmImage := path.Join(instanceDir, "image.qcow2")
 	fi, err := os.Stat(vmImage)
 	if err != nil {
@@ -656,7 +667,7 @@ func computeInstanceDiskspace(instanceDir string) int {
 }
 
 func (q *qemuV) stats() (disk, memory, cpu int) {
-	disk = computeInstanceDiskspace(q.instanceDir)
+	disk = q.computeInstanceDiskspace(q.instanceDir)
 	memory = -1
 	cpu = -1
 

ciao-launcher/qemu_test.go

@@ -154,6 +154,7 @@ func TestGenerateQEMULaunchParams(t *testing.T) {
 
 	params := genQEMUParams(nil)
 	cfg.Legacy = true
+	cfg.Image = "some_image"
 	genParams := generateQEMULaunchParams(&cfg, "/var/lib/ciao/instance/1/seed.iso",
 		"/var/lib/ciao/instance/1", nil, "ciao")
 	if !reflect.DeepEqual(params, genParams) {

ciao-launcher/start_instance.go

@@ -106,8 +106,10 @@ func processStart(cmd *insStartCmd, instanceDir string, vm virtualizer, conn ser
 	}
 
 	if cfg.Image == "" {
-		err = fmt.Errorf("No backing image specified")
-		return nil, &startError{err, payloads.InvalidData}
+		if len(cfg.Volumes) == 0 || !cfg.Volumes[0].Bootable {
+			err = fmt.Errorf("No backing image specified")
+			return nil, &startError{err, payloads.InvalidData}
+		}
 	}
 
 	if networking {
@@ -118,9 +120,11 @@ func processStart(cmd *insStartCmd, instanceDir string, vm virtualizer, conn ser
 		}
 	}
 
-	err = ensureBackingImage(vm)
-	if err != nil {
-		return nil, &startError{err, payloads.ImageFailure}
+	if cfg.Image != "" {
+		err = ensureBackingImage(vm)
+		if err != nil {
+			return nil, &startError{err, payloads.ImageFailure}
+		}
 	}
 
 	st.backingImageCheck = time.Now()

ciao-launcher/tests/ciao-launcher-server/server.go

@@ -168,6 +168,18 @@ func (ts *testServer) ErrorNotify(uuid string, err ssntp.Error, frame *ssntp.Fra
 		if err == nil {
 			e = &payload
 		}
+	case ssntp.AttachVolumeFailure:
+		payload := payloads.ErrorAttachVolumeFailure{}
+		err := yaml.Unmarshal(frame.Payload, &payload)
+		if err == nil {
+			e = &payload
+		}
+	case ssntp.DetachVolumeFailure:
+		payload := payloads.ErrorDetachVolumeFailure{}
+		err := yaml.Unmarshal(frame.Payload, &payload)
+		if err == nil {
+			e = &payload
+		}
 	}
 
 	c.events = append(c.events, e)

ciao-launcher/tests/examples/start_legacy_volume_boot.yaml

@@ -0,0 +1,43 @@
+---
+start:
+  requested_resources:
+     - type: vcpus
+       value: 2
+     - type: mem_mb
+       value: 370
+     - type: disk_mb
+       value: 8000
+  instance_uuid: d7d86208-b46c-4465-9018-ee14087d415f
+  tenant_uuid: 67d86208-000-4465-9018-fe14087d415f
+  fw_type: legacy
+  networking:
+    vnic_mac: 02:00:e6:f5:af:f9
+    vnic_uuid: 67d86208-b46c-0000-9018-fe14087d415f
+    concentrator_ip: 192.168.42.21
+    concentrator_uuid: 67d86208-b46c-4465-0000-fe14087d415f
+    subnet: 192.168.8.0/21
+    private_ip: 192.168.8.2
+  storage:
+    id: 69e84267-ed01-4738-b15f-b47de06b62e7
+    boot: true
+
+...
+---
+#cloud-config
+runcmd:
+  - [ touch, "/etc/bootdone" ]
+users:
+  - name: demouser
+    gecos: CIAO Demo User
+    lock-passwd: false
+    passwd: $1$vzmNmLLD$04bivxcjdXRzZLUd.enRl1
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh-authorized-keys:
+    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDerQfD+qkb0V0XdQs8SBWqy4sQmqYFP96n/kI4Cq162w4UE8pTxy0ozAPldOvBJjljMvgaNKSAddknkhGcrNUvvJsUcZFm2qkafi32WyBdGFvIc45A+8O7vsxPXgHEsS9E3ylEALXAC3D0eX7pPtRiAbasLlY+VcACRqr3bPDSZTfpCmIkV2334uZD9iwOvTVeR+FjGDqsfju4DyzoAIqpPasE0+wk4Vbog7osP+qvn1gj5kQyusmr62+t0wx+bs2dF5QemksnFOswUrv9PGLhZgSMmDQrRYuvEfIAC7IdN/hfjTn0OokzljBiuWQ4WIIba/7xTYLVujJV65qH3heaSMxJJD7eH9QZs9RdbbdTXMFuJFsHV2OF6wZRp18tTNZZJMqiHZZSndC5WP1WrUo3Au/9a+ighSaOiVddHsPG07C/TOEnr3IrwU7c9yIHeeRFHmcQs9K0+n9XtrmrQxDQ9/mLkfje80Ko25VJ/QpAQPzCKh2KfQ4RD+/PxBUScx/lHIHOIhTSCh57ic629zWgk0coSQDi4MKSa5guDr3cuDvt4RihGviDM6V68ewsl0gh6Z9c0Hw7hU0vky4oxak5AiySiPz0FtsOnAzIL0UON+yMuKzrJgLjTKodwLQ0wlBXu43cD+P8VXwQYeqNSzfrhBnHqsrMf4lTLtc7kDDTcw== ciao@ciao
+...
+---
+{
+  "uuid": "ciao",
+  "hostname": "ciao"
+}
+...

ciao-launcher/vmconfig.go

@@ -24,6 +24,11 @@ import (
 	"github.com/golang/glog"
 )
 
+type volumeConfig struct {
+	UUID     string
+	Bootable bool
+}
+
 type vmConfig struct {
 	Cpus        int
 	Mem         int
@@ -41,7 +46,7 @@ type vmConfig struct {
 	ConcUUID    string
 	VnicUUID    string
 	SSHPort     int
-	Volumes     map[string]struct{}
+	Volumes     []volumeConfig
 }
 
 func loadVMConfig(instanceDir string) (*vmConfig, error) {
@@ -62,10 +67,6 @@ func loadVMConfig(instanceDir string) (*vmConfig, error) {
 		return nil, err
 	}
 
-	if cfg.Volumes == nil {
-		cfg.Volumes = make(map[string]struct{})
-	}
-
 	return cfg, nil
 }
 
@@ -86,3 +87,25 @@ func (cfg *vmConfig) save(instanceDir string) error {
 
 	return cfgFile.Close()
 }
+
+func (cfg *vmConfig) findVolume(UUID string) *volumeConfig {
+	for i := range cfg.Volumes {
+		if cfg.Volumes[i].UUID == UUID {
+			return &cfg.Volumes[i]
+		}
+	}
+	return nil
+}
+
+func (cfg *vmConfig) removeVolume(UUID string) {
+	for i := range cfg.Volumes {
+		if cfg.Volumes[i].UUID == UUID {
+			vols := cfg.Volumes
+			cfg.Volumes = vols[:i]
+			if i+1 < len(vols) {
+				cfg.Volumes = append(cfg.Volumes, vols[i+1:]...)
+			}
+			break
+		}
+	}
+}