feat(shared-pci-device): add vgpu to vm resource and update website doc

Author: mristok

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
 	github.com/agext/levenshtein v1.2.2 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect

go.sum

@@ -9,8 +9,9 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
-github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
+github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

vsphere/internal/virtualdevice/virtual_machine_device_subresource.go

@@ -1091,62 +1091,147 @@ func PciPassthroughPostCloneOperation(d *schema.ResourceData, c *govmomi.Client,
 	return applyConfig.VirtualDevice, applyConfig.Spec, nil
 }
 
-// SharedPciPassthroughApplyOperation checks for changes in a virtual machine's
-// Shared PCI passthrough device and creates config specs to apply apply to the
-// virtual machine.
-func SharedPciPassthroughApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
-	old, new := d.GetChange("shared_pci_device_id")
-	oldDevId := old.(string)
-	newDevId := new.(string)
+// modifyVirtualSharedPciDevices will take a list of devices and an operation and
+// will create the appropriate config spec.
+func (c *pciApplyConfig) modifyVirtualSharedPciDevices(devList *schema.Set, op types.VirtualDeviceConfigSpecOperation) error {
+	log.Printf("VirtualMachine: Creating Shared PCI device specs %v", op)
+	for _, devId := range devList.List() {
+		log.Printf("[DEBUG] modifyVirtualSharedPciDevices: Appending %v spec for %s", op, devId.(string))
 
-	var specs []types.BaseVirtualDeviceConfigSpec
-	if oldDevId == newDevId {
-		return l, specs, nil
-	}
+		dev := &types.VirtualPCIPassthrough{
+			VirtualDevice: types.VirtualDevice{
+				DynamicData: types.DynamicData{},
+				Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{
+					Vgpu: devId.(string),
+				},
+			},
+		}
 
-	if oldDevId != "" {
-		vm, err := virtualmachine.FromUUID(c, d.Id())
+		vm, err := virtualmachine.FromUUID(c.Client, c.ResourceData.Id())
 		if err != nil {
-			return nil, nil, err
+			return err
 		}
+
 		vprops, err := virtualmachine.Properties(vm)
 		if err != nil {
-			return nil, nil, err
+			return err
 		}
+
 		// This will only find a device for delete operations.
 		for _, vmDevP := range vprops.Config.Hardware.Device {
 			if vmDev, ok := vmDevP.(*types.VirtualPCIPassthrough); ok {
-				if vmDev.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo).Vgpu == oldDevId {
-					dspec, err := object.VirtualDeviceList{vmDev}.ConfigSpec(types.VirtualDeviceConfigSpecOperationRemove)
-					if err != nil {
-						return nil, nil, err
-					}
-					specs = append(specs, dspec...)
-
-					l = applyDeviceChange(l, dspec)
-					d.Set("reboot_required", true)
+				if vmDev.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo).Vgpu == devId {
+					dev = vmDev
 				}
 			}
 		}
-	}
 
-	if newDevId != "" {
-		dev := &types.VirtualPCIPassthrough{
-			VirtualDevice: types.VirtualDevice{
-				DynamicData: types.DynamicData{},
-				Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{
-					Vgpu: newDevId,
-				},
-			},
-		}
-		dspec, err := object.VirtualDeviceList{dev}.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd)
+		dspec, err := object.VirtualDeviceList{dev}.ConfigSpec(op)
 		if err != nil {
-			return nil, nil, err
+			return err
 		}
-		specs = append(specs, dspec...)
-		l = applyDeviceChange(l, dspec)
-		d.Set("reboot_required", true)
+
+		c.Spec = append(c.Spec, dspec...)
+		c.VirtualDevice = applyDeviceChange(c.VirtualDevice, dspec)
+	}
+	log.Printf("VirtualMachine: Shared PCI device specs created")
+	return nil
+}
+
+// SharedPciApplyOperation checks for changes in a virtual machine's
+// Shared PCI device and creates config specs to apply apply to the
+// virtual machine.
+func SharedPciApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
+	log.Printf("[DEBUG] SharedPciApplyOperation: Looking for shared PCI device changes")
+
+	// Get current (old) and new devices
+	old, newValue := d.GetChange("shared_pci_device_id")
+	oldDevIds := old.(*schema.Set)
+	newDevIds := newValue.(*schema.Set)
+
+	// Compare
+	delDevs := oldDevIds.Difference(newDevIds)
+	addDevs := newDevIds.Difference(oldDevIds)
+
+	// Create base apply config
+	applyConfig := &pciApplyConfig{
+		Client:        c,
+		ResourceData:  d,
+		Spec:          []types.BaseVirtualDeviceConfigSpec{},
+		VirtualDevice: l,
+	}
+
+	// If there are no changes, return as is
+	if addDevs.Len() == 0 && delDevs.Len() == 0 {
+		log.Printf("[DEBUG] SharedPciApplyOperation: No shared PCI device additions/deletions")
+		return applyConfig.VirtualDevice, applyConfig.Spec, nil
+	}
+
+	// Set reboot
+	_ = d.Set("reboot_required", true)
+
+	// Add new Shared PCI devices
+	log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device additions",
+		addDevs.Len())
+	err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd)
+	if err != nil {
+		return nil, nil, err
 	}
 
-	return l, specs, nil
+	// Remove deleted Shared PCI devices
+	log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device deletions",
+		delDevs.Len())
+	err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return applyConfig.VirtualDevice, applyConfig.Spec, nil
+}
+
+// SharedPciPostCloneOperation normalizes the Shared PCI devices
+// on a newly-cloned virtual machine and outputs any necessary device change
+// operations. It also sets the state in advance of the post-create read.
+func SharedPciPostCloneOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
+	log.Printf("[DEBUG] SharedPCIPostCloneOperation: Looking for shared PCI device changes post-clone")
+
+	// Get current (old) and new devices
+	old, newValue := d.GetChange("shared_pci_device_id")
+	oldDevIds := old.(*schema.Set)
+	newDevIds := newValue.(*schema.Set)
+
+	// Compare
+	delDevs := oldDevIds.Difference(newDevIds)
+	addDevs := newDevIds.Difference(oldDevIds)
+
+	// Create base apply config
+	applyConfig := &pciApplyConfig{
+		Client:        c,
+		ResourceData:  d,
+		Spec:          []types.BaseVirtualDeviceConfigSpec{},
+		VirtualDevice: l,
+	}
+
+	// If there are no changes, return as is
+	if addDevs.Len() <= 0 && delDevs.Len() <= 0 {
+		log.Printf("[DEBUG] SharedPCIPostCloneOperation: No shared PCI device additions/deletions post-clone")
+		return applyConfig.VirtualDevice, applyConfig.Spec, nil
+	}
+
+	// Add new Shared PCI devices
+	log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device additions post-clone",
+		addDevs.Len())
+	err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Remove deleted Shared PCI devices
+	log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device deletions post-clone",
+		delDevs.Len())
+	err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove)
+	if err != nil {
+		return nil, nil, err
+	}
+	return applyConfig.VirtualDevice, applyConfig.Spec, nil
 }

vsphere/resource_vsphere_file.go

@@ -149,13 +149,10 @@ func createDirectory(datastoreFileManager *object.DatastoreFileManager, f *file)
 
 // fileUpload - upload file to a vSphere datastore
 func fileUpload(client *govmomi.Client, dc *object.Datacenter, ds *object.Datastore, source, destination string) error {
-	dsurl, err := ds.URL(context.TODO(), dc, destination)
-	if err != nil {
-		return err
-	}
+	dsurl := ds.NewURL(destination)
 
 	p := soap.DefaultUpload
-	err = client.Client.UploadFile(context.TODO(), source, dsurl, &p)
+	err := client.Client.UploadFile(context.TODO(), source, dsurl, &p)
 	if err != nil {
 		return err
 	}

vsphere/resource_vsphere_virtual_machine.go

@@ -244,9 +244,9 @@ func resourceVSphereVirtualMachine() *schema.Resource {
 			Elem:        &schema.Schema{Type: schema.TypeString},
 		},
 		"shared_pci_device_id": {
-			Type:        schema.TypeString,
+			Type:        schema.TypeSet,
 			Optional:    true,
-			Description: "Id of Shared PCI passthrough device, 'grid_rtx8000-8q'",
+			Description: "A list of Shared PCI passthrough device, 'grid_rtx8000-8q'",
 			Elem:        &schema.Schema{Type: schema.TypeString},
 		},
 		"clone": {
@@ -515,28 +515,34 @@ func resourceVSphereVirtualMachineRead(d *schema.ResourceData, meta interface{})
 
 	// Read the virtual machine PCI passthrough devices
 	var pciDevs []string
+	var sharedPciDevs []string
 	for _, dev := range vprops.Config.Hardware.Device {
 		if pci, ok := dev.(*types.VirtualPCIPassthrough); ok {
-			if pciBacking, ok := pci.Backing.(*types.VirtualPCIPassthroughDeviceBackingInfo); ok {
-				devId := pciBacking.Id
+			switch t := pci.Backing.(type) {
+			case *types.VirtualPCIPassthroughDeviceBackingInfo:
+				devId := t.Id
 				pciDevs = append(pciDevs, devId)
-			} else {
-				if pciBacking, ok := pci.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo); ok {
-					err = d.Set("shared_pci_device_id", pciBacking.Vgpu)
-					if err != nil {
-						return err
-					}
-				} else {
-					log.Printf("[WARN] Ignoring VM %q VirtualPCIPassthrough device with backing type of %T",
-						vm.InventoryPath, pci.Backing)
-				}
+				log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T",
+					vm.InventoryPath, devId, pci.Backing)
+			case *types.VirtualPCIPassthroughVmiopBackingInfo:
+				dev := t.Vgpu
+				sharedPciDevs = append(sharedPciDevs, dev)
+				log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T",
+					vm.InventoryPath, dev, pci.Backing)
+			default:
+				log.Printf("[WARN] Ignoring VM %q VirtualPCIPassthrough device with backing type of %T",
+					vm.InventoryPath, pci.Backing)
 			}
 		}
 	}
 	err = d.Set("pci_device_id", pciDevs)
 	if err != nil {
 		return err
 	}
+	err = d.Set("shared_pci_device_id", sharedPciDevs)
+	if err != nil {
+		return err
+	}
 
 	// Perform pending device read operations.
 	devices := object.VirtualDeviceList(vprops.Config.Hardware.Device)
@@ -1675,6 +1681,17 @@ func resourceVSphereVirtualMachinePostDeployChanges(d *schema.ResourceData, meta
 		)
 	}
 	cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...)
+	// Shared PCI devices
+	devices, delta, err = virtualdevice.SharedPciPostCloneOperation(d, client, devices)
+	if err != nil {
+		return resourceVSphereVirtualMachineRollbackCreate(
+			d,
+			meta,
+			vm,
+			fmt.Errorf("error processing shared PCI device changes post-clone: %s", err),
+		)
+	}
+	cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...)
 	log.Printf("[DEBUG] %s: Final device list: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceListString(devices))
 	log.Printf("[DEBUG] %s: Final device change cfgSpec: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceChangeString(cfgSpec.DeviceChange))
 
@@ -1978,8 +1995,8 @@ func applyVirtualDevices(d *schema.ResourceData, c *govmomi.Client, l object.Vir
 		return nil, err
 	}
 	spec = virtualdevice.AppendDeviceChangeSpec(spec, delta...)
-	// Shared PCI passthrough device
-	l, delta, err = virtualdevice.SharedPciPassthroughApplyOperation(d, c, l)
+	// Shared PCI device
+	l, delta, err = virtualdevice.SharedPciApplyOperation(d, c, l)
 	if err != nil {
 		return nil, err
 	}

website/docs/r/virtual_machine.html.markdown

@@ -616,6 +616,8 @@ The following options are general virtual machine and provider workflow options:
 
 * `clone` - (Optional) When specified, the virtual machine will be created as a clone of a specified template. Optional customization options can be submitted for the resource. See [creating a virtual machine from a template](#creating-a-virtual-machine-from-a-template) for more information.
 
+~> **NOTE:** Cloning requires vCenter Server and is not supported on direct ESXi host connections.
+
 * `extra_config_reboot_required` - (Optional) Allow the virtual machine to be rebooted when a change to `extra_config` occurs. Default: `true`.
 
 * `custom_attributes` - (Optional) Map of custom attribute ids to attribute value strings to set for virtual machine. Please refer to the [`vsphere_custom_attributes`][docs-setting-custom-attributes] resource for more information on setting custom attributes.
@@ -661,14 +663,10 @@ The following options are general virtual machine and provider workflow options:
 
 * `network_interface` - (Required) A specification for a virtual NIC on the virtual machine. See [network interface options](#network-interface-options) for more information.
 
-* `pci_device_id` - (Optional) List of host PCI device IDs in which to create PCI passthroughs.
-
-* `shared_pci_device_id` - (Optional) A shared PCI device ID to create PCI passthrough.
-
-~> **NOTE:** Cloning requires vCenter Server and is not supported on direct ESXi host connections.
-
 * `ovf_deploy` - (Optional) When specified, the virtual machine will be deployed from the provided OVF/OVA template. See [creating a virtual machine from an OVF/OVA template](#creating-a-virtual-machine-from-an-ovf-ova-template) for more information.
 
+* `pci_device_id` - (Optional) List of host PCI device IDs in which to create PCI passthroughs.
+
 * `replace_trigger` - (Optional) Triggers replacement of resource whenever it changes.
 
 For example, `replace_trigger = sha256(format("%s-%s",data.template_file.cloud_init_metadata.rendered,data.template_file.cloud_init_userdata.rendered))` will fingerprint the changes in cloud-init metadata and userdata templates. This will enable a replacement of the resource whenever the dependant template renders a new configuration. (Forces a replacement.)
@@ -685,6 +683,16 @@ For example, `replace_trigger = sha256(format("%s-%s",data.template_file.cloud_i
 
 * `scsi_bus_sharing` - (Optional) The type of SCSI bus sharing for the virtual machine SCSI controller. One of `physicalSharing`, `virtualSharing`, and `noSharing`. Default: `noSharing`.
 
+* `shared_pci_device_id` - (Optional) List of shared PCI device(s) to create
+shared PCI passthrough.
+
+For example, attaching a vGPU to the virtual machine:
+`shared_pci_device_id = ["grid_a100d-40c"]`
+
+~> **NOTE:** The use of vGPU requires that the VM has memory reservation set equal
+to the amount of provisioned memory. This can be accomplished by leveraging
+the [`memory_reservation`](#memory_reservation) option.
+
 * `storage_policy_id` - (Optional) The ID of the storage policy to assign to the home directory of a virtual machine.
 
 * `tags` - (Optional) The IDs of any tags to attach to this resource. Please refer to the [`vsphere_tag`][docs-applying-tags] resource for more information on applying tags to virtual machine resources.