WIP: towards underlay definition

simulator/container_host_system.go

@@ -36,6 +36,11 @@ const GB = 1000 * MB
 const TB = 1000 * GB
 const PB = 1000 * TB
 
+const (
+	advOptPrefixPnicToUnderlayPrefix = "RUN.underlay."
+	advOptContainerBackingImage      = "RUN.container"
+)
+
 type simHost struct {
 	host *HostSystem
 	c    *container
@@ -132,14 +137,31 @@ func createSimulationHost(ctx *Context, host *HostSystem) (*simHost, error) {
 		}
 	}
 
-	// TODO: extract the underlay's from a topology config
-	// create a bridge for each broadcast domain a pnic is connected to
-	dockerNet = append(dockerNet, defaultUnderlayBridgeName)
+	// a pnic does not have an IP so this is purely a connectivity statement, not a network identity, however this is not how docker works
+	// so we're going to end up with a veth (our pnic) that does have an IP assigned.
+	// For now we're going to simply ignore that IP. //TODO: figure out whether we _need_ to do something with it.
+	for i := range host.Config.Network.Pnic {
+		pnicName := host.Config.Network.Pnic[i].Device
+
+		queryRes := advOpts.QueryOptions(&types.QueryOptions{Name: advOptPrefixPnicToUnderlayPrefix + pnicName}).(*methods.QueryOptionsBody).Res
+		bridge := queryRes.Returnval[0].GetOptionValue().Value.(string)
+
+		dockerNet = append(dockerNet, bridge)
+	}
 
+	// determine the management
 	// TODO: add in vSwitches if we know them at this point
+	mgmtSwitch := ""
+	vmNet := ""
+	for _, vswitch := range host.Config.Network.Vswitch {
+		vmnic := vswitch.Spec.Policy.NicTeaming.NicOrder.ActiveNic[0]
+		switchName := vswitch.Name
 
-	// - a pnic does not have an IP so this is purely a connectivity statement, not a network identity
-	// ? how is this underlay topology expressed? Initially we can assume a flat topology with all hosts on the same broadcast domain
+		for _, pg := range vswitch.Portgroup {
+
+		}
+
+	}
 
 	// if there's a DVS that doesn't have a bridge, create the bridge
 
@@ -163,6 +185,11 @@ func createSimulationHost(ctx *Context, host *HostSystem) (*simHost, error) {
 		}
 	}
 
+	// TODO iterate over the following to update the IPs and MACs:
+	// 1. host.Config.Network.Pnic
+	// 2. host.Config.Network.Vnic
+	// 3. host.Config.VirtualNicManagerInfo.NetConfig
+
 	return sh, nil
 }
 

simulator/esx/host_config_filesystemvolume.go

@@ -33,6 +33,7 @@ const PB = 1000 * TB
 // HostConfigInfo is the default template for the HostSystem config property.
 // Capture method:
 // govc object.collect -s -dump HostSystem:ha-host config.fileSystemVolume
+//   - slightly modified for uuids and DiskName
 var HostFileSystemVolumeInfo = types.HostFileSystemVolumeInfo{
 	VolumeTypeList: []string{"VMFS", "NFS", "NFS41", "vsan", "VVOL", "VFFS", "OTHER", "PMEM"},
 	MountInfo: []types.HostFileSystemMountInfo{

simulator/host_system.go

@@ -86,7 +86,8 @@ func NewHostSystem(host mo.HostSystem) *HostSystem {
 	deepCopy(esx.AdvancedOptions, &cfg.Option)
 
 	// add a supported option to the AdvancedOption manager
-	simOption := types.OptionDef{ElementDescription: types.ElementDescription{Key: "RUN.container"}}
+	simOption := types.OptionDef{ElementDescription: types.ElementDescription{Key: advOptContainerBackingImage}}
+	// TODO: how do we enter patterns here? Or should we stick to a list in the value?
 	hs.Config.OptionDef = append(hs.Config.OptionDef, simOption)
 
 	config := []struct {
@@ -127,9 +128,6 @@ func (h *HostSystem) configure(ctx *Context, spec types.HostConnectSpec, connect
 	h.Summary.Hardware.Uuid = id
 	h.Hardware.SystemInfo.Uuid = id
 
-	// bind to a simulation host with container backing if specified by options
-	// TODO: decide whether to require this at host creation or allow binding during a reconfigure
-	// TODO: handle the error return
 	var err error
 	h.sh, err = createSimulationHost(ctx, h)
 	if err != nil {
@@ -140,11 +138,12 @@ func (h *HostSystem) configure(ctx *Context, spec types.HostConnectSpec, connect
 // configureContainerBacking sets up _this_ host for simulation using a container backing.
 // Args:
 //
-//	image - the container image with which to simulate the host
-//	mounts - array of mount info that should be translated into /vmfs/volumes/... mounts backed by container volumes
-func (h *HostSystem) configureContainerBacking(ctx *Context, image string, mounts []types.HostFileSystemMountInfo) {
+//		image - the container image with which to simulate the host
+//		mounts - array of mount info that should be translated into /vmfs/volumes/... mounts backed by container volumes
+//	 	networks - names of bridges to use for underlays. Will create a pNIC for each. The first will be treated as the management network.
+func (h *HostSystem) configureContainerBacking(ctx *Context, image string, mounts []types.HostFileSystemMountInfo, networks []string) {
 	option := &types.OptionValue{
-		Key:   "RUN.container",
+		Key:   advOptContainerBackingImage,
 		Value: image,
 	}
 
@@ -161,6 +160,201 @@ func (h *HostSystem) configureContainerBacking(ctx *Context, image string, mount
 			MountInfo:      mounts,
 		}
 	}
+
+	// force at least a management network
+	if len(networks) == 0 {
+		networks = []string{defaultUnderlayBridgeName}
+	}
+
+	// purge pNICs from the template - it makes no sense to keep them for a sim host
+	h.Config.Network.Pnic = make([]types.PhysicalNic, len(networks))
+
+	for i, net := range networks {
+		name := fmt.Sprintf("vmnic%d", i)
+
+		// we don't have a natural field for annotating which pNIC is connected to which network, so stash it in an adv option.
+		option := &types.OptionValue{
+			Key:   advOptPrefixPnicToUnderlayPrefix + name,
+			Value: net,
+		}
+		fault = advOpts.UpdateOptions(&types.UpdateOptions{ChangedValue: []types.BaseOptionValue{option}}).Fault()
+		if fault != nil {
+			panic(fault)
+		}
+
+		h.Config.Network.Pnic[i] = types.PhysicalNic{
+			Key:             "key-vim.host.PhysicalNic-" + name,
+			Device:          name,
+			Pci:             fmt.Sprintf("0000:%2d:00.0", i+1),
+			Driver:          "vcsim-bridge",
+			DriverVersion:   "1.2.10.0",
+			FirmwareVersion: "1.57, 0x80000185",
+			LinkSpeed: &types.PhysicalNicLinkInfo{
+				SpeedMb: 10000,
+				Duplex:  true,
+			},
+			ValidLinkSpecification: []types.PhysicalNicLinkInfo{
+				{
+					SpeedMb: 10000,
+					Duplex:  true,
+				},
+			},
+			Spec: types.PhysicalNicSpec{
+				Ip:                            &types.HostIpConfig{},
+				LinkSpeed:                     (*types.PhysicalNicLinkInfo)(nil),
+				EnableEnhancedNetworkingStack: types.NewBool(false),
+				EnsInterruptEnabled:           types.NewBool(false),
+			},
+			WakeOnLanSupported: false,
+			Mac:                "00:00:00:00:00:00",
+			FcoeConfiguration: &types.FcoeConfig{
+				PriorityClass: 3,
+				SourceMac:     "00:00:00:00:00:00",
+				VlanRange: []types.FcoeConfigVlanRange{
+					{},
+				},
+				Capabilities: types.FcoeConfigFcoeCapabilities{},
+				FcoeActive:   false,
+			},
+			VmDirectPathGen2Supported:             types.NewBool(false),
+			VmDirectPathGen2SupportedMode:         "",
+			ResourcePoolSchedulerAllowed:          types.NewBool(false),
+			ResourcePoolSchedulerDisallowedReason: nil,
+			AutoNegotiateSupported:                types.NewBool(true),
+			EnhancedNetworkingStackSupported:      types.NewBool(false),
+			EnsInterruptSupported:                 types.NewBool(false),
+			RdmaDevice:                            "",
+			DpuId:                                 "",
+		}
+
+		// there's too much repetition of state in host.Config.Network for it to be viable to use this directly
+		// as the config reference - it's a serialization of an interlinked data structure but without references.
+		// Outcomes:
+		//	bridge - created for all vSwitch (host prefixed) and dvSwitch (not host scoped)
+		//  host connection to bridge - if vNIC has port of type "host", connect to vSwitch/dvSwitch bridge
+		    // TODO: validate this interpretation of "host" as port type.
+		//  host is connected to mgmt network at minimum and "management" netConfig uses vNIC connected to mgmt net
+		//  vNICs have IPs and MACs that correspond to the veths in the host container
+		//  ? pNIC per vSwitch with uplink - this would be purely for data consistency in returned info, not for sim connectivity
+
+		// Approach:
+		// Implement API methods for creating switches and vmknics. If we do this then it's the same path for the API live as used
+		// to process the template during host creation.
+		// 1. implement AddvSwitch method, and invoke it for each vswitch in the template - this creates bridge
+		// 2. implement AddVmkNic method, and invoke it for each in the template - this calls "connect"
+
+		// vNIC connectivity is dictated within host by:
+		// In host:
+		// * on the same portgroup
+		// * on different portgroups with the same VLANID, on the same vSwitch
+		// Across hosts:
+		// * route to shared underlay, same VLANID
+		// ? delta is "does the vSwitch have an uplink?"
+		// ! cannot reconnect all containers to a different bridge if uplink is added as that will cause re-IP.
+		// > could mess around with ip tables rules to BLOCK comms and have all vNICs on same VLAN connect to the same bridge, but orchestration pain.
+		// ? have pnic as bridge, hosts connected bridges for pnic and vswitch. pnic added as uplink constructs iptables forwarding rules from vswitch bridge?
+
+
+		// !!!! uncertain of approach!! Draw it out in Miro.
+
+		// 2023-04-28
+		// * Create a bridge network per underlay (treat isolated logical networks as a separate underlay).
+		// * connect VMs/CRXs directly to the underlay bridge. Decide which underlays to connect to based on vNIC->vSwitch->Uplink.
+		// * Ignore VLAN for now.
+		// * Hosts connect to mgmt network - don't need to do anything else.
+		// * need a way to allow infravisor tests to connect hosts to an "overlay" network
+
+		// underlay - vswitch - vlanID
+
+		// first underlay is the management network
+		if i == 0 {
+			// 1. determine which vmk has the management service enabled
+			// 2. determine which vSwitch the mgmt vmk is connected to
+			// 3. ensure that vswitch has the mgmt vmnic as its uplink
+			mgmtVmkNic := ""
+
+			// purge the IPs and MACs that are going to be backed by veths as the info will be incorrect
+			for cfgIdx := range h.Config.Network.VirtualNicManagerInfo.NetConfig {
+				config := &h.Config.Network.VirtualNicManagerInfo.NetConfig[cfgIdx]
+				for candidateIdx := range config.CandidateVnic {
+					candidate := &config.CandidateVnic[candidateIdx]
+					candidate.Spec.IP.IpAddress = "0.0.0.0"
+					candidate.Spec.IP.SubnetMask = "0.0.0.0"
+					candidate.Spec.Mac = "00:00:00:00:00:00"
+
+					if config.NicType == "management" && candidate.Key == config.SelectedVnic[0] {
+						mgmtVmkNic = candidate.Device
+					}
+				}
+			}
+
+			if mgmtVmkNic == "" {
+				panic("expected ESX to have a management vmknic")
+			}
+
+			for vnicIdx := range h.Config.Network.VirtualNicManagerInfo.Vnic {
+				vnic := &h.Config.Network.VirtualNicManagerInfo.Vnic[vnicIdx]
+				vnic.Spec.IP.IpPAddress = "0.0.0.0"
+				vnic.Spec.IP.SubnetMask = "0.0.0.0"
+				vnic.Spec.Mac = "00:00:00:00:00:00"
+
+					if config.NicType == "management" && candidate.Key == config.SelectedVnic[0] {
+						mgmtVmkNic = candidate.Device
+					}
+				}
+			}
+
+
+			VirtualNicManagerInfo: &types.HostVirtualNicManagerInfo{
+				NetConfig: []types.VirtualNicManagerNetConfig{
+					{
+						NicType:            "management",
+						MultiSelectAllowed: true,
+						CandidateVnic:      []types.HostVirtualNic{
+							{
+								Device:    mgmtVmkNic,
+								Key:       "management.key-vim.host.VirtualNic-vmk1",
+								Portgroup: "Management Network",
+								Spec:      types.HostVirtualNicSpec{
+									Ip: &types.HostIpConfig{
+										Dhcp:       false,
+										IpAddress:  "172.27.27.2",
+										SubnetMask: "255.255.0.0",
+										IpV6Config: &types.HostIpConfigIpV6AddressConfiguration{
+											IpV6Address: []types.HostIpConfigIpV6Address{
+												{
+													IpAddress:    "fe80::250:56ff:fe65:d28a",
+													PrefixLength: 64,
+													Origin:       "other",
+													DadState:     "preferred",
+													Lifetime:     (*time.Time)(nil),
+													Operation:    "",
+												},
+											},
+											AutoConfigurationEnabled: types.NewBool(false),
+											DhcpV6Enabled:            types.NewBool(false),
+										},
+									},
+									Mac:                    "00:50:56:65:d2:8a",
+									DistributedVirtualPort: (*types.DistributedVirtualSwitchPortConnection)(nil),
+									Portgroup:              "Management Network",
+									Mtu:                    1500,
+									TsoEnabled:             types.NewBool(true),
+									NetStackInstanceKey:    "defaultTcpipStack",
+									OpaqueNetwork:          (*types.HostVirtualNicOpaqueNetworkSpec)(nil),
+									ExternalId:             "",
+									PinnedPnic:             "",
+									IpRouteSpec:            (*types.HostVirtualNicIpRouteSpec)(nil),
+									SystemOwned:            types.NewBool(false),
+									DpuId:                  "",
+								},
+								Port: "",
+							},
+						},
+						SelectedVnic: []string{"management.key-vim.host.VirtualNic-vmk1"},
+					},
+		}
+	}
 }
 
 func (h *HostSystem) event() types.HostEvent {