WIP: Create underlay networks for simhost pNICs

This uses an existing bridge or creates a new one as
needed for simhost pNICs.

It does NOT currently rework the config for mgmt vmk
to hold the IP address assigned to the container.

Next steps:
* figure out how the mgmt NetConfig gets constructed
and associated with the pNIC. What parts of the config
passed in do we preserve vs discard?
* update container_host_system to inject IP into the
appropriate locations once available

simulator/container.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"os"
 	"os/exec"
 	"path"
@@ -38,6 +39,7 @@ var (
 
 const (
 	deleteWithContainer = "lifecycle=container"
+	createdByVcsim      = "createdBy=vcsim"
 )
 
 func init() {
@@ -97,6 +99,11 @@ func extractNameAndUid(containerName string) (name string, uid string, err error
 	return parts[0], parts[1], nil
 }
 
+func prefixToMask(prefix int) string {
+	mask := net.CIDRMask(prefix, 32)
+	return fmt.Sprintf("%d.%d.%d.%d", mask[0], mask[1], mask[2], mask[3])
+}
+
 type tarEntry struct {
 	header  *tar.Header
 	content []byte
@@ -248,6 +255,65 @@ func createVolume(volumeName string, labels []string, files []tarEntry) (string,
 	return uid, err
 }
 
+// createBridge creates a bridge network if one does not already exist
+// returns:
+//
+//	uid - string
+//	err - error or nil
+func createBridge(bridgeName string, labels ...string) (string, error) {
+
+	// {"CreatedAt":"2023-07-11 19:22:25.45027052 +0000 UTC","Driver":"bridge","ID":"fe52c7502c5d","IPv6":"false","Internal":"false","Labels":"goodbye=,hello=","Name":"testnet","Scope":"local"}
+	type bridgeNet struct {
+		CreatedAt string
+		Driver    string
+		ID        string
+		IPv6      string
+		Internal  string
+		Labels    string
+		Name      string
+		Scope     string
+	}
+
+	// if the underlay bridge already exists, return that
+	// we don't check for a specific label or similar so that it's possible to use a bridge created by other frameworks for composite testing
+	var bridge bridgeNet
+	cmd := exec.Command("docker", "network", "ls", "--format", "json", "-f", fmt.Sprintf("name=%s$", bridgeName))
+	out, err := cmd.Output()
+	if err != nil {
+		log.Printf("vcsim %s: %s", cmd.Args, err)
+	}
+
+	// unfortunately docker returns an empty string not an empty json doc
+	if len(out) != 0 {
+		err = json.Unmarshal(out, &bridge)
+		if err != nil {
+			log.Printf("vcsim %s: %s", cmd.Args, err)
+			return "", err
+		}
+
+		return bridge.ID, nil
+	}
+
+	run := []string{"network", "create", "--label", createdByVcsim}
+	for i := range labels {
+		run = append(run, "--label", labels[i])
+	}
+	run = append(run, bridgeName)
+
+	cmd = exec.Command("docker", run...)
+	out, err = cmd.Output()
+	if err != nil {
+		log.Printf("vcsim %s: %s", cmd.Args, err)
+		return "", err
+	}
+
+	// the ID returned in network ls is only 12 characters, so normalize to that
+	id := string(out[0:12])
+	log.Printf("vcsim %s: id=%s", cmd.Args, id)
+
+	return id, nil
+}
+
 // create
 //   - name - pretty name, eg. vm name
 //   - id - uuid or similar - this is merged into container name rather than dictating containerID

simulator/container_host_system.go

@@ -46,46 +46,25 @@ type simHost struct {
 	c    *container
 }
 
-// createSimulationHost inspects the provided HostSystem and creates a simHost binding for it if
-// the vm.Config.ExtraConfig set contains a key "RUN.container".
-// If the ExtraConfig set does not contain that key, this returns nil.
-// Methods on the simHost type are written to check for nil object so the return from this call can be blindly
-// assigned and invoked without the caller caring about whether a binding for a backing container was warranted.
-func createSimulationHost(ctx *Context, host *HostSystem) (*simHost, error) {
-	sh := &simHost{
-		host: host,
-	}
-
-	advOpts := ctx.Map.Get(host.ConfigManager.AdvancedOption.Reference()).(*OptionManager)
-	fault := advOpts.QueryOptions(&types.QueryOptions{Name: "RUN.container"}).(*methods.QueryOptionsBody).Fault()
-	if fault != nil {
-		if _, ok := fault.VimFault().(*types.InvalidName); ok {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("errror retrieving container backing from host config manager: %+v", fault.VimFault())
-	}
-
-	// assemble env
-	var dockerEnv []string
+// createSimHostMounts iterates over the provide filesystem mount info, creating docker volumes. It does _not_ delete volumes
+// already created if creation of one fails.
+// Returns:
+// volume mounts: mount options suitable to pass directly to docker
+// exec commands: a set of commands to run in the sim host after creation
+// error: if construction of the above outputs fails
+func createSimHostMounts(ctx *Context, containerName string, mounts []types.HostFileSystemMountInfo) ([]string, [][]string, error) {
 	var dockerVol []string
-	var dockerNet []string
 	var symlinkCmds [][]string
 
-	var err error
-
-	hName := host.Summary.Config.Name
-	hUuid := host.Summary.Hardware.Uuid
-	containerName := constructContainerName(hName, hUuid)
-
-	for i := range host.Config.FileSystemVolume.MountInfo {
-		info := &host.Config.FileSystemVolume.MountInfo[i]
+	for i := range mounts {
+		info := &mounts[i]
 		name := info.Volume.GetHostFileSystemVolume().Name
 
 		// NOTE: if we ever need persistence cross-invocation we can look at encoding the disk info as a label
 		labels := []string{"name=" + name, "container=" + containerName, deleteWithContainer}
 		dockerUuid, err := createVolume("", labels, nil)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		uuid := volumeIDtoHostVolumeUUID(dockerUuid)
@@ -130,61 +109,156 @@ func createSimulationHost(ctx *Context, host *HostSystem) (*simHost, error) {
 		}
 
 		dockerVol = append(dockerVol, fmt.Sprintf("%s:/vmfs/volumes/%s:%s", dockerUuid, uuid, opt))
+
+		// create symlinks from /vmfs/volumes/ for the Volume Name - the direct mount (path) is only the uuid
+		// ? can we do this via a script in the ESX image instead of via exec?
+		// ? are the volume names exposed in any manner inside the host? They must be because these mounts exist but where does that come from? Chicken and egg problem? ConfigStore?
 		symlinkCmds = append(symlinkCmds, []string{"ln", "-s", fmt.Sprintf("/vmfs/volumes/%s", uuid), fmt.Sprintf("/vmfs/volumes/%s", name)})
 		if strings.HasPrefix(name, "OSDATA") {
 			symlinkCmds = append(symlinkCmds, []string{"mkdir", "-p", "/var/lib/vmware"})
 			symlinkCmds = append(symlinkCmds, []string{"ln", "-s", fmt.Sprintf("/vmfs/volumes/%s", uuid), "/var/lib/vmware/osdata"})
 		}
 	}
 
+	return dockerVol, symlinkCmds, nil
+}
+
+// createSimHostNetworks creates the networks for the host if not already created. Because we expect multiple hosts on the same network to act as a cluster
+// it's likely that only the first host will create networks.
+// This includes:
+// * bridge network per-pNIC
+// * bridge network per-DVS
+//
+// Returns:
+// * array of networks to attach to
+// * array of commands to run
+// * error
+func createSimHostNetworks(ctx *Context, containerName string, networkInfo *types.HostNetworkInfo, advOpts *OptionManager) ([]string, [][]string, error) {
+	var dockerNet []string
+	var cmds [][]string
+
+	existingNets := make(map[string]string)
+
 	// a pnic does not have an IP so this is purely a connectivity statement, not a network identity, however this is not how docker works
 	// so we're going to end up with a veth (our pnic) that does have an IP assigned.
-	// For now we're going to simply ignore that IP. //TODO: figure out whether we _need_ to do something with it.
-	for i := range host.Config.Network.Pnic {
-		pnicName := host.Config.Network.Pnic[i].Device
+	// For now we're going to simply ignore that IP. //TODO: figure out whether we _need_ to do something with it at this point
+	for i := range networkInfo.Pnic {
+		pnicName := networkInfo.Pnic[i].Device
+
+		bridge := getPnicUnderlay(advOpts, pnicName)
 
-		queryRes := advOpts.QueryOptions(&types.QueryOptions{Name: advOptPrefixPnicToUnderlayPrefix + pnicName}).(*methods.QueryOptionsBody).Res
-		bridge := queryRes.Returnval[0].GetOptionValue().Value.(string)
+		if pnic, attached := existingNets[bridge]; attached {
+			return nil, nil, fmt.Errorf("cannot attach multiple pNICs to the same underlay: %s and %s both attempting to connect to %s for %s", pnic, pnicName, bridge, containerName)
+		}
+
+		_, err := createBridge(bridge)
+		if err != nil {
+			return nil, nil, err
+		}
 
 		dockerNet = append(dockerNet, bridge)
+		existingNets[bridge] = pnicName
 	}
 
-	// determine the management
-	// TODO: add in vSwitches if we know them at this point
-	mgmtSwitch := ""
-	vmNet := ""
-	for _, vswitch := range host.Config.Network.Vswitch {
-		vmnic := vswitch.Spec.Policy.NicTeaming.NicOrder.ActiveNic[0]
-		switchName := vswitch.Name
+	return dockerNet, cmds, nil
+}
+
+func getPnicUnderlay(advOpts *OptionManager, pnicName string) string {
+	queryRes := advOpts.QueryOptions(&types.QueryOptions{Name: advOptPrefixPnicToUnderlayPrefix + pnicName}).(*methods.QueryOptionsBody).Res
+	return queryRes.Returnval[0].GetOptionValue().Value.(string)
+}
 
-		for _, pg := range vswitch.Portgroup {
+// createSimulationHostcreates a simHost binding if the host.ConfigManager.AdvancedOption set contains a key "RUN.container".
+// If the set does not contain that key, this returns nil.
+// Methods on the simHost type are written to check for nil object so the return from this call can be blindly
+// assigned and invoked without the caller caring about whether a binding for a backing container was warranted.
+//
+// The created simhost is based off of the details of the supplied host system.
+// VMFS locations are created based on FileSystemMountInfo
+// Bridge networks are created to simulate underlay networks - one per pNIC. You cannot connect two pNICs to the same underlay.
+//
+// On Network connectivity - initially this is using docker network constructs. This means we cannot easily use nested "ip netns" so we cannot
+// have a perfect representation of the ESX structure: pnic(veth)->vswtich(bridge)->{vmk,vnic}(veth)
+// Instead we have the following:
+// * bridge network per underlay - everything connects directly to the underlay
+// * VMs/CRXs connect to the underlay dictated by the Uplink pNIC attached to their vSwitch
+// * hostd vmknic gets the "host" container IP - we don't currently support multiple vmknics with different IPs
+// * no support for mocking VLANs
+func createSimulationHost(ctx *Context, host *HostSystem) (*simHost, error) {
+	sh := &simHost{
+		host: host,
+	}
 
+	advOpts := ctx.Map.Get(host.ConfigManager.AdvancedOption.Reference()).(*OptionManager)
+	fault := advOpts.QueryOptions(&types.QueryOptions{Name: "RUN.container"}).(*methods.QueryOptionsBody).Fault()
+	if fault != nil {
+		if _, ok := fault.VimFault().(*types.InvalidName); ok {
+			return nil, nil
 		}
+		return nil, fmt.Errorf("errror retrieving container backing from host config manager: %+v", fault.VimFault())
+	}
+
+	// assemble env
+	var dockerEnv []string
 
+	var execCmds [][]string
+
+	var err error
+
+	hName := host.Summary.Config.Name
+	hUuid := host.Summary.Hardware.Uuid
+	containerName := constructContainerName(hName, hUuid)
+
+	// create volumes and mounts
+	dockerVol, volCmds, err := createSimHostMounts(ctx, containerName, host.Config.FileSystemVolume.MountInfo)
+	if err != nil {
+		return nil, err
 	}
+	execCmds = append(execCmds, volCmds...)
 
-	// if there's a DVS that doesn't have a bridge, create the bridge
+	// create networks
+	dockerNet, netCmds, err := createSimHostNetworks(ctx, containerName, host.Config.Network, advOpts)
+	if err != nil {
+		return nil, err
+	}
+	execCmds = append(execCmds, netCmds...)
 
+	// create the container
 	sh.c, err = create(ctx, hName, hUuid, dockerNet, dockerVol, nil, dockerEnv, "alpine", []string{"sleep", "infinity"})
 	if err != nil {
 		return nil, err
 	}
 
+	// start the container
 	err = sh.c.start(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	// create symlinks from /vmfs/volumes/ for the Volume Name - the direct mount (path) is only the uuid
-	// ? can we do this via a script in the ESX image? are the volume names exposed in any manner instead the host? They must be because these mounts exist
-	// but where does that come from? Chicken and egg problem? ConfigStore?
-	for _, symlink := range symlinkCmds {
-		_, err := sh.c.exec(ctx, symlink)
+	// run post-creation steps
+	for _, cmd := range execCmds {
+		_, err := sh.c.exec(ctx, cmd)
 		if err != nil {
 			return nil, err
 		}
 	}
 
+	_, detail, err := sh.c.inspect()
+
+	for i := range host.Config.Network.Pnic {
+		pnic := &host.Config.Network.Pnic[i]
+		bridge := getPnicUnderlay(advOpts, pnic.Device)
+		settings := detail.NetworkSettings.Networks[bridge]
+
+		// it doesn't really make sense at an ESX level to set this information as IP bindings are associated with
+		// vnics (VMs) or vmknics (daemons such as hostd).
+		// However it's a useful location to stash this info in a manner that can be retrieved at a later date.
+		pnic.Spec.Ip.IpAddress = settings.IPAddress
+		pnic.Spec.Ip.SubnetMask = prefixToMask(settings.IPPrefixLen)
+
+		pnic.Mac = settings.MacAddress
+	}
+
 	// TODO iterate over the following to update the IPs and MACs:
 	// 1. host.Config.Network.Pnic
 	// 2. host.Config.Network.Vnic

simulator/container_host_system_test.go

@@ -172,7 +172,7 @@ func TestHostContainerBacking(t *testing.T) {
 	ctx := SpoofContext()
 
 	hs := NewHostSystem(esx.HostSystem)
-	hs.configureContainerBacking(ctx, "alpine", defaultSimVolumes)
+	hs.configureContainerBacking(ctx, "alpine", defaultSimVolumes, "vcsim-mgmt-underlay")
 
 	hs.configure(ctx, types.HostConnectSpec{}, true)