enable dual NIC support in transparent VLAN

cni/network/multitenancy.go

@@ -230,6 +230,7 @@ func (m *Multitenancy) GetAllNetworkContainers(
 		ifInfo.IPConfigs = append(ifInfo.IPConfigs, ipconfig)
 		ifInfo.Routes = routes
 		ifInfo.NICType = cns.InfraNIC
+		ifInfo.SkipDefaultRoutes = ncResponses[i].SkipDefaultRoutes
 
 		// assuming we only assign infra nics in this function
 		ipamResult.interfaceInfo[m.getInterfaceInfoKey(ifInfo.NICType, i)] = ifInfo

cni/network/network_linux.go

@@ -127,7 +127,7 @@ func platformInit(cniConfig *cni.NetworkConfig) {}
 
 // isDualNicFeatureSupported returns if the dual nic feature is supported. Currently it's only supported for windows hnsv2 path
 func (plugin *NetPlugin) isDualNicFeatureSupported(netNs string) bool {
-	return false
+	return true
 }
 
 func getOverlayGateway(_ *net.IPNet) (net.IP, error) {

cns/NetworkContainerContract.go

@@ -126,6 +126,7 @@ type CreateNetworkContainerRequest struct {
 	Routes                     []Route
 	AllowHostToNCCommunication bool
 	AllowNCToHostCommunication bool
+	SkipDefaultRoutes          bool
 	EndpointPolicies           []NetworkContainerRequestPolicies
 	NCStatus                   v1alpha.NCStatus
 	NetworkInterfaceInfo       NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code
@@ -161,10 +162,10 @@ func (req *CreateNetworkContainerRequest) String() string {
 	return fmt.Sprintf("CreateNetworkContainerRequest"+
 		"{Version: %s, NetworkContainerType: %s, NetworkContainerid: %s, PrimaryInterfaceIdentifier: %s, "+
 		"LocalIPConfiguration: %+v, IPConfiguration: %+v, SecondaryIPConfigs: %+v, MultitenancyInfo: %+v, "+
-		"AllowHostToNCCommunication: %t, AllowNCToHostCommunication: %t, NCStatus: %s, NetworkInterfaceInfo: %+v}",
+		"AllowHostToNCCommunication: %t, AllowNCToHostCommunication: %t, SkipDefaultRoutes: %t, NCStatus: %s, NetworkInterfaceInfo: %+v}",
 		req.Version, req.NetworkContainerType, req.NetworkContainerid, req.PrimaryInterfaceIdentifier, req.LocalIPConfiguration,
 		req.IPConfiguration, req.SecondaryIPConfigs, req.MultiTenancyInfo, req.AllowHostToNCCommunication, req.AllowNCToHostCommunication,
-		string(req.NCStatus), req.NetworkInterfaceInfo)
+		req.SkipDefaultRoutes, string(req.NCStatus), req.NetworkInterfaceInfo)
 }
 
 // NetworkContainerRequestPolicies - specifies policies associated with create network request
@@ -497,6 +498,7 @@ type GetNetworkContainerResponse struct {
 	Response                   Response
 	AllowHostToNCCommunication bool
 	AllowNCToHostCommunication bool
+	SkipDefaultRoutes          bool
 	NetworkInterfaceInfo       NetworkInterfaceInfo
 }
 

cns/restserver/util.go

@@ -530,6 +530,7 @@ func (service *HTTPRestService) getAllNetworkContainerResponses(
 			LocalIPConfiguration:       savedReq.LocalIPConfiguration,
 			AllowHostToNCCommunication: savedReq.AllowHostToNCCommunication,
 			AllowNCToHostCommunication: savedReq.AllowNCToHostCommunication,
+			SkipDefaultRoutes:          savedReq.SkipDefaultRoutes,
 			NetworkInterfaceInfo:       savedReq.NetworkInterfaceInfo,
 		}
 
@@ -831,6 +832,8 @@ func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus
 
 	primaryIPCfg := ncStatus.CreateNetworkContainerRequest.IPConfiguration
 
+	podIPInfo.SkipDefaultRoutes = ncStatus.CreateNetworkContainerRequest.SkipDefaultRoutes
+
 	podIPInfo.PodIPConfig = cns.IPSubnet{
 		IPAddress:    ipConfigStatus.IPAddress,
 		PrefixLength: primaryIPCfg.IPSubnet.PrefixLength,
@@ -933,6 +936,7 @@ func (service *HTTPRestService) handleGetNetworkContainers(w http.ResponseWriter
 			LocalIPConfiguration:       ncDetails.CreateNetworkContainerRequest.LocalIPConfiguration,
 			AllowHostToNCCommunication: ncDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication,
 			AllowNCToHostCommunication: ncDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication,
+			SkipDefaultRoutes:          ncDetails.CreateNetworkContainerRequest.SkipDefaultRoutes,
 		}
 		networkContainers[i] = getNcResp
 		i++

network/transparent_vlan_endpointclient_linux.go

@@ -400,14 +400,38 @@
 	return nil
 }
 
+// Set ARP proxy on the vlan interface to respond to ARP requests for the gateway IP
+func (client *TransparentVlanEndpointClient) setArpProxy(ifName string) error {
+	cmd := fmt.Sprintf("echo 1 > /proc/sys/net/ipv4/conf/%v/proxy_arp", ifName)
+	_, err := client.plClient.ExecuteRawCommand(cmd)
+	if err != nil {
+		logger.Error("Failed to set ARP proxy", zap.String("interface", ifName), zap.Error(err))
+	} else {
+		logger.Info("ARP proxy enabled", zap.String("interface", ifName))
+	}
+	return err
+}
+
 func (client *TransparentVlanEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error {
 	if err := client.AddSnatEndpointRules(); err != nil {
 		return errors.Wrap(err, "failed to add snat endpoint rules")
 	}
 	logger.Info("[transparent-vlan] Adding tunneling rules in vnet namespace")
 	err := ExecuteInNS(client.nsClient, client.vnetNSName, func() error {
-		return client.AddVnetRules(epInfo)
+		if err := client.AddVnetRules(epInfo); err != nil {
+			return err
+		}
+
+		// Set ARP proxy on vnet veth (inside vnet namespace)
+		logger.Info("calling setArpProxy for", zap.String("vnetVethName", client.vnetVethName))
+		if err := client.setArpProxy(client.vnetVethName); err != nil {
+			logger.Error("setArpProxy failed with", zap.Error(err))
+			return err
+		}
+
+		return nil
 	})
+
 	return err
 }
 
@@ -519,9 +543,19 @@
 		}
 	}
 
-	if err := client.addDefaultRoutes(client.containerVethName, 0); err != nil {
-		return errors.Wrap(err, "failed container ns add default routes")
+	if epInfo.SkipDefaultRoutes {
+		logger.Info("Skipping adding default routes in container ns as requested")
+		if err := client.addCustomRoutes(client.containerVethName, epInfo.Subnets[0].Gateway, epInfo.Subnets[0].Prefix, 0); err != nil {
+			return errors.Wrap(err, "failed container ns add custom routes")
+		}
+		return nil
+	} else {
+		logger.Info("Adding default routes in container ns")
+		if err := client.addDefaultRoutes(client.containerVethName, 0); err != nil {
+			return errors.Wrap(err, "failed container ns add default routes")
+		}
 	}
+
 	if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil {
 		return errors.Wrap(err, "failed container ns add default arp")
 	}
@@ -614,6 +648,38 @@
 	return nil
 }
 
+// Helper that creates routing rules for the current NS which direct packets
+// to the virtual gateway ip on linkToName device interface
+// Route 1: 169.254.2.1 dev <linkToName>
+// Route 2: default via 169.254.2.1 dev <linkToName>
+func (client *TransparentVlanEndpointClient) addCustomRoutes(linkToName string, gatewayIP net.IP, subnetCIDR net.IPNet, table int) error {
+	// Add route for virtualgwip (ip route add <gatewayIP> dev <linkToName>)
+	gWIP, gwNet, _ := net.ParseCIDR(gatewayIP.String() + "/32")
+	routeInfo := RouteInfo{
+		Dst:   *gwNet,
+		Scope: netlink.RT_SCOPE_LINK,
+		Table: table,
+	}
+	// Difference between interface name in addRoutes and DevName: in RouteInfo?
+	if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil {
+		return err
+	}
+
+	// Add subnet route (ip route add <subnetCIDR> via <gatewayIP> dev <linkToName>)
+	subnetPrefix, subnetIPNet, _ := net.ParseCIDR(subnetCIDR.String())
+	dstIP := net.IPNet{IP: subnetPrefix, Mask: subnetIPNet.Mask}
+	routeInfo = RouteInfo{
+		Dst:   dstIP,
+		Gw:    gWIP,
+		Table: table,
+	}
+
+	if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil {
+		return err
+	}
+	return nil
+}
+
 // Helper that creates arp entry for the current NS which maps the virtual
 // gateway (169.254.2.1) to destMac on a particular interfaceName
 // Example: (169.254.2.1) at 12:34:56:78:9a:bc [ether] PERM on <interfaceName>