Skip to content

Commit

Permalink
Add support for cni none on windows and initial windows-bgp backend
Browse files Browse the repository at this point in the history 
Signed-off-by: Roberto Bonafiglia <[email protected]>
  • Loading branch information
@rbrtbnfgl
rbrtbnfgl committed Jul 10, 2023
1 parent 356adec commit 3c0847d
Show file tree
Hide file tree
Showing 5 changed files with 147 additions and 56 deletions.
2 changes: 2 additions & 0 deletions Dockerfile.windows
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ RUN unzip calico-windows-${CALICO_VERSION}.zip
RUN mv CalicoWindows/calico-node.exe rancher/
RUN mv CalicoWindows/cni/calico.exe rancher/
RUN mv CalicoWindows/cni/calico-ipam.exe rancher/
RUN mv CalicoWindows/confd confd/

FROM scratch AS windows-runtime
COPY --from=containerd /usr/local/bin/*.exe /bin/
COPY --from=windows-runtime-collect ./rancher/* /bin/
COPY --from=windows-runtime-collect ./confd/ /bin/confd
72 changes: 61 additions & 11 deletions pkg/pebinaryexecutor/pebinary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/Microsoft/hcsshim"
"github.com/Microsoft/hcsshim/hcn"
"github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io"
"github.com/k3s-io/k3s/pkg/cli/cmds"
daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/daemons/executor"
Expand All @@ -24,9 +25,11 @@ import (
"github.com/rancher/rke2/pkg/logging"
win "github.com/rancher/rke2/pkg/windows"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
)

var (
ssldirs = []string{
Expand All @@ -50,14 +53,22 @@ type PEBinaryConfig struct {
KubeConfigKubeProxy string
DisableETCD bool
IsServer bool
cni win.Calico
CNI string
CNIConfig win.Calico
}

type CloudProviderConfig struct {
Name string
Path string
}

const (
CNINone = "none"
CNICalico = "calico"
CNICilium = "cilium"
CNICanal = "canal"
)

// Bootstrap prepares the binary executor to run components by setting the system default registry
// and staging the kubelet and containerd binaries. On servers, it also ensures that manifests are
// copied in to place and in sync with the system configuration.
Expand Down Expand Up @@ -94,10 +105,22 @@ func (p *PEBinaryConfig) Bootstrap(ctx context.Context, nodeConfig *daemonconfig

restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)

if err := p.cni.Setup(ctx, nodeConfig, restConfig, p.DataDir); err != nil {
p.CNI, err = getCniType(restConfig)
if err != nil {
return err
}

switch p.CNI {
case "", CNICalico:
if err := p.CNIConfig.Setup(ctx, nodeConfig, restConfig, p.DataDir); err != nil {
return err
}
case CNINone:
logrus.Info("Skipping CNI setup")
default:
logrus.Fatal("Unsupported CNI: ", p.CNI)
}

// required to initialize KubeProxy
p.KubeConfigKubeProxy = nodeConfig.AgentConfig.KubeConfigKubeProxy

Expand Down Expand Up @@ -135,11 +158,13 @@ func (p *PEBinaryConfig) Kubelet(ctx context.Context, args []string) error {
go func() {
for {
cniCtx, cancel := context.WithCancel(ctx)
go func() {
if err := p.cni.Start(cniCtx); err != nil {
logrus.Errorf("error in cni start: %s", err)
}
}()
if p.CNI != CNINone {
go func() {
if err := p.CNIConfig.Start(cniCtx); err != nil {
logrus.Errorf("error in cni start: %s", err)
}
}()
}

cmd := exec.CommandContext(ctx, p.KubeletPath, cleanArgs...)
cmd.Stdout = logOut
Expand All @@ -156,9 +181,13 @@ func (p *PEBinaryConfig) Kubelet(ctx context.Context, args []string) error {

// KubeProxy starts the kubeproxy in a subprocess with watching goroutine.
func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error {
if p.CNI == CNINone {
return nil
}

extraArgs := map[string]string{
"network-name": p.cni.CNICfg.OverlayNetName,
"bind-address": p.cni.CNICfg.IP,
"network-name": p.CNIConfig.CNICfg.OverlayNetName,
"bind-address": p.CNIConfig.CNICfg.IP,
}

if err := hcn.DSRSupported(); err == nil {
Expand All @@ -167,7 +196,7 @@ func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error {
extraArgs["enable-dsr"] = "true"
}

if p.cni.CNICfg.Name == "Calico" {
if p.CNIConfig.CNICfg.Name == "Calico" {
var vip string
for range time.Tick(time.Second * 5) {
endpoint, err := hcsshim.GetHNSEndpointByName("Calico_ep")
Expand Down Expand Up @@ -258,6 +287,27 @@ func getArgs(argsMap map[string]string) []string {
return args
}

func getCniType(restConfig *rest.Config) (string, error) {
hc, err := helm.NewFactoryFromConfig(restConfig)
if err != nil {
return "", err
}
hl, err := hc.Helm().V1().HelmChart().List(metav1.NamespaceSystem, metav1.ListOptions{})
if err != nil {
return "", err
}
for _, h := range hl.Items {
if h.Name == win.CalicoChart {
return CNICalico, nil
} else if h.Name == "rke2-cilium" {
return CNICilium, nil
} else if h.Name == "rke2-canal" {
return CNICanal, nil
}
}
return CNINone, nil
}

// setWindowsAgentSpecificSettings configures the correct paths needed for Windows
func setWindowsAgentSpecificSettings(dataDir string, nodeConfig *daemonconfig.Node) {
nodeConfig.AgentConfig.CNIBinDir = filepath.Join("c:\\", dataDir, "bin")
Expand Down
1 change: 1 addition & 0 deletions pkg/rke2/rke2_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,5 +69,6 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto
KubeletPath: cfg.KubeletPath,
DisableETCD: clx.Bool("disable-etcd"),
IsServer: isServer,
CNI: "",
}, nil
}
97 changes: 69 additions & 28 deletions pkg/windows/calico.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/k3s-io/k3s/pkg/version"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
opv1 "github.com/tigera/operator/api/v1"
authv1 "k8s.io/api/authentication/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -104,7 +105,11 @@ users:
"Value": {
"Type": "SDNROUTE",
"DestinationPrefix": "{{ .ServiceCIDR }}",
"NeedEncap": true
{{- if eq .Mode "vxlan" }}
"NeedEncap": true
{{- else }}
"NeedEncap": false
{{- end }}
}
}
]
Expand Down Expand Up @@ -265,6 +270,9 @@ func (c *Calico) Start(ctx context.Context) error {
break
}
go startFelix(ctx, c.CNICfg)
if c.CNICfg.Mode == "windows-bgp" {
go startConfd(ctx, c.CNICfg)
}

return nil
}
Expand All @@ -280,22 +288,22 @@ func (c *Calico) generateCalicoNetworks() error {
// 2 - c.CNICfg.Interface which set if NodeAddressAutodetection is set (Calico HelmChart)
// 3 - nodeIP if defined
// 4 - None of the above. In that case, by default the interface with the default route is picked
vxlanAdapter := os.Getenv("VXLAN_ADAPTER")
if vxlanAdapter == "" {
networkAdapter := os.Getenv("VXLAN_ADAPTER")
if networkAdapter == "" {
if c.CNICfg.Interface != "" {
vxlanAdapter = c.CNICfg.Interface
networkAdapter = c.CNICfg.Interface
}

if c.CNICfg.Interface == "" && c.CNICfg.IP != "" {
iFace, err := findInterface(c.CNICfg.IP)
if err != nil {
return err
}
vxlanAdapter = iFace
networkAdapter = iFace
}
}

mgmt, err := createHnsNetwork(c.CNICfg.Mode, vxlanAdapter)
mgmt, err := createHnsNetwork(c.CNICfg.Mode, networkAdapter)
if err != nil {
return err
}
Expand Down Expand Up @@ -338,39 +346,72 @@ func (c *Calico) overrideCalicoConfigByHelm(restConfig *rest.Config) error {
}
logrus.Debugf("calico override found: %s\n", string(b))
if nodeV4 := overrides.Installation.CalicoNetwork.NodeAddressAutodetectionV4; nodeV4 != nil {
IPAutoDetectionMethod, err := nodeAddressAutodetection(*nodeV4)
c.CNICfg.IPAutoDetectionMethod, c.CNICfg.Interface, err = findCalicoInterface(nodeV4)
if err != nil {
return err
}
logrus.Debugf("this is IPAutoDetectionMethod: %s", IPAutoDetectionMethod)
c.CNICfg.IPAutoDetectionMethod = IPAutoDetectionMethod

var calicoInterface string
if strings.Contains(IPAutoDetectionMethod, "cidrs") {
calicoInterface, err = findInterfaceCIDR(nodeV4.CIDRS)
if err != nil {
return err
}
}
if bgpEnabled := overrides.Installation.CalicoNetwork.BGP; bgpEnabled != nil {
if *bgpEnabled == opv1.BGPEnabled {
c.CNICfg.Mode = "windows-bgp"
}
}
return nil
}

if strings.Contains(IPAutoDetectionMethod, "interface") {
calicoInterface, err = findInterfaceRegEx(nodeV4.Interface)
if err != nil {
return err
}
func findCalicoInterface(nodeV4 *opv1.NodeAddressAutodetection) (IPAutoDetectionMethod, calicoInterface string, err error) {
IPAutoDetectionMethod, err = nodeAddressAutodetection(*nodeV4)
if err != nil {
return "", "", err
}

if strings.Contains(IPAutoDetectionMethod, "cidrs") {
calicoInterface, err = findInterfaceCIDR(nodeV4.CIDRS)
if err != nil {
return "", "", err
}
}

if strings.Contains(IPAutoDetectionMethod, "can-reach") {
calicoInterface, err = findInterfaceReach(nodeV4.CanReach)
if err != nil {
return err
}
if strings.Contains(IPAutoDetectionMethod, "interface") {
calicoInterface, err = findInterfaceRegEx(nodeV4.Interface)
if err != nil {
return "", "", err
}
}
if strings.Contains(IPAutoDetectionMethod, "can-reach") {
calicoInterface, err = findInterfaceReach(nodeV4.CanReach)
if err != nil {
return "", "", err
}
}
return
}

c.CNICfg.Interface = calicoInterface
func startConfd(ctx context.Context, config *CalicoConfig) {
outputFile, err := os.Create(calicoLogPath + "confd.log")
if err != nil {
logrus.Fatalf("error creating confd.log: %v", err)
return
}
defer outputFile.Close()

return nil
specificEnvs := []string{
fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
}

args := []string{
"-confd",
fmt.Sprintf( "-confd-confdir=%s", filepath.Join(config.CNI.BinDir, "confd")),
}

logrus.Infof("Confd Envs: %s", append(generateGeneralCalicoEnvs(config), specificEnvs...))
cmd := exec.CommandContext(ctx, "calico-node.exe", args...)
cmd.Env = append(generateGeneralCalicoEnvs(config), specificEnvs...)
cmd.Stdout = outputFile
cmd.Stderr = outputFile
_ = os.Chdir(filepath.Join(config.CNI.BinDir, "confd"))
_ = cmd.Run()
logrus.Error("Confd exited")
}

func startFelix(ctx context.Context, config *CalicoConfig) {
Expand Down
31 changes: 14 additions & 17 deletions pkg/windows/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,18 @@ import (
)

// createHnsNetwork creates the network that will connect nodes and returns its managementIP
func createHnsNetwork(backend string, vxlanAdapter string) (string, error) {
func createHnsNetwork(backend string, networkAdapter string) (string, error) {
var network hcsshim.HNSNetwork
if backend == "vxlan" {
// Ignoring the return because both true and false without an error represent that the firewall rule was created or already exists
if _, err := wapi.FirewallRuleAdd("OverlayTraffic4789UDP", "Overlay network traffic UDP", "", "4789", wapi.NET_FW_IP_PROTOCOL_UDP, wapi.NET_FW_PROFILE2_ALL); err != nil {
return "", fmt.Errorf("error creating firewall rules: %v", err)
}
logrus.Infof("Creating VXLAN network using the vxlanAdapter: %s", vxlanAdapter)
logrus.Infof("Creating VXLAN network using the vxlanAdapter: %s", networkAdapter)
network = hcsshim.HNSNetwork{
Type: "Overlay",
Name: CalicoHnsNetworkName,
NetworkAdapterName: vxlanAdapter,
NetworkAdapterName: networkAdapter,
Subnets: []hcsshim.Subnet{
{
AddressPrefix: "192.168.255.0/30",
Expand All @@ -45,21 +45,18 @@ func createHnsNetwork(backend string, vxlanAdapter string) (string, error) {
},
}
} else {
return "", fmt.Errorf("The Calico backend %s is not supported. Only vxlan backend is supported", backend)
network = hcsshim.HNSNetwork{
Type: "L2Bridge",
Name: CalicoHnsNetworkName,
NetworkAdapterName: networkAdapter,
Subnets: []hcsshim.Subnet{
{
AddressPrefix: "192.168.255.0/30",
GatewayAddress: "192.168.255.1",
},
},
}
}
// Currently, only vxlan is supported. Leaving the code for future
//} else {
// network = hcsshim.HNSNetwork{
// Type: "L2Bridge",
// Name: CalicoHnsNetworkName,
// Subnets: []hcsshim.Subnet{
// {
// AddressPrefix: "192.168.255.0/30",
// GatewayAddress: "192.168.255.1",
// },
// },
// }
//}

if _, err := network.Create(); err != nil {
return "", fmt.Errorf("error creating the %s network: %v", CalicoHnsNetworkName, err)
Expand Down

0 comments on commit 3c0847d

Please sign in to comment.