diff --git a/Dockerfile.windows b/Dockerfile.windows index dc8729b9fd..87741f31e3 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -95,7 +95,9 @@ RUN unzip calico-windows-${CALICO_VERSION}.zip RUN mv CalicoWindows/calico-node.exe rancher/ RUN mv CalicoWindows/cni/calico.exe rancher/ RUN mv CalicoWindows/cni/calico-ipam.exe rancher/ +RUN mv CalicoWindows/confd confd/ FROM scratch AS windows-runtime COPY --from=containerd /usr/local/bin/*.exe /bin/ COPY --from=windows-runtime-collect ./rancher/* /bin/ +COPY --from=windows-runtime-collect ./confd/ /bin/confd diff --git a/pkg/pebinaryexecutor/pebinary.go b/pkg/pebinaryexecutor/pebinary.go index 447e7047e6..960261f630 100644 --- a/pkg/pebinaryexecutor/pebinary.go +++ b/pkg/pebinaryexecutor/pebinary.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" + "github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io" "github.com/k3s-io/k3s/pkg/cli/cmds" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/executor" @@ -24,7 +25,9 @@ import ( "github.com/rancher/rke2/pkg/logging" win "github.com/rancher/rke2/pkg/windows" "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -50,7 +53,8 @@ type PEBinaryConfig struct { KubeConfigKubeProxy string DisableETCD bool IsServer bool - cni win.Calico + CNI string + CNIConfig win.Calico } type CloudProviderConfig struct { @@ -58,6 +62,13 @@ type CloudProviderConfig struct { Path string } +const ( + CNINone = "none" + CNICalico = "calico" + CNICilium = "cilium" + CNICanal = "canal" +) + // Bootstrap prepares the binary executor to run components by setting the system default registry // and staging the kubelet and containerd binaries. On servers, it also ensures that manifests are // copied in to place and in sync with the system configuration. @@ -94,10 +105,22 @@ func (p *PEBinaryConfig) Bootstrap(ctx context.Context, nodeConfig *daemonconfig restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) - if err := p.cni.Setup(ctx, nodeConfig, restConfig, p.DataDir); err != nil { + p.CNI, err = getCniType(restConfig) + if err != nil { return err } + switch p.CNI { + case "", CNICalico: + if err := p.CNIConfig.Setup(ctx, nodeConfig, restConfig, p.DataDir); err != nil { + return err + } + case CNINone: + logrus.Info("Skipping CNI setup") + default: + logrus.Fatal("Unsupported CNI: ", p.CNI) + } + // required to initialize KubeProxy p.KubeConfigKubeProxy = nodeConfig.AgentConfig.KubeConfigKubeProxy @@ -135,11 +158,13 @@ func (p *PEBinaryConfig) Kubelet(ctx context.Context, args []string) error { go func() { for { cniCtx, cancel := context.WithCancel(ctx) - go func() { - if err := p.cni.Start(cniCtx); err != nil { - logrus.Errorf("error in cni start: %s", err) - } - }() + if p.CNI != CNINone { + go func() { + if err := p.CNIConfig.Start(cniCtx); err != nil { + logrus.Errorf("error in cni start: %s", err) + } + }() + } cmd := exec.CommandContext(ctx, p.KubeletPath, cleanArgs...) cmd.Stdout = logOut @@ -156,9 +181,13 @@ func (p *PEBinaryConfig) Kubelet(ctx context.Context, args []string) error { // KubeProxy starts the kubeproxy in a subprocess with watching goroutine. func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error { + if p.CNI == CNINone { + return nil + } + extraArgs := map[string]string{ - "network-name": p.cni.CNICfg.OverlayNetName, - "bind-address": p.cni.CNICfg.IP, + "network-name": p.CNIConfig.CNICfg.OverlayNetName, + "bind-address": p.CNIConfig.CNICfg.IP, } if err := hcn.DSRSupported(); err == nil { @@ -167,7 +196,7 @@ func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error { extraArgs["enable-dsr"] = "true" } - if p.cni.CNICfg.Name == "Calico" { + if p.CNIConfig.CNICfg.Name == "Calico" { var vip string for range time.Tick(time.Second * 5) { endpoint, err := hcsshim.GetHNSEndpointByName("Calico_ep") @@ -258,6 +287,28 @@ func getArgs(argsMap map[string]string) []string { return args } +func getCniType(restConfig *rest.Config) (string, error) { + hc, err := helm.NewFactoryFromConfig(restConfig) + if err != nil { + return "", err + } + hl, err := hc.Helm().V1().HelmChart().List(metav1.NamespaceSystem, metav1.ListOptions{}) + if err != nil { + return "", err + } + for _, h := range hl.Items { + switch h.Name { + case win.CalicoChart: + return CNICalico, nil + case "rke2-cilium": + return CNICilium, nil + case "rke2-canal": + return CNICanal, nil + } + } + return CNINone, nil +} + // setWindowsAgentSpecificSettings configures the correct paths needed for Windows func setWindowsAgentSpecificSettings(dataDir string, nodeConfig *daemonconfig.Node) { nodeConfig.AgentConfig.CNIBinDir = filepath.Join("c:\\", dataDir, "bin") diff --git a/pkg/rke2/rke2_windows.go b/pkg/rke2/rke2_windows.go index 7b9503b062..5b9d095e6c 100644 --- a/pkg/rke2/rke2_windows.go +++ b/pkg/rke2/rke2_windows.go @@ -69,5 +69,6 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto KubeletPath: cfg.KubeletPath, DisableETCD: clx.Bool("disable-etcd"), IsServer: isServer, + CNI: "", }, nil } diff --git a/pkg/windows/calico.go b/pkg/windows/calico.go index d0e4103f15..c8ef326a2a 100644 --- a/pkg/windows/calico.go +++ b/pkg/windows/calico.go @@ -17,6 +17,7 @@ import ( "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" "github.com/sirupsen/logrus" + opv1 "github.com/tigera/operator/api/v1" authv1 "k8s.io/api/authentication/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,7 +105,11 @@ users: "Value": { "Type": "SDNROUTE", "DestinationPrefix": "{{ .ServiceCIDR }}", - "NeedEncap": true + {{- if eq .Mode "vxlan" }} + "NeedEncap": true + {{- else }} + "NeedEncap": false + {{- end }} } } ] @@ -265,6 +270,9 @@ func (c *Calico) Start(ctx context.Context) error { break } go startFelix(ctx, c.CNICfg) + if c.CNICfg.Mode == "windows-bgp" { + go startConfd(ctx, c.CNICfg) + } return nil } @@ -280,10 +288,10 @@ func (c *Calico) generateCalicoNetworks() error { // 2 - c.CNICfg.Interface which set if NodeAddressAutodetection is set (Calico HelmChart) // 3 - nodeIP if defined // 4 - None of the above. In that case, by default the interface with the default route is picked - vxlanAdapter := os.Getenv("VXLAN_ADAPTER") - if vxlanAdapter == "" { + networkAdapter := os.Getenv("VXLAN_ADAPTER") + if networkAdapter == "" { if c.CNICfg.Interface != "" { - vxlanAdapter = c.CNICfg.Interface + networkAdapter = c.CNICfg.Interface } if c.CNICfg.Interface == "" && c.CNICfg.IP != "" { @@ -291,11 +299,11 @@ func (c *Calico) generateCalicoNetworks() error { if err != nil { return err } - vxlanAdapter = iFace + networkAdapter = iFace } } - mgmt, err := createHnsNetwork(c.CNICfg.Mode, vxlanAdapter) + mgmt, err := createHnsNetwork(c.CNICfg.Mode, networkAdapter) if err != nil { return err } @@ -338,39 +346,72 @@ func (c *Calico) overrideCalicoConfigByHelm(restConfig *rest.Config) error { } logrus.Debugf("calico override found: %s\n", string(b)) if nodeV4 := overrides.Installation.CalicoNetwork.NodeAddressAutodetectionV4; nodeV4 != nil { - IPAutoDetectionMethod, err := nodeAddressAutodetection(*nodeV4) + c.CNICfg.IPAutoDetectionMethod, c.CNICfg.Interface, err = findCalicoInterface(nodeV4) if err != nil { return err } - logrus.Debugf("this is IPAutoDetectionMethod: %s", IPAutoDetectionMethod) - c.CNICfg.IPAutoDetectionMethod = IPAutoDetectionMethod - - var calicoInterface string - if strings.Contains(IPAutoDetectionMethod, "cidrs") { - calicoInterface, err = findInterfaceCIDR(nodeV4.CIDRS) - if err != nil { - return err - } + } + if bgpEnabled := overrides.Installation.CalicoNetwork.BGP; bgpEnabled != nil { + if *bgpEnabled == opv1.BGPEnabled { + c.CNICfg.Mode = "windows-bgp" } + } + return nil +} - if strings.Contains(IPAutoDetectionMethod, "interface") { - calicoInterface, err = findInterfaceRegEx(nodeV4.Interface) - if err != nil { - return err - } +func findCalicoInterface(nodeV4 *opv1.NodeAddressAutodetection) (IPAutoDetectionMethod, calicoInterface string, err error) { + IPAutoDetectionMethod, err = nodeAddressAutodetection(*nodeV4) + if err != nil { + return "", "", err + } + + if strings.Contains(IPAutoDetectionMethod, "cidrs") { + calicoInterface, err = findInterfaceCIDR(nodeV4.CIDRS) + if err != nil { + return "", "", err } + } - if strings.Contains(IPAutoDetectionMethod, "can-reach") { - calicoInterface, err = findInterfaceReach(nodeV4.CanReach) - if err != nil { - return err - } + if strings.Contains(IPAutoDetectionMethod, "interface") { + calicoInterface, err = findInterfaceRegEx(nodeV4.Interface) + if err != nil { + return "", "", err } + } + if strings.Contains(IPAutoDetectionMethod, "can-reach") { + calicoInterface, err = findInterfaceReach(nodeV4.CanReach) + if err != nil { + return "", "", err + } + } + return +} - c.CNICfg.Interface = calicoInterface +func startConfd(ctx context.Context, config *CalicoConfig) { + outputFile, err := os.Create(calicoLogPath + "confd.log") + if err != nil { + logrus.Fatalf("error creating confd.log: %v", err) + return } + defer outputFile.Close() - return nil + specificEnvs := []string{ + fmt.Sprintf("PATH=%s", os.Getenv("PATH")), + } + + args := []string{ + "-confd", + fmt.Sprintf( "-confd-confdir=%s", filepath.Join(config.CNI.BinDir, "confd")), + } + + logrus.Infof("Confd Envs: %s", append(generateGeneralCalicoEnvs(config), specificEnvs...)) + cmd := exec.CommandContext(ctx, "calico-node.exe", args...) + cmd.Env = append(generateGeneralCalicoEnvs(config), specificEnvs...) + cmd.Stdout = outputFile + cmd.Stderr = outputFile + _ = os.Chdir(filepath.Join(config.CNI.BinDir, "confd")) + _ = cmd.Run() + logrus.Error("Confd exited") } func startFelix(ctx context.Context, config *CalicoConfig) { diff --git a/pkg/windows/utils.go b/pkg/windows/utils.go index 3e760947f2..a56abf4284 100644 --- a/pkg/windows/utils.go +++ b/pkg/windows/utils.go @@ -22,18 +22,18 @@ import ( ) // createHnsNetwork creates the network that will connect nodes and returns its managementIP -func createHnsNetwork(backend string, vxlanAdapter string) (string, error) { +func createHnsNetwork(backend string, networkAdapter string) (string, error) { var network hcsshim.HNSNetwork if backend == "vxlan" { // Ignoring the return because both true and false without an error represent that the firewall rule was created or already exists if _, err := wapi.FirewallRuleAdd("OverlayTraffic4789UDP", "Overlay network traffic UDP", "", "4789", wapi.NET_FW_IP_PROTOCOL_UDP, wapi.NET_FW_PROFILE2_ALL); err != nil { return "", fmt.Errorf("error creating firewall rules: %v", err) } - logrus.Infof("Creating VXLAN network using the vxlanAdapter: %s", vxlanAdapter) + logrus.Infof("Creating VXLAN network using the vxlanAdapter: %s", networkAdapter) network = hcsshim.HNSNetwork{ Type: "Overlay", Name: CalicoHnsNetworkName, - NetworkAdapterName: vxlanAdapter, + NetworkAdapterName: networkAdapter, Subnets: []hcsshim.Subnet{ { AddressPrefix: "192.168.255.0/30", @@ -45,21 +45,18 @@ func createHnsNetwork(backend string, vxlanAdapter string) (string, error) { }, } } else { - return "", fmt.Errorf("The Calico backend %s is not supported. Only vxlan backend is supported", backend) + network = hcsshim.HNSNetwork{ + Type: "L2Bridge", + Name: CalicoHnsNetworkName, + NetworkAdapterName: networkAdapter, + Subnets: []hcsshim.Subnet{ + { + AddressPrefix: "192.168.255.0/30", + GatewayAddress: "192.168.255.1", + }, + }, + } } - // Currently, only vxlan is supported. Leaving the code for future - //} else { - // network = hcsshim.HNSNetwork{ - // Type: "L2Bridge", - // Name: CalicoHnsNetworkName, - // Subnets: []hcsshim.Subnet{ - // { - // AddressPrefix: "192.168.255.0/30", - // GatewayAddress: "192.168.255.1", - // }, - // }, - // } - //} if _, err := network.Create(); err != nil { return "", fmt.Errorf("error creating the %s network: %v", CalicoHnsNetworkName, err)