Skip to content

Commit

Permalink
Move sourceVIP reservation to CNI code
Browse files Browse the repository at this point in the history
Signed-off-by: Manuel Buil <[email protected]>
  • Loading branch information
manuelbuil committed Jan 18, 2024
1 parent 49ef16c commit c6fee0e
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 22 deletions.
28 changes: 7 additions & 21 deletions pkg/pebinaryexecutor/pebinary.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"strings"
"time"

"github.com/Microsoft/hcsshim"
"github.com/Microsoft/hcsshim/hcn"
"github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io"
"github.com/k3s-io/k3s/pkg/cli/cmds"
Expand Down Expand Up @@ -187,10 +186,17 @@ func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error {
}

CNIConfig := p.CniPlugin.GetConfig()
vip, err := p.CniPlugin.ReserveSourceVip(ctx)
if err != nil || vip == "" {
logrus.Errorf("Failed to reserve VIP for kube-proxy: %s", err)
}
logrus.Infof("Reserved VIP for kube-proxy: %s", vip)


extraArgs := map[string]string{
"network-name": CNIConfig.OverlayNetName,
"bind-address": CNIConfig.NodeIP,
"source-vip": vip,
}

if err := hcn.DSRSupported(); err == nil {
Expand All @@ -199,26 +205,6 @@ func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error {
extraArgs["enable-dsr"] = "true"
}

if CNIConfig.Name == "Calico" {
var vip string
for range time.Tick(time.Second * 5) {
endpoint, err := hcsshim.GetHNSEndpointByName("Calico_ep")
if err != nil {
logrus.WithError(err).Warning("can't find Calico_ep HNS endpoint, retrying")
continue
}
vip = endpoint.IPAddress.String()
break
}
extraArgs["source-vip"] = vip
}

logrus.Infof("Deleting HNS policies before kube-proxy starts.")
policies, _ := hcsshim.HNSListPolicyListRequest()
for _, policy := range policies {
policy.Delete()
}

args = append(getArgs(extraArgs), args...)

logrus.Infof("Running RKE2 kube-proxy %s", args)
Expand Down
30 changes: 30 additions & 0 deletions pkg/windows/calico.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"text/template"
"time"

"github.com/Microsoft/hcsshim"
"github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io"
daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/version"
Expand All @@ -23,6 +24,7 @@ import (
authv1 "k8s.io/api/authentication/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"sigs.k8s.io/yaml"
Expand Down Expand Up @@ -279,6 +281,14 @@ func (c *Calico) Start(ctx context.Context) error {
go startConfd(ctx, c.CNICfg, logPath)
}

// Delete policies in case calico network is being reused
policies, _ := hcsshim.HNSListPolicyListRequest()
for _, policy := range policies {
policy.Delete()
}

logrus.Info("Calico started correctly")

return nil
}

Expand Down Expand Up @@ -486,3 +496,23 @@ func generateGeneralCalicoEnvs(config *CalicoConfig) []string {
fmt.Sprintf("USE_POD_CIDR=%t", autoConfigureIpam(config.IpamType)),
}
}

// ReserveSourceVip reserves a source VIP for kube-proxy
func (c *Calico) ReserveSourceVip(ctx context.Context) (string, error) {
var vip string

if err := wait.PollImmediateWithContext(ctx, 5*time.Second, 5*time.Minute, func(ctx context.Context) (bool, error) {
// calico-node is creating an endpoint named Calico_ep for this purpose
endpoint, err := hcsshim.GetHNSEndpointByName("Calico_ep")
if err != nil {
logrus.WithError(err).Warning("can't find Calico_ep HNS endpoint, retrying")
return false, nil
}
vip = endpoint.IPAddress.String()
return true, nil
}); err != nil {
return "", err
}

return vip, nil
}
2 changes: 1 addition & 1 deletion pkg/windows/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type CniPlugin interface {
Setup(ctx context.Context, nodeConfig *daemonconfig.Node, restConfig *rest.Config, dataDir string) error
Start(ctx context.Context) error
GetConfig() *CNICommonConfig
ReserveSourceVip(ctx context.Context) (string, error)
}

type KubeConfig struct {
Expand All @@ -37,7 +38,6 @@ type CNICommonConfig struct {
NodeIP string
VxlanVNI string
VxlanPort string
KubeProxySourceVIP string
Interface string
IpamType string
CNIVersion string
Expand Down

0 comments on commit c6fee0e

Please sign in to comment.