Update operator to enable LoadBalancer kube-controller (#3490)

api/v1/installation_types.go

@@ -698,13 +698,17 @@ type IPPool struct {
 	// AllowedUse controls what the IP pool will be used for.  If not specified or empty, defaults to
 	// ["Tunnel", "Workload"] for back-compatibility
 	AllowedUses []IPPoolAllowedUse `json:"allowedUses,omitempty" validate:"omitempty"`
+
+	// AssignmentMode determines if IP addresses from this pool should be  assigned automatically or on request only
+	AssignmentMode pcv1.AssignmentMode `json:"assignmentMode,omitempty" validate:"omitempty,assignmentMode"`
 }
 
 type IPPoolAllowedUse string
 
 const (
-	IPPoolAllowedUseWorkload IPPoolAllowedUse = "Workload"
-	IPPoolAllowedUseTunnel   IPPoolAllowedUse = "Tunnel"
+	IPPoolAllowedUseWorkload     IPPoolAllowedUse = "Workload"
+	IPPoolAllowedUseTunnel       IPPoolAllowedUse = "Tunnel"
+	IPPoolAllowedUseLoadBalancer IPPoolAllowedUse = "LoadBalancer"
 )
 
 // ToProjectCalicoV1 converts an IPPool to a crd.projectcalico.org/v1 IPPool resource.
@@ -761,6 +765,8 @@ func (p *IPPool) ToProjectCalicoV1() (*pcv1.IPPool, error) {
 		pool.Spec.AllowedUses = append(pool.Spec.AllowedUses, pcv1.IPPoolAllowedUse(use))
 	}
 
+	pool.Spec.AssignmentMode = p.AssignmentMode
+
 	return &pool, nil
 }
 
@@ -816,6 +822,8 @@ func (p *IPPool) FromProjectCalicoV1(crd pcv1.IPPool) {
 	for _, use := range crd.Spec.AllowedUses {
 		p.AllowedUses = append(p.AllowedUses, IPPoolAllowedUse(use))
 	}
+
+	p.AssignmentMode = crd.Spec.AssignmentMode
 }
 
 // CNIPluginType describes the type of CNI plugin used.

pkg/apis/crd.projectcalico.org/v1/ippool.go

@@ -74,13 +74,17 @@ type IPPoolSpec struct {
 	// AllowedUse controls what the IP pool will be used for.  If not specified or empty, defaults to
 	// ["Tunnel", "Workload"] for back-compatibility
 	AllowedUses []IPPoolAllowedUse `json:"allowedUses,omitempty" validate:"omitempty"`
+
+	// AssignmentMode determines if IP addresses from this pool should be  assigned automatically or on request only
+	AssignmentMode AssignmentMode `json:"assignmentMode,omitempty" validate:"omitempty,assignmentMode"`
 }
 
 type IPPoolAllowedUse string
 
 const (
-	IPPoolAllowedUseWorkload IPPoolAllowedUse = "Workload"
-	IPPoolAllowedUseTunnel   IPPoolAllowedUse = "Tunnel"
+	IPPoolAllowedUseWorkload      IPPoolAllowedUse = "Workload"
+	IPPoolAllowedUseTunnel        IPPoolAllowedUse = "Tunnel"
+	IPPoolsAllowedUseLoadBalancer IPPoolAllowedUse = "LoadBalancer"
 )
 
 type VXLANMode string
@@ -99,6 +103,13 @@ const (
 	IPIPModeCrossSubnet IPIPMode = "CrossSubnet"
 )
 
+type AssignmentMode string
+
+const (
+	Automatic AssignmentMode = "Automatic"
+	Manual    AssignmentMode = "Manual"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // IPPoolList contains a list of IPPool resources.

pkg/controller/ippool/defaults.go

@@ -235,6 +235,10 @@ func fillDefaults(ctx context.Context, client client.Client, instance *operator.
 				}
 			}
 		}
+
+		if pool.AssignmentMode == "" {
+			pool.AssignmentMode = crdv1.Automatic
+		}
 	}
 	return nil
 }

pkg/controller/ippool/pool_controller_test.go

@@ -452,6 +452,7 @@ var _ = table.DescribeTable("Test OpenShift IP pool defaulting",
 					BlockSize:             &twentySix,
 					AllowedUses:           []operator.IPPoolAllowedUse{operator.IPPoolAllowedUseWorkload, operator.IPPoolAllowedUseTunnel},
 					DisableNewAllocations: &false_,
+					AssignmentMode:        "Automatic",
 				},
 			},
 		}),
@@ -481,6 +482,7 @@ var _ = table.DescribeTable("Test OpenShift IP pool defaulting",
 					BlockSize:             &twentySix,
 					AllowedUses:           []operator.IPPoolAllowedUse{operator.IPPoolAllowedUseWorkload, operator.IPPoolAllowedUseTunnel},
 					DisableNewAllocations: &false_,
+					AssignmentMode:        "Automatic",
 				},
 			},
 		}),
@@ -518,6 +520,7 @@ var _ = table.DescribeTable("Test OpenShift IP pool defaulting",
 					BlockSize:             &twentySix,
 					AllowedUses:           []operator.IPPoolAllowedUse{operator.IPPoolAllowedUseWorkload, operator.IPPoolAllowedUseTunnel},
 					DisableNewAllocations: &false_,
+					AssignmentMode:        "Automatic",
 				},
 			},
 		}),

pkg/controller/ippool/validation.go

@@ -42,9 +42,30 @@ func ValidatePools(instance *operator.Installation) error {
 		}
 		names[pool.Name] = true
 
+		// Check if pool is for LoadBalancer
+		isLoadBalancer := false
+		for _, u := range pool.AllowedUses {
+			if u == operator.IPPoolAllowedUseLoadBalancer {
+				isLoadBalancer = true
+			}
+		}
+
+		// Check if pool is set as LoadBalancer no other allowed use is specified
+		if isLoadBalancer {
+			for _, u := range pool.AllowedUses {
+				if u != operator.IPPoolAllowedUseLoadBalancer {
+					return fmt.Errorf("IP pool %s AllowedUse LoadBalancer cannot be used with Workload/Tunnel", pool.Name)
+				}
+			}
+		}
+
 		// Verify NAT outgoing values.
 		switch pool.NATOutgoing {
-		case operator.NATOutgoingEnabled, operator.NATOutgoingDisabled:
+		case operator.NATOutgoingEnabled:
+		case operator.NATOutgoingDisabled:
+			if isLoadBalancer {
+				return fmt.Errorf("IP pool %s NATOutgoing cannot be disabled with allowedUse LoadBalancer", pool.Name)
+			}
 		default:
 			return fmt.Errorf("%s is invalid for natOutgoing, should be one of %s",
 				pool.NATOutgoing, strings.Join(operator.NATOutgoingTypesString, ","))
@@ -54,6 +75,11 @@ func ValidatePools(instance *operator.Installation) error {
 		if pool.NodeSelector == "" {
 			return fmt.Errorf("IP pool nodeSelector should not be empty")
 		}
+
+		if isLoadBalancer && pool.NodeSelector != "all()" {
+			return fmt.Errorf("IP pool nodeSelector should be set to all() if allowedUse is LoadBalancer")
+		}
+
 		if instance.Spec.CNI == nil {
 			// We expect this to be defaulted by the core Installation controller prior to the IP pool controller
 			// being invoked, but check just in case.
@@ -67,8 +93,10 @@ func ValidatePools(instance *operator.Installation) error {
 
 		// Verify the Encapsulation mode is valid.
 		switch pool.Encapsulation {
-		case operator.EncapsulationIPIP, operator.EncapsulationIPIPCrossSubnet:
-		case operator.EncapsulationVXLAN, operator.EncapsulationVXLANCrossSubnet:
+		case operator.EncapsulationIPIP, operator.EncapsulationIPIPCrossSubnet, operator.EncapsulationVXLAN, operator.EncapsulationVXLANCrossSubnet:
+			if isLoadBalancer {
+				return fmt.Errorf("IP pool encapsulation must be none if allowedUse is LoadBalancer")
+			}
 		case operator.EncapsulationNone:
 		default:
 			return fmt.Errorf("%s is invalid for ipPool.encapsulation, should be one of %s",
@@ -104,6 +132,10 @@ func ValidatePools(instance *operator.Installation) error {
 				}
 			}
 		}
+
+		if isLoadBalancer && *pool.DisableBGPExport {
+			return fmt.Errorf("IP pool disable bgp export must be false when AllowedUse is LoadBalancer")
+		}
 	}
 	return nil
 }

pkg/controller/migration/convert/ippools.go

@@ -263,6 +263,7 @@ func convertPool(src crdv1.IPPool) (operatorv1.IPPool, error) {
 
 	p.NodeSelector = src.Spec.NodeSelector
 	p.DisableBGPExport = &src.Spec.DisableBGPExport
+	p.AssignmentMode = src.Spec.AssignmentMode
 
 	return p, nil
 }

pkg/crds/operator/operator.tigera.io_installations.yaml

@@ -1256,6 +1256,11 @@ spec:
                           items:
                             type: string
                           type: array
+                        assignmentMode:
+                          description: AssignmentMode determines if IP addresses from
+                            this pool should be  assigned automatically or on request
+                            only
+                          type: string
                         blockSize:
                           description: |-
                             BlockSize specifies the CIDR prefex length to use when allocating per-node IP blocks from
@@ -9608,6 +9613,11 @@ spec:
                               items:
                                 type: string
                               type: array
+                            assignmentMode:
+                              description: AssignmentMode determines if IP addresses
+                                from this pool should be  assigned automatically or
+                                on request only
+                              type: string
                             blockSize:
                               description: |-
                                 BlockSize specifies the CIDR prefex length to use when allocating per-node IP blocks from

pkg/render/kubecontrollers/kube-controllers.go

@@ -107,7 +107,7 @@ type KubeControllersConfiguration struct {
 
 func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent {
 	kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg, KubeController)
-	enabledControllers := []string{"node"}
+	enabledControllers := []string{"node", "loadbalancer"}
 	if cfg.Installation.Variant == operatorv1.TigeraSecureEnterprise {
 		kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...)
 		kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules,
@@ -360,6 +360,11 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration, kubeContr
 			Resources: []string{"pods"},
 			Verbs:     []string{"get", "list", "watch"},
 		},
+		{
+			APIGroups: []string{""},
+			Resources: []string{"services", "services/status"},
+			Verbs:     []string{"get", "list", "update", "watch"},
+		},
 		{
 			// IPAM resources are manipulated in response to node and block updates, as well as periodic triggers.
 			APIGroups: []string{"crd.projectcalico.org"},
@@ -368,7 +373,7 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration, kubeContr
 		},
 		{
 			APIGroups: []string{"crd.projectcalico.org"},
-			Resources: []string{"blockaffinities", "ipamblocks", "ipamhandles", "networksets"},
+			Resources: []string{"blockaffinities", "ipamblocks", "ipamhandles", "networksets", "ipamconfigs"},
 			Verbs:     []string{"get", "list", "create", "update", "delete", "watch"},
 		},
 		{

pkg/render/kubecontrollers/kube-controllers_test.go

@@ -185,7 +185,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		// Verify env
 		expectedEnv := []corev1.EnvVar{
 			{Name: "DATASTORE_TYPE", Value: "kubernetes"},
-			{Name: "ENABLED_CONTROLLERS", Value: "node"},
+			{Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer"},
 			{Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: "default"},
 			{Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: "false"},
 		}
@@ -247,14 +247,14 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/kube-controllers:" + components.ComponentTigeraKubeControllers.Version))
 		envs := dp.Spec.Template.Spec.Containers[0].Env
 		Expect(envs).To(ContainElement(corev1.EnvVar{
-			Name: "ENABLED_CONTROLLERS", Value: "node,service,federatedservices,usage",
+			Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage",
 		}))
 
 		Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1))
 		Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1))
 
 		clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole)
-		Expect(clusterRole.Rules).To(HaveLen(19))
+		Expect(clusterRole.Rules).To(HaveLen(20))
 
 		ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service)
 		Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless")
@@ -341,7 +341,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle"))
 
 		clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole)
-		Expect(clusterRole.Rules).To(HaveLen(18))
+		Expect(clusterRole.Rules).To(HaveLen(19))
 		Expect(clusterRole.Rules).To(ContainElement(
 			rbacv1.PolicyRule{
 				APIGroups: []string{""},
@@ -388,7 +388,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		envs := dp.Spec.Template.Spec.Containers[0].Env
 		Expect(envs).To(ContainElement(corev1.EnvVar{
 			Name:  "ENABLED_CONTROLLERS",
-			Value: "node,service,federatedservices,usage",
+			Value: "node,loadbalancer,service,federatedservices,usage",
 		}))
 
 		Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1))
@@ -544,7 +544,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/kube-controllers:" + components.ComponentTigeraKubeControllers.Version))
 
 		clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole)
-		Expect(clusterRole.Rules).To(HaveLen(18))
+		Expect(clusterRole.Rules).To(HaveLen(19))
 		Expect(clusterRole.Rules).To(ContainElement(
 			rbacv1.PolicyRule{
 				APIGroups: []string{""},

test/pool_test.go

@@ -142,6 +142,7 @@ var _ = Describe("IPPool FV tests", func() {
 		Expect(ipPools.Items[0].Spec.Disabled).To(Equal(false))
 		Expect(ipPools.Items[0].Spec.BlockSize).To(Equal(26))
 		Expect(ipPools.Items[0].Spec.NodeSelector).To(Equal("all()"))
+		Expect(ipPools.Items[0].Spec.AssignmentMode).To(Equal(crdv1.Automatic))
 
 		// Verify that a default IPv6 pool was created.
 		Expect(ipPools.Items[1].Name).To(Equal("default-ipv6-ippool"))
@@ -150,6 +151,7 @@ var _ = Describe("IPPool FV tests", func() {
 		Expect(ipPools.Items[1].Spec.Disabled).To(Equal(false))
 		Expect(ipPools.Items[1].Spec.BlockSize).To(Equal(122))
 		Expect(ipPools.Items[1].Spec.NodeSelector).To(Equal("all()"))
+		Expect(ipPools.Items[1].Spec.AssignmentMode).To(Equal(crdv1.Automatic))
 
 		// Expect the default pools to be marked as managed by the operator.
 		for _, p := range ipPools.Items {
@@ -190,6 +192,7 @@ var _ = Describe("IPPool FV tests", func() {
 		Expect(ipPools.Items[0].Spec.BlockSize).To(Equal(26))
 		Expect(ipPools.Items[0].Spec.NodeSelector).To(Equal("all()"))
 		Expect(ipPools.Items[0].Labels).To(HaveLen(1))
+		Expect(ipPools.Items[0].Spec.AssignmentMode).To(Equal(crdv1.Automatic))
 	})
 
 	It("should assume ownership of legacy default IP pools", func() {
@@ -209,6 +212,7 @@ var _ = Describe("IPPool FV tests", func() {
 					crdv1.IPPoolAllowedUseWorkload,
 					crdv1.IPPoolAllowedUseTunnel,
 				},
+				AssignmentMode: crdv1.Automatic,
 			},
 		}
 		Expect(c.Create(context.Background(), &ipPool)).To(Succeed())
@@ -283,6 +287,7 @@ var _ = Describe("IPPool FV tests", func() {
 		Expect(v3Pools.Items[0].Spec.NodeSelector).To(Equal("all()"))
 		Expect(v3Pools.Items[0].Spec.IPIPMode).To(Equal(v3.IPIPMode(v3.IPIPModeAlways)))
 		Expect(v3Pools.Items[0].Spec.VXLANMode).To(Equal(v3.VXLANMode(v3.VXLANModeNever)))
+		Expect(ipPools.Items[0].Spec.AssignmentMode).To(Equal(crdv1.Automatic))
 	})
 
 	// This test verifies that the IP pool controller doesn't assume ownership of IP pools that may exist in the
@@ -311,7 +316,7 @@ var _ = Describe("IPPool FV tests", func() {
 		}
 		Expect(c.Create(context.Background(), &ipPool)).To(Succeed())
 
-		// Create an Installation referencing the IP pool by CIDR, mimicing the upgrade case. We expect
+		// Create an Installation referencing the IP pool by CIDR, mimicking the upgrade case. We expect
 		// the operator to default the IP pool, filling in any missing fields. But it won't
 		// update IP pool in the cluster since it doesn't match exactly.
 		spec := operator.InstallationSpec{