Split internal and external kube-apiserver addresses

This allows us to select the right IP address when we have multiple load balancers (one for internal, one for external) as we are planning to have on GCE. Co-authored-by: Ciprian Hacman <[email protected]>
kubernetes · Feb 18, 2024 · b3fdf46 · b3fdf46
1 parent 24ab206
commit b3fdf46
Show file tree

Hide file tree

Showing 20 changed files with 134 additions and 70 deletions.
diff --git a/pkg/apis/nodeup/config.go b/pkg/apis/nodeup/config.go
@@ -150,6 +150,7 @@ type BootConfig struct {
 	ConfigServer *ConfigServerOptions `json:",omitempty"`
 	// APIServerIPs is the API server IP addresses.
 	// This field is used for adding an alias for api.internal. in /etc/hosts, when Topology.DNS.Type == DNSTypeNone.
+	// These addresses are also added to the kops-controller TLS certificate.
 	APIServerIPs []string `json:",omitempty"`
 	// ClusterName is the name of the cluster.
 	ClusterName string `json:",omitempty"`

diff --git a/pkg/commands/toolbox_enroll.go b/pkg/commands/toolbox_enroll.go
@@ -120,12 +120,12 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer
 			// 	apiserverAdditionalIPs = append(apiserverAdditionalIPs, ingress.Hostname)
 			// }
 			if ingress.IP != "" {
-				wellKnownAddresses[wellknownservices.KubeAPIServer] = append(wellKnownAddresses[wellknownservices.KubeAPIServer], ingress.IP)
+				wellKnownAddresses[wellknownservices.KubeAPIServerExternal] = append(wellKnownAddresses[wellknownservices.KubeAPIServerExternal], ingress.IP)
 			}
 		}
 	}
 
-	if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 {
+	if len(wellKnownAddresses[wellknownservices.KubeAPIServerExternal]) == 0 {
 		// TODO: Should we support DNS?
 		return fmt.Errorf("unable to determine IP address for kube-apiserver")
 	}

diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go
@@ -199,11 +199,14 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			SecurityGroups: []*awstasks.SecurityGroup{
 				b.LinkToELBSecurityGroup("api"),
 			},
-			SubnetMappings:    nlbSubnetMappings,
-			Tags:              tags,
-			WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
-			VPC:               b.LinkToVPC(),
-			Type:              fi.PtrTo("network"),
+			SubnetMappings: nlbSubnetMappings,
+			Tags:           tags,
+			WellKnownServices: []wellknownservices.WellKnownService{
+				wellknownservices.KubeAPIServerInternal,
+				wellknownservices.KubeAPIServerExternal,
+			},
+			VPC:  b.LinkToVPC(),
+			Type: fi.PtrTo("network"),
 		}
 
 		// Wait for all load balancer components to be created (including network interfaces needed for NoneDNS).
@@ -241,8 +244,11 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 				Timeout: fi.PtrTo(int64(300)),
 			},
 
-			Tags:              tags,
-			WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
+			Tags: tags,
+			WellKnownServices: []wellknownservices.WellKnownService{
+				wellknownservices.KubeAPIServerInternal,
+				wellknownservices.KubeAPIServerExternal,
+			},
 		}
 
 		if b.Cluster.UsesNoneDNS() {
@@ -554,8 +560,8 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 				ToPort:        fi.PtrTo(int64(4)),
 			})
 			if b.Cluster.UsesNoneDNS() {
-				nlb.WellKnownServices = append(nlb.WellKnownServices, wellknownservices.KopsController)
-				clb.WellKnownServices = append(clb.WellKnownServices, wellknownservices.KopsController)
+				nlb.WellKnownServices = append(nlb.WellKnownServices, wellknownservices.KopsControllerInternal)
+				clb.WellKnownServices = append(clb.WellKnownServices, wellknownservices.KopsControllerInternal)
 
 				c.AddTask(&awstasks.SecurityGroupRule{
 					Name:          fi.PtrTo(fmt.Sprintf("kops-controller-elb-to-cp%s", suffix)),

diff --git a/pkg/model/azuremodel/api_loadbalancer.go b/pkg/model/azuremodel/api_loadbalancer.go
@@ -50,16 +50,20 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er
 
 	// Create LoadBalancer for API ELB
 	lb := &azuretasks.LoadBalancer{
-		Name:              fi.PtrTo(b.NameForLoadBalancer()),
-		Lifecycle:         b.Lifecycle,
-		ResourceGroup:     b.LinkToResourceGroup(),
-		Tags:              map[string]*string{},
-		WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
+		Name:          fi.PtrTo(b.NameForLoadBalancer()),
+		Lifecycle:     b.Lifecycle,
+		ResourceGroup: b.LinkToResourceGroup(),
+		Tags:          map[string]*string{},
 	}
 
+	lb.WellKnownServices = append(lb.WellKnownServices,
+		wellknownservices.KubeAPIServerExternal,
+		wellknownservices.KubeAPIServerInternal)
+
 	switch lbSpec.Type {
 	case kops.LoadBalancerTypeInternal:
 		lb.External = to.Ptr(false)
+
 		subnet, err := b.subnetForLoadBalancer()
 		if err != nil {
 			return err
@@ -83,7 +87,7 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er
 	c.AddTask(lb)
 
 	if b.Cluster.UsesLegacyGossip() || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() {
-		lb.WellKnownServices = append(lb.WellKnownServices, wellknownservices.KopsController)
+		lb.WellKnownServices = append(lb.WellKnownServices, wellknownservices.KopsControllerInternal)
 	}
 
 	return nil

diff --git a/pkg/model/domodel/api_loadbalancer.go b/pkg/model/domodel/api_loadbalancer.go
@@ -61,11 +61,15 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er
 
 	// Create LoadBalancer for API LB
 	loadbalancer := &dotasks.LoadBalancer{
-		Name:              fi.PtrTo(loadbalancerName),
-		Region:            fi.PtrTo(b.Cluster.Spec.Networking.Subnets[0].Region),
-		DropletTag:        fi.PtrTo(clusterMasterTag),
-		Lifecycle:         b.Lifecycle,
-		WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KopsController, wellknownservices.KubeAPIServer},
+		Name:       fi.PtrTo(loadbalancerName),
+		Region:     fi.PtrTo(b.Cluster.Spec.Networking.Subnets[0].Region),
+		DropletTag: fi.PtrTo(clusterMasterTag),
+		Lifecycle:  b.Lifecycle,
+		WellKnownServices: []wellknownservices.WellKnownService{
+			wellknownservices.KopsControllerInternal,
+			wellknownservices.KubeAPIServerExternal,
+			wellknownservices.KubeAPIServerInternal,
+		},
 	}
 
 	if b.Cluster.Spec.Networking.NetworkID != "" {

diff --git a/pkg/model/gcemodel/api_loadbalancer.go b/pkg/model/gcemodel/api_loadbalancer.go
@@ -67,8 +67,11 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
 	ipAddress := &gcetasks.Address{
 		Name: s(b.NameForIPAddress("api")),
 
-		Lifecycle:         b.Lifecycle,
-		WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
+		Lifecycle: b.Lifecycle,
+		WellKnownServices: []wellknownservices.WellKnownService{
+			wellknownservices.KubeAPIServerInternal,
+			wellknownservices.KubeAPIServerExternal,
+		},
 	}
 	c.AddTask(ipAddress)
 
@@ -88,7 +91,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
 		},
 	})
 	if b.Cluster.UsesNoneDNS() {
-		ipAddress.WellKnownServices = append(ipAddress.WellKnownServices, wellknownservices.KopsController)
+		ipAddress.WellKnownServices = append(ipAddress.WellKnownServices, wellknownservices.KopsControllerInternal)
 
 		c.AddTask(&gcetasks.ForwardingRule{
 			Name:                s(b.NameForForwardingRule("kops-controller")),
@@ -208,8 +211,11 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 			Purpose:       s("SHARED_LOADBALANCER_VIP"),
 			Subnetwork:    subnet,
 
-			WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
-			Lifecycle:         b.Lifecycle,
+			WellKnownServices: []wellknownservices.WellKnownService{
+				wellknownservices.KubeAPIServerExternal,
+				wellknownservices.KubeAPIServerInternal,
+			},
+			Lifecycle: b.Lifecycle,
 		}
 		c.AddTask(ipAddress)
 
@@ -229,7 +235,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 			},
 		})
 		if b.Cluster.UsesNoneDNS() {
-			ipAddress.WellKnownServices = append(ipAddress.WellKnownServices, wellknownservices.KopsController)
+			ipAddress.WellKnownServices = append(ipAddress.WellKnownServices, wellknownservices.KopsControllerInternal)
 
 			c.AddTask(&gcetasks.ForwardingRule{
 				Name:                s(b.NameForForwardingRule("kops-controller-" + sn.Name)),

diff --git a/pkg/model/hetznermodel/loadbalancer.go b/pkg/model/hetznermodel/loadbalancer.go
@@ -65,7 +65,11 @@ func (b *LoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
 			hetzner.TagKubernetesClusterName: b.ClusterName(),
 		},
 
-		WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer, wellknownservices.KopsController},
+		WellKnownServices: []wellknownservices.WellKnownService{
+			wellknownservices.KopsControllerInternal,
+			wellknownservices.KubeAPIServerExternal,
+			wellknownservices.KubeAPIServerInternal,
+		},
 	}
 
 	c.AddTask(&loadbalancer)

diff --git a/pkg/model/openstackmodel/servergroup.go b/pkg/model/openstackmodel/servergroup.go
@@ -201,7 +201,9 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContex
 		c.AddTask(portTask)
 
 		if b.Cluster.UsesNoneDNS() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
-			portTask.WellKnownServices = append(portTask.WellKnownServices, wellknownservices.KubeAPIServer)
+			portTask.WellKnownServices = append(portTask.WellKnownServices,
+				wellknownservices.KubeAPIServerExternal,
+				wellknownservices.KubeAPIServerInternal)
 		}
 
 		metaWithName := make(map[string]string)
@@ -243,7 +245,10 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContex
 				if ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
 					// Ensure the floating IP is included in the TLS certificate,
 					// if we're not going to use an alias for it
-					t.WellKnownServices = append(t.WellKnownServices, wellknownservices.KubeAPIServer, wellknownservices.KopsController)
+					t.WellKnownServices = append(t.WellKnownServices,
+						wellknownservices.KubeAPIServerExternal,
+						wellknownservices.KubeAPIServerInternal,
+						wellknownservices.KopsControllerInternal)
 				}
 				instanceTask.FloatingIP = t
 			}
@@ -337,7 +342,8 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
 		}
 		c.AddTask(lbfipTask)
 
-		lbfipTask.WellKnownServices = append(lbfipTask.WellKnownServices, wellknownservices.KubeAPIServer)
+		lbfipTask.WellKnownServices = append(lbfipTask.WellKnownServices, wellknownservices.KubeAPIServerInternal)
+		lbfipTask.WellKnownServices = append(lbfipTask.WellKnownServices, wellknownservices.KubeAPIServerExternal)
 
 		poolTask := &openstacktasks.LBPool{
 			Name:         fi.PtrTo(fmt.Sprintf("%s-https", fi.ValueOf(lbTask.Name))),

diff --git a/...servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml b/...servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml
@@ -10,7 +10,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-1-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -19,7 +20,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-2-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -28,7 +30,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-3-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -62,7 +65,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-1-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master
 ID: null
@@ -153,7 +157,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-2-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master
 ID: null
@@ -244,7 +249,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-3-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master
 ID: null

diff --git a/...oup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml b/...oup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml
@@ -37,7 +37,8 @@ LB:
 Lifecycle: Sync
 Name: fip-api.cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-internal
+- kube-apiserver-external
 ---
 AvailabilityZone: zone-1
 ConfigDrive: false

diff --git a/.../servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml b/.../servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml
@@ -37,7 +37,8 @@ LB:
 Lifecycle: Sync
 Name: fip-master-public-name
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-internal
+- kube-apiserver-external
 ---
 AvailabilityZone: zone-1
 ConfigDrive: false

diff --git a/...l/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml b/...l/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml
@@ -22,7 +22,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-a-1-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -31,7 +32,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-b-1-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -40,7 +42,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-c-1-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -74,7 +77,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-a-1-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master-a
 ID: null
@@ -165,7 +169,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-b-1-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master-b
 ID: null
@@ -256,7 +261,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-c-1-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master-c
 ID: null

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml
@@ -10,7 +10,8 @@ LB: null
 Lifecycle: Sync
 Name: fip-master-1-cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-external
+- kube-apiserver-internal
 - kops-controller
 ---
 ID: null
@@ -30,7 +31,8 @@ FloatingIP:
   Lifecycle: Sync
   Name: fip-master-1-cluster
   WellKnownServices:
-  - kube-apiserver
+  - kube-apiserver-external
+  - kube-apiserver-internal
   - kops-controller
 GroupName: master
 ID: null

diff --git a/...up/single-zone-setup-3-masters-1-node-without-bastion-with-API-loadbalancer-dns-none.yaml b/...up/single-zone-setup-3-masters-1-node-without-bastion-with-API-loadbalancer-dns-none.yaml
@@ -31,7 +31,8 @@ LB:
 Lifecycle: Sync
 Name: fip-api.cluster
 WellKnownServices:
-- kube-apiserver
+- kube-apiserver-internal
+- kube-apiserver-external
 ---
 AvailabilityZone: zone-1
 ConfigDrive: false