Use nodeAffinity to determine if an addon supports ARM64 (#6695)

pkg/actions/nodegroup/create.go

@@ -89,7 +89,7 @@ func (m *Manager) Create(ctx context.Context, options CreateOpts, nodegroupFilte
 		return err
 	}
 
-	if err := m.checkARMSupport(ctx, ctl, rawClient, cfg, options.SkipOutdatedAddonsCheck); err != nil {
+	if err := m.checkARMSupport(ctx, rawClient, cfg, options.SkipOutdatedAddonsCheck); err != nil {
 		return err
 	}
 
@@ -287,14 +287,9 @@ func (m *Manager) postNodeCreationTasks(ctx context.Context, clientSet kubernete
 	return nil
 }
 
-func (m *Manager) checkARMSupport(ctx context.Context, ctl *eks.ClusterProvider, rawClient *kubernetes.RawClient, cfg *api.ClusterConfig, skipOutdatedAddonsCheck bool) error {
-	kubeProvider := m.ctl
-	kubernetesVersion, err := kubeProvider.ServerVersion(rawClient)
-	if err != nil {
-		return err
-	}
+func (m *Manager) checkARMSupport(ctx context.Context, rawClient *kubernetes.RawClient, cfg *api.ClusterConfig, skipOutdatedAddonsCheck bool) error {
 	if api.ClusterHasInstanceType(cfg, instanceutils.IsARMInstanceType) {
-		upToDate, err := defaultaddons.DoAddonsSupportMultiArch(ctx, ctl.AWSProvider.EKS(), rawClient, kubernetesVersion, ctl.AWSProvider.Region())
+		upToDate, err := defaultaddons.DoAddonsSupportMultiArch(ctx, rawClient.ClientSet())
 		if err != nil {
 			return err
 		}

pkg/actions/nodegroup/create_test.go

@@ -165,7 +165,6 @@ var _ = DescribeTable("Create", func(t ngEntry) {
 		},
 		expectedCalls: func(k *fakes.FakeKubeProvider, _ *utilFakes.FakeNodegroupFilter) {
 			Expect(k.NewRawClientCallCount()).To(Equal(1))
-			Expect(k.ServerVersionCallCount()).To(Equal(1))
 		},
 		expectedErr: errors.Wrap(errors.New("shared node security group missing, to fix this run 'eksctl update cluster --name=my-cluster --region='"), "cluster compatibility check failed")}),
 
@@ -176,7 +175,6 @@ var _ = DescribeTable("Create", func(t ngEntry) {
 		},
 		expectedCalls: func(k *fakes.FakeKubeProvider, f *utilFakes.FakeNodegroupFilter) {
 			Expect(k.NewRawClientCallCount()).To(Equal(1))
-			Expect(k.ServerVersionCallCount()).To(Equal(1))
 			Expect(f.SetOnlyLocalCallCount()).To(Equal(1))
 		},
 		expectedErr: errors.New("err"),
@@ -191,7 +189,6 @@ var _ = DescribeTable("Create", func(t ngEntry) {
 		},
 		expectedCalls: func(k *fakes.FakeKubeProvider, f *utilFakes.FakeNodegroupFilter) {
 			Expect(k.NewRawClientCallCount()).To(Equal(1))
-			Expect(k.ServerVersionCallCount()).To(Equal(1))
 			Expect(f.SetOnlyLocalCallCount()).To(Equal(1))
 		},
 		expectedErr: errors.New("failed to determine if aws-node uses IRSA"),
@@ -408,7 +405,6 @@ var _ = DescribeTable("Create", func(t ngEntry) {
 		},
 		expectedCalls: func(k *fakes.FakeKubeProvider, f *utilFakes.FakeNodegroupFilter) {
 			Expect(k.NewRawClientCallCount()).To(Equal(1))
-			Expect(k.ServerVersionCallCount()).To(Equal(1))
 			Expect(f.SetOnlyLocalCallCount()).To(Equal(1))
 		},
 	}),
@@ -419,7 +415,6 @@ var _ = DescribeTable("Create", func(t ngEntry) {
 		},
 		expectedCalls: func(k *fakes.FakeKubeProvider, f *utilFakes.FakeNodegroupFilter) {
 			Expect(k.NewRawClientCallCount()).To(Equal(1))
-			Expect(k.ServerVersionCallCount()).To(Equal(1))
 			Expect(f.SetOnlyLocalCallCount()).To(Equal(1))
 		},
 	}),
@@ -441,7 +436,6 @@ var _ = DescribeTable("Create", func(t ngEntry) {
 		},
 		expectedCalls: func(k *fakes.FakeKubeProvider, f *utilFakes.FakeNodegroupFilter) {
 			Expect(k.NewRawClientCallCount()).To(Equal(1))
-			Expect(k.ServerVersionCallCount()).To(Equal(1))
 			Expect(f.SetOnlyLocalCallCount()).To(Equal(1))
 		},
 	}),

pkg/addons/default/addons.go

@@ -2,10 +2,15 @@ package defaultaddons
 
 import (
 	"context"
+	"fmt"
 
-	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/kris-nova/logger"
+	"github.com/pkg/errors"
+
 	"github.com/weaveworks/eksctl/pkg/awsapi"
 	"github.com/weaveworks/eksctl/pkg/kubernetes"
 )
@@ -18,37 +23,63 @@ type AddonInput struct {
 }
 
 // DoAddonsSupportMultiArch checks if the coredns/kubeproxy/awsnode support multi arch nodegroups
-// We know that AWS node requires 1.6.3+ to work, so we check for that
-// Kubeproxy/coredns we don't know what version adds support, so we just ensure its up-to-date before proceeding.
-// TODO: we should know what versions of kubeproxy/coredns added support, rather than always erroring if they are out of date
-func DoAddonsSupportMultiArch(ctx context.Context, eksAPI awsapi.EKS, rawClient kubernetes.RawClientInterface, controlPlaneVersion string, region string) (bool, error) {
-	input := AddonInput{
-		RawClient:           rawClient,
-		ControlPlaneVersion: controlPlaneVersion,
-		Region:              region,
-		EKSAPI:              eksAPI,
-	}
-	kubeProxyUpToDate, err := IsKubeProxyUpToDate(ctx, input)
+// We know that AWS node requires 1.6.3+ to work, so we check for that.
+// For kube-proxy and CoreDNS, we do not know what version adds support, so we just ensure they contain a node affinity
+// that allows them to be scheduled on ARM64 nodes.
+func DoAddonsSupportMultiArch(ctx context.Context, clientSet kubernetes.Interface) (bool, error) {
+	kubeProxy, err := getKubeProxy(ctx, clientSet)
 	if err != nil {
-		return true, err
+		return false, err
 	}
-	if !kubeProxyUpToDate {
+
+	if kubeProxy != nil && !supportsMultiArch(kubeProxy.Spec.Template.Spec) {
 		return false, nil
 	}
 
-	awsNodeUpToDate, err := DoesAWSNodeSupportMultiArch(ctx, input)
+	awsNodeSupportsMultiArch, err := DoesAWSNodeSupportMultiArch(ctx, clientSet)
 	if err != nil {
-		return true, err
+		return false, err
 	}
-	if !awsNodeUpToDate {
+	if !awsNodeSupportsMultiArch {
 		return false, nil
 	}
 
-	coreDNSUpToDate, err := IsCoreDNSUpToDate(ctx, input)
+	coreDNS, err := getCoreDNS(ctx, clientSet)
+	if err != nil {
+		return false, err
+	}
+	return coreDNS == nil || supportsMultiArch(coreDNS.Spec.Template.Spec), nil
+}
+
+// supportsMultiArch returns true if the PodSpec contains a node affinity that allows the pod to be scheduled on
+// multiple architectures.
+func supportsMultiArch(podSec corev1.PodSpec) bool {
+	if podSec.Affinity == nil || podSec.Affinity.NodeAffinity == nil || podSec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		return false
+	}
+	for _, nodeSelectorTerm := range podSec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
+		for _, me := range nodeSelectorTerm.MatchExpressions {
+			if me.Key == corev1.LabelArchStable && me.Operator == corev1.NodeSelectorOpIn {
+				for _, val := range me.Values {
+					if val == "arm64" {
+						return true
+					}
+				}
+			}
+		}
+	}
+	return false
+}
+
+func makeGetError(err error, resourceName string) error {
 	if err != nil {
-		return true, err
+		if apierrors.IsNotFound(err) {
+			logger.Warning("%q was not found", resourceName)
+			return nil
+		}
+		return fmt.Errorf("getting %q: %w", resourceName, err)
 	}
-	return coreDNSUpToDate, nil
+	return nil
 }
 
 // LoadAsset return embedded manifest as a runtime.Object

pkg/addons/default/aws_node.go

@@ -7,12 +7,12 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/kris-nova/logger"
-	"github.com/pkg/errors"
+
 	appsv1 "k8s.io/api/apps/v1"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/weaveworks/eksctl/pkg/addons"
+	"github.com/weaveworks/eksctl/pkg/kubernetes"
 
 	// For go:embed
 	_ "embed"
@@ -30,14 +30,13 @@ const (
 var latestAWSNodeYaml []byte
 
 // DoesAWSNodeSupportMultiArch makes sure awsnode supports ARM nodes
-func DoesAWSNodeSupportMultiArch(ctx context.Context, input AddonInput) (bool, error) {
-	clusterDaemonSet, err := input.RawClient.ClientSet().AppsV1().DaemonSets(metav1.NamespaceSystem).Get(ctx, AWSNode, metav1.GetOptions{})
+func DoesAWSNodeSupportMultiArch(ctx context.Context, clientSet kubernetes.Interface) (bool, error) {
+	clusterDaemonSet, err := getAWSNode(ctx, clientSet)
 	if err != nil {
-		if apierrs.IsNotFound(err) {
-			logger.Warning("%q was not found", AWSNode)
-			return true, nil
-		}
-		return false, errors.Wrapf(err, "getting %q", AWSNode)
+		return false, err
+	}
+	if clusterDaemonSet == nil {
+		return true, nil
 	}
 
 	minVersion := semver.Version{
@@ -71,13 +70,12 @@ func DoesAWSNodeSupportMultiArch(ctx context.Context, input AddonInput) (bool, e
 // UpdateAWSNode will update the `aws-node` add-on and returns true
 // if an update is available.
 func UpdateAWSNode(ctx context.Context, input AddonInput, plan bool) (bool, error) {
-	clusterDaemonSet, err := input.RawClient.ClientSet().AppsV1().DaemonSets(metav1.NamespaceSystem).Get(ctx, AWSNode, metav1.GetOptions{})
+	clusterDaemonSet, err := getAWSNode(ctx, input.RawClient.ClientSet())
 	if err != nil {
-		if apierrs.IsNotFound(err) {
-			logger.Warning("%q was not found", AWSNode)
-			return false, nil
-		}
-		return false, errors.Wrapf(err, "getting %q", AWSNode)
+		return false, err
+	}
+	if clusterDaemonSet == nil {
+		return false, nil
 	}
 
 	resourceList, err := newList(latestAWSNodeYaml)
@@ -171,3 +169,8 @@ func UpdateAWSNode(ctx context.Context, input AddonInput, plan bool) (bool, erro
 	logger.Info("%q is now up-to-date", AWSNode)
 	return false, nil
 }
+
+func getAWSNode(ctx context.Context, clientSet kubernetes.Interface) (*appsv1.DaemonSet, error) {
+	d, err := clientSet.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(ctx, AWSNode, metav1.GetOptions{})
+	return d, makeGetError(err, AWSNode)
+}

pkg/addons/default/aws_node_test.go

@@ -5,6 +5,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
 	v1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -33,7 +34,7 @@ var _ = Describe("AWS Node", func() {
 			input.ControlPlaneVersion = "1.15.0"
 			rawClient.AssumeObjectsMissing = false
 
-			needsUpdate, err := da.DoesAWSNodeSupportMultiArch(context.Background(), input)
+			needsUpdate, err := da.DoesAWSNodeSupportMultiArch(context.Background(), rawClient.ClientSet())
 			Expect(err).NotTo(HaveOccurred())
 			Expect(needsUpdate).To(BeFalse())
 		})
@@ -42,7 +43,7 @@ var _ = Describe("AWS Node", func() {
 			loadSamples(rawClient, "testdata/sample-1.16-eksbuild.1.json")
 			rawClient.AssumeObjectsMissing = false
 
-			needsUpdate, err := da.DoesAWSNodeSupportMultiArch(context.Background(), input)
+			needsUpdate, err := da.DoesAWSNodeSupportMultiArch(context.Background(), rawClient.ClientSet())
 			Expect(err).NotTo(HaveOccurred())
 			Expect(needsUpdate).To(BeTrue())
 		})
@@ -51,7 +52,7 @@ var _ = Describe("AWS Node", func() {
 			loadSamples(rawClient, "testdata/sample-1.16-v1.7.json")
 			rawClient.AssumeObjectsMissing = false
 
-			needsUpdate, err := da.DoesAWSNodeSupportMultiArch(context.Background(), input)
+			needsUpdate, err := da.DoesAWSNodeSupportMultiArch(context.Background(), rawClient.ClientSet())
 			Expect(err).NotTo(HaveOccurred())
 			Expect(needsUpdate).To(BeTrue())
 		})

pkg/addons/default/coredns.go

@@ -8,15 +8,16 @@ import (
 
 	"github.com/kris-nova/logger"
 	"github.com/pkg/errors"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/weaveworks/eksctl/pkg/addons"
-
 	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
 	"github.com/weaveworks/eksctl/pkg/fargate/coredns"
+	"github.com/weaveworks/eksctl/pkg/kubernetes"
 
 	// For go:embed
 	_ "embed"
@@ -32,55 +33,6 @@ const (
 //go:embed assets/coredns*.json
 var coreDNSDir embed.FS
 
-func IsCoreDNSUpToDate(ctx context.Context, input AddonInput) (bool, error) {
-	kubeDNSDeployment, err := input.RawClient.ClientSet().AppsV1().Deployments(metav1.NamespaceSystem).Get(ctx, CoreDNS, metav1.GetOptions{})
-	if err != nil {
-		if apierrs.IsNotFound(err) {
-			logger.Warning("%q was not found", CoreDNS)
-			return true, nil
-		}
-		return false, errors.Wrapf(err, "getting %q", CoreDNS)
-	}
-
-	// if Deployment is present, go through our list of assets
-	list, err := loadAssetCoreDNS(input.ControlPlaneVersion)
-	if err != nil {
-		return false, err
-	}
-
-	for _, rawObj := range list.Items {
-		resource, err := input.RawClient.NewRawResource(rawObj.Object)
-		if err != nil {
-			return false, err
-		}
-		if resource.GVK.Kind != "Deployment" {
-			continue
-		}
-		if resource.Info.Name != "coredns" {
-			continue
-		}
-		deployment, ok := resource.Info.Object.(*appsv1.Deployment)
-		if !ok {
-			return false, fmt.Errorf("expected type %T; got %T", &appsv1.Deployment{}, resource.Info.Object)
-		}
-		if err := addons.UseRegionalImage(&deployment.Spec.Template, input.Region); err != nil {
-			return false, err
-		}
-		if computeType, ok := kubeDNSDeployment.Spec.Template.Annotations[coredns.ComputeTypeAnnotationKey]; ok {
-			deployment.Spec.Template.Annotations[coredns.ComputeTypeAnnotationKey] = computeType
-		}
-		tagMismatch, err := addons.ImageTagsDiffer(
-			deployment.Spec.Template.Spec.Containers[0].Image,
-			kubeDNSDeployment.Spec.Template.Spec.Containers[0].Image,
-		)
-		if err != nil {
-			return false, err
-		}
-		return !tagMismatch, err
-	}
-	return true, nil
-}
-
 // UpdateCoreDNS will update the `coredns` add-on and returns true
 // if an update is available
 func UpdateCoreDNS(ctx context.Context, input AddonInput, plan bool) (bool, error) {
@@ -165,6 +117,11 @@ func UpdateCoreDNS(ctx context.Context, input AddonInput, plan bool) (bool, erro
 	return false, nil
 }
 
+func getCoreDNS(ctx context.Context, clientSet kubernetes.Interface) (*appsv1.Deployment, error) {
+	d, err := clientSet.AppsV1().Deployments(metav1.NamespaceSystem).Get(ctx, CoreDNS, metav1.GetOptions{})
+	return d, makeGetError(err, CoreDNS)
+}
+
 func loadAssetCoreDNS(controlPlaneVersion string) (*metav1.List, error) {
 	if strings.HasPrefix(controlPlaneVersion, "1.10.") {
 		return nil, errors.New("CoreDNS is not supported on Kubernetes 1.10")

pkg/addons/default/coredns_test.go

@@ -68,34 +68,6 @@ var _ = Describe("default addons - coredns", func() {
 			)
 		})
 	})
-
-	Context("IsCoreDNSUpToDate", func() {
-		BeforeEach(func() {
-			createCoreDNSFromTestSample(rawClient, kubernetesVersion)
-			_, err := da.UpdateCoreDNS(context.Background(), input, false)
-			Expect(err).NotTo(HaveOccurred())
-		})
-
-		Context("when CoreDNS is NOT up to date", func() {
-			BeforeEach(func() {
-				input.ControlPlaneVersion = "1.22.x"
-			})
-
-			It("reports 'false'", func() {
-				isUpToDate, err := da.IsCoreDNSUpToDate(context.Background(), input)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(isUpToDate).To(Equal(false))
-			})
-		})
-
-		Context("when CoreDNS is up to date", func() {
-			It("reports 'true'", func() {
-				isUpToDate, err := da.IsCoreDNSUpToDate(context.Background(), input)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(isUpToDate).To(Equal(true))
-			})
-		})
-	})
 })
 
 func createCoreDNSFromTestSample(rawClient *testutils.FakeRawClient, kubernetesVersion string) {