cherry-pick: Support AWS RGT APIs with feature flag (kubernetes-sigs#…

…3186) (kubernetes-sigs#3193)

* Add support for RGT APIs with feature flag

* revert change in iam policies and refactor tagging manager

* list resrouces by arn in RGT

* handle potential replication in returned RGT resources

controllers/ingress/group_controller.go

@@ -54,7 +54,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
 	enhancedBackendBuilder := ingress.NewDefaultEnhancedBackendBuilder(k8sClient, annotationParser, authConfigBuilder)
 	referenceIndexer := ingress.NewDefaultReferenceIndexer(enhancedBackendBuilder, authConfigBuilder, logger)
 	trackingProvider := tracking.NewDefaultProvider(ingressTagPrefix, controllerConfig.ClusterName)
-	elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, logger)
+	elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, cloud.RGT(), logger)
 	modelBuilder := ingress.NewDefaultModelBuilder(k8sClient, eventRecorder,
 		cloud.EC2(), cloud.ACM(),
 		annotationParser, subnetsResolver,

controllers/service/service_controller.go

@@ -41,7 +41,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
 
 	annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
 	trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
-	elbv2TaggingManager := elbv2.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, logger)
+	elbv2TaggingManager := elbv2.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, cloud.RGT(), logger)
 	serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
 	modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
 		elbv2TaggingManager, controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils)

docs/deploy/configurations.md

@@ -158,5 +158,6 @@ They are a set of kye=value pairs that describe AWS load balance controller feat
 | EndpointsFailOpen                     | string                          | true           | Enable or disable allowing endpoints with `ready:unknown` state in the target groups. |
 | EnableServiceController               | string                          | true           | Toggles support for `Service` type resources. |
 | EnableIPTargetType                    | string                          | true           | Used to toggle support for target-type `ip` across `Ingress` and `Service` type resources. |
+| EnableRGTAPI                       | string                          | false          | If enabled, the tagging manager will describe resource tags via RGT APIs, otherwise via ELB APIs. In order to enable RGT API, `tag:GetResources` is needed in controller IAM policy. |
 | SubnetsClusterTagCheck                | string                          | true           | Enable or disable the check for `kubernetes.io/cluster/${cluster-name}` during subnet auto-discovery |
 | NLBHealthCheckAdvancedConfiguration   | string                          | true           | Enable or disable advanced health check configuration for NLB, for example health check timeout |

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/aws/aws-sdk-go v1.44.184
+	github.com/aws/aws-sdk-go-v2 v0.18.0
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/gavv/httpexpect/v2 v2.9.0
 	github.com/go-logr/logr v1.2.3

go.sum

@@ -79,6 +79,9 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo=
 github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
+github.com/aws/aws-sdk-go-v2 v0.18.0 h1:qZ+woO4SamnH/eEbjM2IDLhRNwIwND/RQyVlBLp3Jqg=
+github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

pkg/aws/services/rgt.go

@@ -1,13 +1,22 @@
 package services
 
 import (
+	"context"
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
 )
 
+const (
+	ResourceTypeELBTargetGroup  = "elasticloadbalancing:targetgroup"
+	ResourceTypeELBLoadBalancer = "elasticloadbalancing:loadbalancer"
+)
+
 type RGT interface {
 	resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
+
+	GetResourcesAsList(ctx context.Context, input *resourcegroupstaggingapi.GetResourcesInput) ([]*resourcegroupstaggingapi.ResourceTagMapping, error)
 }
 
 // NewRGT constructs new RGT implementation.
@@ -17,6 +26,29 @@ func NewRGT(session *session.Session) RGT {
 	}
 }
 
+var _ RGT = (*defaultRGT)(nil)
+
 type defaultRGT struct {
 	resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
 }
+
+func (c *defaultRGT) GetResourcesAsList(ctx context.Context, input *resourcegroupstaggingapi.GetResourcesInput) ([]*resourcegroupstaggingapi.ResourceTagMapping, error) {
+	var result []*resourcegroupstaggingapi.ResourceTagMapping
+	if err := c.GetResourcesPagesWithContext(ctx, input, func(output *resourcegroupstaggingapi.GetResourcesOutput, _ bool) bool {
+		for _, i := range output.ResourceTagMappingList {
+			result = append(result, i)
+		}
+		return true
+	}); err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func ParseRGTTags(tags []*resourcegroupstaggingapi.Tag) map[string]string {
+	result := make(map[string]string, len(tags))
+	for _, tag := range tags {
+		result[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
+	}
+	return result
+}

pkg/config/feature_gates.go

@@ -17,6 +17,7 @@ const (
 	EndpointsFailOpen            Feature = "EndpointsFailOpen"
 	EnableServiceController      Feature = "EnableServiceController"
 	EnableIPTargetType           Feature = "EnableIPTargetType"
+	EnableRGTAPI                 Feature = "EnableRGTAPI"
 	SubnetsClusterTagCheck       Feature = "SubnetsClusterTagCheck"
 	NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig"
 )
@@ -52,6 +53,7 @@ func NewFeatureGates() FeatureGates {
 			EndpointsFailOpen:            true,
 			EnableServiceController:      true,
 			EnableIPTargetType:           true,
+			EnableRGTAPI:                 false,
 			SubnetsClusterTagCheck:       true,
 			NLBHealthCheckAdvancedConfig: true,
 		},

pkg/deploy/elbv2/tagging_manager.go

@@ -2,9 +2,12 @@ package elbv2
 
 import (
 	"context"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	awssdk "github.com/aws/aws-sdk-go/aws"
 	elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
@@ -93,26 +96,28 @@ type TaggingManager interface {
 }
 
 // NewDefaultTaggingManager constructs default TaggingManager.
-func NewDefaultTaggingManager(elbv2Client services.ELBV2, vpcID string, featureGates config.FeatureGates, logger logr.Logger) *defaultTaggingManager {
+func NewDefaultTaggingManager(elbv2Client services.ELBV2, vpcID string, featureGates config.FeatureGates, rgt services.RGT, logger logr.Logger) *defaultTaggingManager {
 	return &defaultTaggingManager{
 		elbv2Client:           elbv2Client,
 		vpcID:                 vpcID,
 		featureGates:          featureGates,
 		logger:                logger,
 		describeTagsChunkSize: defaultDescribeTagsChunkSize,
+		rgt:                   rgt,
 	}
 }
 
 var _ TaggingManager = &defaultTaggingManager{}
 
 // default implementation for TaggingManager
-// @TODO: use AWS Resource Groups Tagging API to optimize this implementation once it have PrivateLink support.
+// @TODO: deprecate ELB API and only use AWS Resource Groups Tagging API to optimize this implementation once RGT has PrivateLink support.
 type defaultTaggingManager struct {
 	elbv2Client           services.ELBV2
 	vpcID                 string
 	featureGates          config.FeatureGates
 	logger                logr.Logger
 	describeTagsChunkSize int
+	rgt                   services.RGT
 }
 
 func (m *defaultTaggingManager) ReconcileTags(ctx context.Context, arn string, desiredTags map[string]string, opts ...ReconcileTagsOption) error {
@@ -123,7 +128,7 @@ func (m *defaultTaggingManager) ReconcileTags(ctx context.Context, arn string, d
 	reconcileOpts.ApplyOptions(opts)
 	currentTags := reconcileOpts.CurrentTags
 	if currentTags == nil {
-		tagsByARN, err := m.describeResourceTags(ctx, []string{arn})
+		tagsByARN, err := m.describeResourceTagsNative(ctx, []string{arn})
 		if err != nil {
 			return err
 		}
@@ -188,7 +193,7 @@ func (m *defaultTaggingManager) ListListeners(ctx context.Context, lbARN string)
 	}
 	var tagsByARN map[string]map[string]string
 	if m.featureGates.Enabled(config.ListenerRulesTagging) {
-		tagsByARN, err = m.describeResourceTags(ctx, lsARNs)
+		tagsByARN, err = m.describeResourceTagsNative(ctx, lsARNs)
 		if err != nil {
 			return nil, err
 		}
@@ -221,7 +226,7 @@ func (m *defaultTaggingManager) ListListenerRules(ctx context.Context, lsARN str
 	}
 	var tagsByARN map[string]map[string]string
 	if m.featureGates.Enabled(config.ListenerRulesTagging) {
-		tagsByARN, err = m.describeResourceTags(ctx, lrARNs)
+		tagsByARN, err = m.describeResourceTagsNative(ctx, lrARNs)
 		if err != nil {
 			return nil, err
 		}
@@ -238,12 +243,64 @@ func (m *defaultTaggingManager) ListListenerRules(ctx context.Context, lsARN str
 }
 
 func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilters ...tracking.TagFilter) ([]LoadBalancerWithTags, error) {
+	if m.featureGates.Enabled(config.EnableRGTAPI) {
+		return m.listLoadBalancersRGT(ctx, tagFilters)
+	}
+	return m.listLoadBalancersNative(ctx, tagFilters)
+}
+
+func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters ...tracking.TagFilter) ([]TargetGroupWithTags, error) {
+	if m.featureGates.Enabled(config.EnableRGTAPI) {
+		return m.listTargetGroupsRGT(ctx, tagFilters)
+	}
+	return m.listTargetGroupsNative(ctx, tagFilters)
+
+}
+
+func (m *defaultTaggingManager) listLoadBalancersRGT(ctx context.Context, tagFilters []tracking.TagFilter) ([]LoadBalancerWithTags, error) {
+	// use a map to avoid potential duplication in returned resources
+	resourceTagsByARN := make(map[string][]*resourcegroupstaggingapi.Tag)
+	for _, tagFilter := range tagFilters {
+		req := &resourcegroupstaggingapi.GetResourcesInput{
+			TagFilters:          convertTagFiltersToRGTTagFilters(tagFilter),
+			ResourceTypeFilters: aws.StringSlice([]string{services.ResourceTypeELBLoadBalancer}),
+		}
+		resources, err := m.rgt.GetResourcesAsList(ctx, req)
+		if err != nil {
+			return nil, err
+		}
+		for _, resource := range resources {
+			if _, exists := resourceTagsByARN[aws.StringValue(resource.ResourceARN)]; !exists {
+				resourceTagsByARN[aws.StringValue(resource.ResourceARN)] = resource.Tags
+			}
+		}
+	}
+	var matchedLBs []LoadBalancerWithTags
+	for resourceARN, resourceTags := range resourceTagsByARN {
+		elbv2Req := &elbv2sdk.DescribeLoadBalancersInput{
+			LoadBalancerArns: []*string{&resourceARN},
+		}
+		elbv2Resp, err := m.elbv2Client.DescribeLoadBalancersAsList(ctx, elbv2Req)
+		if err != nil {
+			return nil, err
+		}
+		if len(elbv2Resp) == 0 {
+			return nil, errors.Errorf("no load balancer found for the arn: %v", resourceARN)
+		}
+		matchedLBs = append(matchedLBs, LoadBalancerWithTags{
+			LoadBalancer: elbv2Resp[0],
+			Tags:         services.ParseRGTTags(resourceTags),
+		})
+	}
+	return matchedLBs, nil
+}
+
+func (m *defaultTaggingManager) listLoadBalancersNative(ctx context.Context, tagFilters []tracking.TagFilter) ([]LoadBalancerWithTags, error) {
 	req := &elbv2sdk.DescribeLoadBalancersInput{}
 	lbs, err := m.elbv2Client.DescribeLoadBalancersAsList(ctx, req)
 	if err != nil {
 		return nil, err
 	}
-
 	lbARNsWithinVPC := make([]string, 0, len(lbs))
 	lbByARNWithinVPC := make(map[string]*elbv2sdk.LoadBalancer, len(lbs))
 	for _, lb := range lbs {
@@ -254,7 +311,7 @@ func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilter
 		lbARNsWithinVPC = append(lbARNsWithinVPC, lbARN)
 		lbByARNWithinVPC[lbARN] = lb
 	}
-	tagsByARN, err := m.describeResourceTags(ctx, lbARNsWithinVPC)
+	tagsByARN, err := m.describeResourceTagsNative(ctx, lbARNsWithinVPC)
 	if err != nil {
 		return nil, err
 	}
@@ -279,7 +336,45 @@ func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilter
 	return matchedLBs, nil
 }
 
-func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters ...tracking.TagFilter) ([]TargetGroupWithTags, error) {
+func (m *defaultTaggingManager) listTargetGroupsRGT(ctx context.Context, tagFilters []tracking.TagFilter) ([]TargetGroupWithTags, error) {
+	// use a map to avoid potential duplication in returned resources
+	resourceTagsByARN := make(map[string][]*resourcegroupstaggingapi.Tag)
+	for _, tagFilter := range tagFilters {
+		req := &resourcegroupstaggingapi.GetResourcesInput{
+			TagFilters:          convertTagFiltersToRGTTagFilters(tagFilter),
+			ResourceTypeFilters: aws.StringSlice([]string{services.ResourceTypeELBTargetGroup}),
+		}
+		resources, err := m.rgt.GetResourcesAsList(ctx, req)
+		if err != nil {
+			return nil, err
+		}
+		for _, resource := range resources {
+			if _, exists := resourceTagsByARN[aws.StringValue(resource.ResourceARN)]; !exists {
+				resourceTagsByARN[aws.StringValue(resource.ResourceARN)] = resource.Tags
+			}
+		}
+	}
+	var matchedTGs []TargetGroupWithTags
+	for resourceARN, resourceTags := range resourceTagsByARN {
+		elbv2Req := &elbv2sdk.DescribeTargetGroupsInput{
+			TargetGroupArns: []*string{&resourceARN},
+		}
+		elbv2Resp, err := m.elbv2Client.DescribeTargetGroupsAsList(ctx, elbv2Req)
+		if err != nil {
+			return nil, err
+		}
+		if len(elbv2Resp) == 0 {
+			return nil, errors.Errorf("no target group found for the arn: %v", resourceARN)
+		}
+		matchedTGs = append(matchedTGs, TargetGroupWithTags{
+			TargetGroup: elbv2Resp[0],
+			Tags:        services.ParseRGTTags(resourceTags),
+		})
+	}
+	return matchedTGs, nil
+}
+
+func (m *defaultTaggingManager) listTargetGroupsNative(ctx context.Context, tagFilters []tracking.TagFilter) ([]TargetGroupWithTags, error) {
 	req := &elbv2sdk.DescribeTargetGroupsInput{}
 	tgs, err := m.elbv2Client.DescribeTargetGroupsAsList(ctx, req)
 	if err != nil {
@@ -296,7 +391,7 @@ func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters
 		tgARNsWithinVPC = append(tgARNsWithinVPC, tgARN)
 		tgByARNWithinVPC[tgARN] = tg
 	}
-	tagsByARN, err := m.describeResourceTags(ctx, tgARNsWithinVPC)
+	tagsByARN, err := m.describeResourceTagsNative(ctx, tgARNsWithinVPC)
 	if err != nil {
 		return nil, err
 	}
@@ -321,9 +416,9 @@ func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters
 	return matchedTGs, nil
 }
 
-// describeResourceTags describes tags for elbv2 resources.
+// describeResourceTagsNative describes tags for elbv2 resources.
 // returns tags indexed by resource ARN.
-func (m *defaultTaggingManager) describeResourceTags(ctx context.Context, arns []string) (map[string]map[string]string, error) {
+func (m *defaultTaggingManager) describeResourceTagsNative(ctx context.Context, arns []string) (map[string]map[string]string, error) {
 	tagsByARN := make(map[string]map[string]string, len(arns))
 	arnsChunks := algorithm.ChunkStrings(arns, m.describeTagsChunkSize)
 	for _, arnsChunk := range arnsChunks {
@@ -365,3 +460,15 @@ func convertSDKTagsToTags(sdkTags []*elbv2sdk.Tag) map[string]string {
 	}
 	return tags
 }
+
+// convert tagFilters to RGTTagFilters
+func convertTagFiltersToRGTTagFilters(tagFilter tracking.TagFilter) []*resourcegroupstaggingapi.TagFilter {
+	var RGTTagFilters []*resourcegroupstaggingapi.TagFilter
+	for k, v := range tagFilter {
+		RGTTagFilters = append(RGTTagFilters, &resourcegroupstaggingapi.TagFilter{
+			Key:    aws.String(k),
+			Values: aws.StringSlice(v),
+		})
+	}
+	return RGTTagFilters
+}