From 730ba73f6a29ff86cc24f3d214aa3ae668dbdda9 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Tue, 13 May 2025 11:53:19 -0400 Subject: [PATCH 1/2] add allowlist to awsentity processor for container insights to only append platform type --- plugins/processors/awsentity/config.go | 3 ++ plugins/processors/awsentity/processor.go | 50 +++++++++---------- plugins/processors/awsentity/util.go | 13 ++++- .../appsignals_and_eks_config.yaml | 9 ++-- .../appsignals_and_k8s_config.yaml | 1 + .../appsignals_fallback_and_eks_config.yaml | 1 + .../appsignals_over_fallback_config.yaml | 1 + .../base_container_insights_config.yaml | 1 + .../sampleConfig/container_insights_jmx.yaml | 1 + .../emf_and_kubernetes_config.yaml | 1 + .../emf_and_kubernetes_with_gpu_config.yaml | 1 + .../emf_and_kubernetes_with_kueue_config.yaml | 1 + .../kubernetes_on_prem_config.yaml | 1 + .../kueue_container_insights_config.yaml | 1 + .../logs_and_kubernetes_config.yaml | 1 + .../pipeline/containerinsights/translator.go | 7 ++- .../otel/processor/awsentity/translator.go | 19 +++++++ 17 files changed, 80 insertions(+), 32 deletions(-) diff --git a/plugins/processors/awsentity/config.go b/plugins/processors/awsentity/config.go index 54c662a088..a0afc32eb8 100644 --- a/plugins/processors/awsentity/config.go +++ b/plugins/processors/awsentity/config.go @@ -23,6 +23,9 @@ type Config struct { // EntityType determines the type of entity processing done for // telemetry. Possible values are Service and Resource EntityType string `mapstructure:"entity_type,omitempty"` + // AttributeAllowList is a list of entity resource attributes to append to the metric + // If this is not specified, the processor will append all entity attributes + AttributeAllowList []string `mapstructure:"attribute_allow_list,omitempty"` } // Verify Config implements Processor interface. diff --git a/plugins/processors/awsentity/processor.go b/plugins/processors/awsentity/processor.go index 2a8eb1744f..a0f2cde51e 100644 --- a/plugins/processors/awsentity/processor.go +++ b/plugins/processors/awsentity/processor.go @@ -153,9 +153,9 @@ func (p *awsEntityProcessor) processMetrics(ctx context.Context, md pmetric.Metr if p.config.KubernetesMode != "" { switch p.config.KubernetesMode { case config.ModeEKS: - resourceAttrs.PutStr(entityattributes.AttributeEntityPlatformType, entityattributes.AttributeEntityEKSPlatform) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityPlatformType, entityattributes.AttributeEntityEKSPlatform) default: - resourceAttrs.PutStr(entityattributes.AttributeEntityPlatformType, entityattributes.AttributeEntityK8sPlatform) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityPlatformType, entityattributes.AttributeEntityK8sPlatform) } } else if p.config.Platform == config.ModeEC2 { // ec2tagger processor may have picked up the ASG name from an ec2:DescribeTags call @@ -166,11 +166,11 @@ func (p *awsEntityProcessor) processMetrics(ctx context.Context, md pmetric.Metr } ec2Info = getEC2InfoFromEntityStore() if ec2Info.GetInstanceID() != EMPTY { - resourceAttrs.PutStr(entityattributes.AttributeEntityType, entityattributes.AttributeEntityAWSResource) - resourceAttrs.PutStr(entityattributes.AttributeEntityResourceType, entityattributes.AttributeEntityEC2InstanceResource) - resourceAttrs.PutStr(entityattributes.AttributeEntityIdentifier, ec2Info.GetInstanceID()) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityType, entityattributes.AttributeEntityAWSResource) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityResourceType, entityattributes.AttributeEntityEC2InstanceResource) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityIdentifier, ec2Info.GetInstanceID()) } - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAwsAccountId, ec2Info.GetAccountID()) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAwsAccountId, ec2Info.GetAccountID()) } case entityattributes.Service: if logGroupNamesAttr, ok := resourceAttrs.Get(attributeAwsLogGroupNames); ok { @@ -249,22 +249,22 @@ func (p *awsEntityProcessor) processMetrics(ctx context.Context, md pmetric.Metr InstanceId: ec2Info.GetInstanceID(), ServiceNameSource: entityServiceNameSource, } - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityType, entityattributes.Service) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceName, entityServiceName) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityDeploymentEnvironment, entityEnvironmentName) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityType, entityattributes.Service) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceName, entityServiceName) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityDeploymentEnvironment, entityEnvironmentName) if err := validate.Struct(eksAttributes); err == nil { - resourceAttrs.PutStr(entityattributes.AttributeEntityPlatformType, entityPlatformType) - resourceAttrs.PutStr(entityattributes.AttributeEntityCluster, eksAttributes.Cluster) - resourceAttrs.PutStr(entityattributes.AttributeEntityNamespace, eksAttributes.Namespace) - resourceAttrs.PutStr(entityattributes.AttributeEntityWorkload, eksAttributes.Workload) - resourceAttrs.PutStr(entityattributes.AttributeEntityNode, eksAttributes.Node) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityPlatformType, entityPlatformType) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityCluster, eksAttributes.Cluster) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityNamespace, eksAttributes.Namespace) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityWorkload, eksAttributes.Workload) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityNode, eksAttributes.Node) //Add Instance id attribute only if the application node is same as agent node if eksAttributes.Node == os.Getenv("K8S_NODE_NAME") { - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityInstanceID, eksAttributes.InstanceId) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityInstanceID, eksAttributes.InstanceId) } - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAwsAccountId, ec2Info.GetAccountID()) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceNameSource, entityServiceNameSource) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAwsAccountId, ec2Info.GetAccountID()) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceNameSource, entityServiceNameSource) } p.k8sscraper.Reset() } else if p.config.Platform == config.ModeEC2 { @@ -290,10 +290,10 @@ func (p *awsEntityProcessor) processMetrics(ctx context.Context, md pmetric.Metr } } - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityType, entityattributes.Service) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceName, entityServiceName) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityDeploymentEnvironment, entityEnvironmentName) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAwsAccountId, ec2Info.GetAccountID()) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityType, entityattributes.Service) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceName, entityServiceName) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityDeploymentEnvironment, entityEnvironmentName) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAwsAccountId, ec2Info.GetAccountID()) ec2Attributes := EC2ServiceAttributes{ InstanceId: ec2Info.GetInstanceID(), @@ -301,10 +301,10 @@ func (p *awsEntityProcessor) processMetrics(ctx context.Context, md pmetric.Metr ServiceNameSource: entityServiceNameSource, } if err := validate.Struct(ec2Attributes); err == nil { - resourceAttrs.PutStr(entityattributes.AttributeEntityPlatformType, entityPlatformType) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityInstanceID, ec2Attributes.InstanceId) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAutoScalingGroup, ec2Attributes.AutoScalingGroup) - AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceNameSource, ec2Attributes.ServiceNameSource) + p.PutAttribute(resourceAttrs, entityattributes.AttributeEntityPlatformType, entityPlatformType) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityInstanceID, ec2Attributes.InstanceId) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityAutoScalingGroup, ec2Attributes.AutoScalingGroup) + p.AddAttributeIfNonEmpty(resourceAttrs, entityattributes.AttributeEntityServiceNameSource, ec2Attributes.ServiceNameSource) } } if logGroupNames == EMPTY || (serviceName == EMPTY && environmentName == EMPTY) { diff --git a/plugins/processors/awsentity/util.go b/plugins/processors/awsentity/util.go index 804a97c14c..c7a447a600 100644 --- a/plugins/processors/awsentity/util.go +++ b/plugins/processors/awsentity/util.go @@ -5,8 +5,17 @@ package awsentity import "go.opentelemetry.io/collector/pdata/pcommon" -func AddAttributeIfNonEmpty(p pcommon.Map, key string, value string) { +func (p *awsEntityProcessor) PutAttribute(resourceAttributes pcommon.Map, k string, v string) { + attributeAllowList := p.config.AttributeAllowList + for _, allowedAttribute := range attributeAllowList { + if k == allowedAttribute { + resourceAttributes.PutStr(k, v) + } + } +} + +func (p *awsEntityProcessor) AddAttributeIfNonEmpty(resourceAttributes pcommon.Map, key string, value string) { if value != "" { - p.PutStr(key, value) + p.PutAttribute(resourceAttributes, key, value) } } diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index a54b0d8cad..6b0c037c18 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -334,10 +334,11 @@ processors: kubernetes_mode: EKS platform: ec2 awsentity/resource/containerinsights: - cluster_name: TestCluster - entity_type: Resource - kubernetes_mode: EKS - platform: ec2 + cluster_name: TestCluster + entity_type: Resource + kubernetes_mode: EKS + platform: ec2 + attribute_allow_list: ["com.amazonaws.cloudwatch.entity.internal.platform.type"] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index 6dabfc05a6..dae82eaef8 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -339,6 +339,7 @@ processors: entity_type: Resource kubernetes_mode: K8sEC2 platform: ec2 + attribute_allow_list: ["com.amazonaws.cloudwatch.entity.internal.platform.type"] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index e2cb6bc5b3..6b0c037c18 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -338,6 +338,7 @@ processors: entity_type: Resource kubernetes_mode: EKS platform: ec2 + attribute_allow_list: ["com.amazonaws.cloudwatch.entity.internal.platform.type"] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index e2cb6bc5b3..6b0c037c18 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -338,6 +338,7 @@ processors: entity_type: Resource kubernetes_mode: EKS platform: ec2 + attribute_allow_list: ["com.amazonaws.cloudwatch.entity.internal.platform.type"] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index 29e23aec60..79d0d5147f 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -163,6 +163,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: ec2 + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index c519018c8d..6a339fb2c7 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -198,6 +198,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: ec2 + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 212b2ef389..bb65375b0e 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -436,6 +436,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: onPremise + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml index 890abf47ef..12721935ab 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml @@ -710,6 +710,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: onPremise + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml index a23e854fcd..b6c380dab9 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml @@ -517,6 +517,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: onPremise + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 7f88e84d47..38318e89ba 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -403,6 +403,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: onPremise + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml index bf9a2fa674..40c23bb967 100644 --- a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml @@ -242,6 +242,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: ec2 + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index e4ebe4d40c..ddeb2fd22b 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -430,6 +430,7 @@ processors: awsentity/resource/containerinsights: entity_type: Resource platform: ec2 + attribute_allow_list: [ "com.amazonaws.cloudwatch.entity.internal.platform.type" ] batch/containerinsights: metadata_cardinality_limit: 1000 send_batch_max_size: 0 diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index 91a6fd6b10..c1c996e352 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -10,6 +10,7 @@ import ( "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/pipeline" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/entityattributes" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" @@ -31,6 +32,10 @@ var ( baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) eksKey = common.ConfigKey(baseKey, common.KubernetesKey) ecsKey = common.ConfigKey(baseKey, common.ECSKey) + + entityAllowList = []string{ + entityattributes.AttributeEntityPlatformType, + } ) type translator struct { @@ -78,7 +83,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators switch t.pipelineName { case ciPipelineName: if conf.IsSet(eksKey) { - processors.Set(awsentity.NewTranslatorWithEntityType(awsentity.Resource, common.PipelineNameContainerInsights, false)) + processors.Set(awsentity.NewTranslatorWithEntityTypeAndAllowList(awsentity.Resource, common.PipelineNameContainerInsights, false, entityAllowList)) } // add aws container insights receiver receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) diff --git a/translator/translate/otel/processor/awsentity/translator.go b/translator/translate/otel/processor/awsentity/translator.go index f43df46629..e9ad222dec 100644 --- a/translator/translate/otel/processor/awsentity/translator.go +++ b/translator/translate/otel/processor/awsentity/translator.go @@ -27,6 +27,7 @@ type translator struct { entityType string name string scrapeDatapointAttribute bool + attributeAllowList []string } func NewTranslator() common.ComponentTranslator { @@ -49,6 +50,21 @@ func NewTranslatorWithEntityType(entityType string, name string, scrapeDatapoint } } +func NewTranslatorWithEntityTypeAndAllowList(entityType string, name string, scrapeDatapointAttribute bool, allowList []string) common.ComponentTranslator { + pipelineName := strings.ToLower(entityType) + if name != "" { + pipelineName = pipelineName + "/" + name + } + + return &translator{ + factory: awsentity.NewFactory(), + entityType: entityType, + name: pipelineName, + scrapeDatapointAttribute: scrapeDatapointAttribute, + attributeAllowList: allowList, + } +} + func (t *translator) ID() component.ID { return component.NewIDWithName(t.factory.Type(), t.name) } @@ -88,5 +104,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { // processor can perform different logics for EKS // in EC2 or Non-EC2 cfg.Platform = ctx.Mode() + + cfg.AttributeAllowList = t.attributeAllowList + return cfg, nil } From 75bff36a16b91b95509843dc39c8cfc53717b444 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Tue, 13 May 2025 15:51:39 -0400 Subject: [PATCH 2/2] Make sure we check the length of the allowlist --- plugins/processors/awsentity/util.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/processors/awsentity/util.go b/plugins/processors/awsentity/util.go index c7a447a600..cd4da9e783 100644 --- a/plugins/processors/awsentity/util.go +++ b/plugins/processors/awsentity/util.go @@ -3,14 +3,15 @@ package awsentity -import "go.opentelemetry.io/collector/pdata/pcommon" +import ( + "slices" + + "go.opentelemetry.io/collector/pdata/pcommon" +) func (p *awsEntityProcessor) PutAttribute(resourceAttributes pcommon.Map, k string, v string) { - attributeAllowList := p.config.AttributeAllowList - for _, allowedAttribute := range attributeAllowList { - if k == allowedAttribute { - resourceAttributes.PutStr(k, v) - } + if len(p.config.AttributeAllowList) == 0 || slices.Contains(p.config.AttributeAllowList, k) { + resourceAttributes.PutStr(k, v) } }