diff --git a/docs/contributor/assets/metric-gateway-pipelines.png b/docs/contributor/assets/metric-gateway-pipelines.png index f6616b7b1b..4c8961681f 100644 Binary files a/docs/contributor/assets/metric-gateway-pipelines.png and b/docs/contributor/assets/metric-gateway-pipelines.png differ diff --git a/internal/otelcollector/config/common/constants.go b/internal/otelcollector/config/common/constants.go index e1d15c2d04..fc74f34351 100644 --- a/internal/otelcollector/config/common/constants.go +++ b/internal/otelcollector/config/common/constants.go @@ -149,7 +149,7 @@ const ( // CONNECTORS // ================================================================================ - ComponentIDForwardConnector = "forward/%s" // dynamically filled with pipeline name + ComponentIDForwardConnector = "forward" ComponentIDRoutingConnector = "routing/%s" // dynamically filled with pipeline name ComponentIDEnrichmentRoutingConnector = "routing/enrichment" ComponentIDRuntimeInputRoutingConnector = "routing/runtime-input" diff --git a/internal/otelcollector/config/common/processor_builders.go b/internal/otelcollector/config/common/processor_builders.go index a44c584086..fc5740b43b 100644 --- a/internal/otelcollector/config/common/processor_builders.go +++ b/internal/otelcollector/config/common/processor_builders.go @@ -237,7 +237,7 @@ func FilterSpecsToTraceFilterProcessorConfig(specs []telemetryv1alpha1.FilterSpe return &FilterProcessor{ ErrorMode: defaultFilterProcessorErrorMode, Traces: FilterProcessorTraces{ - // Use span context as it is it is always present even though spanevents is the lowest one + // Use span as context instead of spanevents, because while more granular, spanevents aren't always present // span event filtering is not supported by user-defined filter until filter processor supports context inference Span: mergedConditions, }, diff --git a/internal/otelcollector/config/logagent/config_builder.go b/internal/otelcollector/config/logagent/config_builder.go index 942f964e53..b35058b8e7 100644 --- a/internal/otelcollector/config/logagent/config_builder.go +++ b/internal/otelcollector/config/logagent/config_builder.go @@ -50,6 +50,8 @@ func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.LogPi b.addK8sAttributesProcessor(opts), b.addInsertClusterAttributesProcessor(opts), b.addServiceEnrichmentProcessor(), + // Kyma attributes are dropped before user-defined transform and filter processors + // to prevent user access to internal attributes. b.addDropKymaAttributesProcessor(), b.addUserDefinedTransformProcessor(), b.addUserDefinedFilterProcessor(), diff --git a/internal/otelcollector/config/logagent/config_builder_test.go b/internal/otelcollector/config/logagent/config_builder_test.go index 544275cbe6..6f55325b2b 100644 --- a/internal/otelcollector/config/logagent/config_builder_test.go +++ b/internal/otelcollector/config/logagent/config_builder_test.go @@ -100,7 +100,7 @@ func TestBuildConfig(t *testing.T) { }). Build(), }, - goldenFileName: "two-pipelines-with-transforms.yaml", + goldenFileName: "user-defined-transforms.yaml", }, { name: "two pipelines with user-defined filter", @@ -122,7 +122,7 @@ func TestBuildConfig(t *testing.T) { }). Build(), }, - goldenFileName: "two-pipelines-with-filter.yaml", + goldenFileName: "user-defined-filters.yaml", }, { name: "pipeline with user-defined transform and filter", @@ -140,7 +140,7 @@ func TestBuildConfig(t *testing.T) { }). Build(), }, - goldenFileName: "pipeline-with-transform-filter.yaml", + goldenFileName: "user-defined-transform-filter.yaml", }, } diff --git a/internal/otelcollector/config/logagent/testdata/two-pipelines-with-filter.yaml b/internal/otelcollector/config/logagent/testdata/user-defined-filters.yaml similarity index 100% rename from internal/otelcollector/config/logagent/testdata/two-pipelines-with-filter.yaml rename to internal/otelcollector/config/logagent/testdata/user-defined-filters.yaml diff --git a/internal/otelcollector/config/logagent/testdata/pipeline-with-transform-filter.yaml b/internal/otelcollector/config/logagent/testdata/user-defined-transform-filter.yaml similarity index 100% rename from internal/otelcollector/config/logagent/testdata/pipeline-with-transform-filter.yaml rename to internal/otelcollector/config/logagent/testdata/user-defined-transform-filter.yaml diff --git a/internal/otelcollector/config/logagent/testdata/two-pipelines-with-transforms.yaml b/internal/otelcollector/config/logagent/testdata/user-defined-transforms.yaml similarity index 100% rename from internal/otelcollector/config/logagent/testdata/two-pipelines-with-transforms.yaml rename to internal/otelcollector/config/logagent/testdata/user-defined-transforms.yaml diff --git a/internal/otelcollector/config/loggateway/config_builder.go b/internal/otelcollector/config/loggateway/config_builder.go index b36a103fba..670c8aef6d 100644 --- a/internal/otelcollector/config/loggateway/config_builder.go +++ b/internal/otelcollector/config/loggateway/config_builder.go @@ -48,6 +48,8 @@ func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.LogPi b.addNamespaceFilterProcessor(), b.addInsertClusterAttributesProcessor(opts), b.addServiceEnrichmentProcessor(), + // Kyma attributes are dropped before user-defined transform and filter processors + // to prevent user access to internal attributes. b.addDropKymaAttributesProcessor(), b.addIstioAccessLogsEnrichmentProcessor(opts), b.addUserDefinedTransformProcessor(), diff --git a/internal/otelcollector/config/loggateway/config_builder_test.go b/internal/otelcollector/config/loggateway/config_builder_test.go index c6e01aee7a..5c7acfc81d 100644 --- a/internal/otelcollector/config/loggateway/config_builder_test.go +++ b/internal/otelcollector/config/loggateway/config_builder_test.go @@ -107,7 +107,7 @@ func TestBuildConfig(t *testing.T) { }). Build(), }, - goldenFileName: "two-pipelines-with-transforms.yaml", + goldenFileName: "user-defined-transforms.yaml", }, { name: "two pipelines with user-defined filter", @@ -127,7 +127,7 @@ func TestBuildConfig(t *testing.T) { }). Build(), }, - goldenFileName: "two-pipelines-with-filter.yaml", + goldenFileName: "user-defined-filters.yaml", }, { name: "pipeline with user-defined transform and filter", @@ -143,7 +143,7 @@ func TestBuildConfig(t *testing.T) { Conditions: []string{"IsMatch(log.attributes[\"foo\"], \".*bar.*\")"}, }).Build(), }, - goldenFileName: "pipeline-with-transform-filter.yaml", + goldenFileName: "user-defined-transform-filter.yaml", }, } diff --git a/internal/otelcollector/config/loggateway/testdata/two-pipelines-with-filter.yaml b/internal/otelcollector/config/loggateway/testdata/user-defined-filters.yaml similarity index 100% rename from internal/otelcollector/config/loggateway/testdata/two-pipelines-with-filter.yaml rename to internal/otelcollector/config/loggateway/testdata/user-defined-filters.yaml diff --git a/internal/otelcollector/config/loggateway/testdata/pipeline-with-transform-filter.yaml b/internal/otelcollector/config/loggateway/testdata/user-defined-transform-filter.yaml similarity index 100% rename from internal/otelcollector/config/loggateway/testdata/pipeline-with-transform-filter.yaml rename to internal/otelcollector/config/loggateway/testdata/user-defined-transform-filter.yaml diff --git a/internal/otelcollector/config/loggateway/testdata/two-pipelines-with-transforms.yaml b/internal/otelcollector/config/loggateway/testdata/user-defined-transforms.yaml similarity index 100% rename from internal/otelcollector/config/loggateway/testdata/two-pipelines-with-transforms.yaml rename to internal/otelcollector/config/loggateway/testdata/user-defined-transforms.yaml diff --git a/internal/otelcollector/config/metricagent/config_builder.go b/internal/otelcollector/config/metricagent/config_builder.go index a70ee5d293..64823fa1a2 100644 --- a/internal/otelcollector/config/metricagent/config_builder.go +++ b/internal/otelcollector/config/metricagent/config_builder.go @@ -194,6 +194,8 @@ func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.Metri // Generic processors b.addInsertClusterAttributesProcessor(opts), b.addDropSkipEnrichmentAttributeProcessor(), + // Kyma attributes are dropped before user-defined transform and filter processors + // to prevent user access to internal attributes. b.addDropKymaAttributesProcessor(), b.addUserDefinedTransformProcessor(), b.addUserDefinedFilterProcessor(), diff --git a/internal/otelcollector/config/metricagent/config_builder_test.go b/internal/otelcollector/config/metricagent/config_builder_test.go index 9983cbb75b..a3b0094dcb 100644 --- a/internal/otelcollector/config/metricagent/config_builder_test.go +++ b/internal/otelcollector/config/metricagent/config_builder_test.go @@ -321,7 +321,7 @@ func TestBuildConfig(t *testing.T) { }, { name: "two pipelines with user-defined filter", - goldenFileName: "two-pipelines-with-filter.yaml", + goldenFileName: "user-defined-filters.yaml", pipelines: []telemetryv1alpha1.MetricPipeline{ testutils.NewMetricPipelineBuilder(). WithName("test1"). @@ -341,7 +341,7 @@ func TestBuildConfig(t *testing.T) { }, { name: "pipeline with user-defined transform and filter", - goldenFileName: "pipeline-with-transform-filter.yaml", + goldenFileName: "user-defined-transform-filter.yaml", pipelines: []telemetryv1alpha1.MetricPipeline{ testutils.NewMetricPipelineBuilder(). WithName("test1"). diff --git a/internal/otelcollector/config/metricagent/testdata/two-pipelines-with-filter.yaml b/internal/otelcollector/config/metricagent/testdata/user-defined-filters.yaml similarity index 100% rename from internal/otelcollector/config/metricagent/testdata/two-pipelines-with-filter.yaml rename to internal/otelcollector/config/metricagent/testdata/user-defined-filters.yaml diff --git a/internal/otelcollector/config/metricagent/testdata/pipeline-with-transform-filter.yaml b/internal/otelcollector/config/metricagent/testdata/user-defined-transform-filter.yaml similarity index 100% rename from internal/otelcollector/config/metricagent/testdata/pipeline-with-transform-filter.yaml rename to internal/otelcollector/config/metricagent/testdata/user-defined-transform-filter.yaml diff --git a/internal/otelcollector/config/metricgateway/config_builder.go b/internal/otelcollector/config/metricgateway/config_builder.go index 62e08b475d..cb47f2d945 100644 --- a/internal/otelcollector/config/metricgateway/config_builder.go +++ b/internal/otelcollector/config/metricgateway/config_builder.go @@ -9,6 +9,8 @@ import ( operatorv1alpha1 "github.com/kyma-project/telemetry-manager/apis/operator/v1alpha1" telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/common" + "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" + metricpipelineutils "github.com/kyma-project/telemetry-manager/internal/utils/metricpipeline" ) type buildComponentFunc = common.BuildComponentFunc[*telemetryv1alpha1.MetricPipeline] @@ -41,59 +43,31 @@ func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.Metri queueSize := common.BatchingMaxQueueSize / len(pipelines) - for _, pipeline := range pipelines { - inputPipelineID := formatInputMetricServicePipelineID(&pipeline) - if err := b.AddServicePipeline(ctx, &pipeline, inputPipelineID, - b.addOTLPReceiver(), - b.addKymaStatsReceiver(), - b.addMemoryLimiterProcessor(), - b.addInputRoutingExporter(), - ); err != nil { - return nil, nil, fmt.Errorf("failed to add input service pipeline: %w", err) - } - - enrichmentPipelineID := formatEnrichmentServicePipelineID(&pipeline) - if err := b.AddServicePipeline(ctx, &pipeline, enrichmentPipelineID, - b.addEnrichmentRoutingReceiver(), - b.addK8sAttributesProcessor(opts), - b.addServiceEnrichmentProcessor(), - b.addEnrichmentForwardExporter(), - ); err != nil { - return nil, nil, fmt.Errorf("failed to add enrichment service pipeline: %w", err) - } + if err := b.AddServicePipeline(ctx, nil, "metrics/input", + b.addOTLPReceiver(), + b.addKymaStatsReceiver(), + b.addMemoryLimiterProcessor(), + b.addSetInstrumentationScopeToKymaProcessor(opts), + b.addK8sAttributesProcessor(opts), + b.addServiceEnrichmentProcessor(), + b.addInsertClusterAttributesProcessor(opts), + // Kyma attributes are dropped before user-defined transform and filter processors + // to prevent user access to internal attributes. + b.addDropKymaAttributesProcessor(), + b.addInputForwardExporter(), + ); err != nil { + return nil, nil, fmt.Errorf("failed to add input service pipeline: %w", err) + } - ouputPipelineID := formatOutputServicePipelineID(&pipeline) - if err := b.AddServicePipeline(ctx, &pipeline, ouputPipelineID, - b.addOutputRoutingReceiver(), + for _, pipeline := range pipelines { + outputPipelineID := formatOutputServicePipelineID(&pipeline) + if err := b.AddServicePipeline(ctx, &pipeline, outputPipelineID, b.addOutputForwardReceiver(), - b.addSetInstrumentationScopeToKymaProcessor(opts), - // Input source filters - b.addDropIfRuntimeInputDisabledProcessor(), - b.addDropIfPrometheusInputDisabledProcessor(), - b.addDropIfIstioInputDisabledProcessor(), - b.addDropEnvoyMetricsIfDisabledProcessor(), - b.addDropIfOTLPInputDisabledProcessor(), + // Input source filters if otlp is disabled + b.addDropOTLPIfInputDisabledProcessor(), // Namespace filters - b.addRuntimeNamespaceFilterProcessor(), - b.addPrometheusNamespaceFilterProcessor(), - b.addIstioNamespaceFilterProcessor(), b.addOTLPNamespaceFilterProcessor(), - // Runtime resource filters - b.addDropRuntimePodMetricsProcessor(), - b.addDropRuntimeContainerMetricsProcessor(), - b.addDropRuntimeNodeMetricsProcessor(), - b.addDropRuntimeVolumeMetricsProcessor(), - b.addDropRuntimeDeploymentMetricsProcessor(), - b.addDropRuntimeDaemonSetMetricsProcessor(), - b.addDropRuntimeStatefulSetMetricsProcessor(), - b.addDropRuntimeJobMetricsProcessor(), - // Diagnostic metric filters - b.addDropIstioDiagnosticMetricsProcessor(), - b.addDropPrometheusDiagnosticMetricsProcessor(), - - b.addInsertClusterAttributesProcessor(opts), - b.addDeleteSkipEnrichmentAttributeProcessor(), - b.addDropKymaAttributesProcessor(), + // User defined Transform and Filter b.addUserDefinedTransformProcessor(), b.addUserDefinedFilterProcessor(), // Batch processor (always last) @@ -108,40 +82,283 @@ func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.Metri return b.Config, b.EnvVars, nil } -// Helper functions +// ====================================================== +// Input pipeline components +// ====================================================== +func (b *Builder) addOTLPReceiver() buildComponentFunc { + return b.AddReceiver( + b.StaticComponentID(common.ComponentIDOTLPReceiver), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return &common.OTLPReceiver{ + Protocols: common.ReceiverProtocols{ + HTTP: common.Endpoint{ + Endpoint: fmt.Sprintf("${%s}:%d", common.EnvVarCurrentPodIP, ports.OTLPHTTP), + }, + GRPC: common.Endpoint{ + Endpoint: fmt.Sprintf("${%s}:%d", common.EnvVarCurrentPodIP, ports.OTLPGRPC), + }, + }, + } + }, + ) +} + +func (b *Builder) addKymaStatsReceiver() buildComponentFunc { + return b.AddReceiver( + b.StaticComponentID(common.ComponentIDKymaStatsReceiver), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return &KymaStatsReceiver{ + AuthType: "serviceAccount", + K8sLeaderElector: "k8s_leader_elector", + CollectionInterval: "30s", + Resources: []ModuleGVR{ + {Group: "operator.kyma-project.io", Version: "v1alpha1", Resource: "telemetries"}, + {Group: "telemetry.kyma-project.io", Version: "v1alpha1", Resource: "logpipelines"}, + {Group: "telemetry.kyma-project.io", Version: "v1alpha1", Resource: "tracepipelines"}, + {Group: "telemetry.kyma-project.io", Version: "v1alpha1", Resource: "metricpipelines"}, + }, + } + }, + ) +} + +//nolint:mnd // hardcoded values +func (b *Builder) addMemoryLimiterProcessor() buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDMemoryLimiterProcessor), + func(lp *telemetryv1alpha1.MetricPipeline) any { + return &common.MemoryLimiter{ + CheckInterval: "1s", + LimitPercentage: 75, + SpikeLimitPercentage: 15, + } + }, + ) +} + +func (b *Builder) addSetInstrumentationScopeToKymaProcessor(opts BuildOptions) buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDSetInstrumentationScopeKymaProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return common.InstrumentationScopeProcessorConfig(opts.InstrumentationScopeVersion, common.InputSourceKyma) + }, + ) +} + +func (b *Builder) addK8sAttributesProcessor(opts BuildOptions) buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDK8sAttributesProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return common.K8sAttributesProcessorConfig(opts.Enrichments) + }, + ) +} + +func (b *Builder) addServiceEnrichmentProcessor() buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDServiceEnrichmentProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return common.ResolveServiceNameConfig() + }, + ) +} + +// Resource processors + +func (b *Builder) addInsertClusterAttributesProcessor(opts BuildOptions) buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDInsertClusterAttributesProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return common.InsertClusterAttributesProcessorConfig(opts.ClusterName, opts.ClusterUID, opts.CloudProvider) + }, + ) +} + +func (b *Builder) addInputForwardExporter() buildComponentFunc { + return b.AddExporter( + b.StaticComponentID(common.ComponentIDForwardConnector), + func(ctx context.Context, mp *telemetryv1alpha1.MetricPipeline) (any, common.EnvVars, error) { + return &common.ForwardConnector{}, nil, nil + }, + ) +} + +// ====================================================== +// Output pipeline components +// ====================================================== + +func (b *Builder) addOutputForwardReceiver() buildComponentFunc { + return b.AddReceiver( + b.StaticComponentID(common.ComponentIDForwardConnector), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return &common.ForwardConnector{} + }, + ) +} + +// Input source filter processors + +// if OTLP input is disabled filter drops all metrics where instrumentation scope is not 'kyma' +func (b *Builder) addDropOTLPIfInputDisabledProcessor() buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDDropIfInputSourceOTLPProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + if metricpipelineutils.IsOTLPInputEnabled(mp.Spec.Input) { + return nil + } + + return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ + Metric: []string{common.Not(common.ScopeNameEquals(common.InstrumentationScopeKyma))}, + }) + }, + ) +} + +// Namespace filter processors + +func (b *Builder) addOTLPNamespaceFilterProcessor() buildComponentFunc { + return b.AddProcessor( + func(mp *telemetryv1alpha1.MetricPipeline) string { + return formatOTLPNamespaceFilterID(mp.Name) + }, + func(mp *telemetryv1alpha1.MetricPipeline) any { + input := mp.Spec.Input + if !metricpipelineutils.IsOTLPInputEnabled(input) || input.OTLP == nil || !shouldFilterByNamespace(input.OTLP.Namespaces) { + return nil + } + + return filterByNamespaceProcessorConfig(input.OTLP.Namespaces) + }, + ) +} + +func (b *Builder) addDropKymaAttributesProcessor() buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDDropKymaAttributesProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return common.DropKymaAttributesProcessorConfig() + }, + ) +} + +func (b *Builder) addUserDefinedTransformProcessor() buildComponentFunc { + return b.AddProcessor( + formatUserDefinedTransformProcessorID, + func(mp *telemetryv1alpha1.MetricPipeline) any { + if len(mp.Spec.Transforms) == 0 { + return nil // No transforms, no processor needed + } + + transformStatements := common.TransformSpecsToProcessorStatements(mp.Spec.Transforms) + transformProcessor := common.MetricTransformProcessorConfig(transformStatements) + + return transformProcessor + }, + ) +} + +func (b *Builder) addUserDefinedFilterProcessor() buildComponentFunc { + return b.AddProcessor( + formatUserDefinedFilterProcessorID, + func(mp *telemetryv1alpha1.MetricPipeline) any { + if mp.Spec.Filters == nil { + return nil // No filters, no processor needed + } + + return common.FilterSpecsToMetricFilterProcessorConfig(mp.Spec.Filters) + }, + ) +} -func enrichmentRoutingConnectorConfig(mp *telemetryv1alpha1.MetricPipeline) common.RoutingConnector { - enrichmentPipelineID := formatEnrichmentServicePipelineID(mp) - outputPipelineID := formatOutputServicePipelineID(mp) +// Batch processor - return common.RoutingConnector{ - DefaultPipelines: []string{enrichmentPipelineID}, - ErrorMode: "ignore", - Table: []common.RoutingConnectorTableEntry{ - { - Statement: fmt.Sprintf("route() where attributes[\"%s\"] == \"true\"", common.SkipEnrichmentAttribute), - Pipelines: []string{outputPipelineID}, - }, +//nolint:mnd // hardcoded values +func (b *Builder) addBatchProcessor() buildComponentFunc { + return b.AddProcessor( + b.StaticComponentID(common.ComponentIDBatchProcessor), + func(mp *telemetryv1alpha1.MetricPipeline) any { + return &common.BatchProcessor{ + SendBatchSize: 1024, + Timeout: "10s", + SendBatchMaxSize: 1024, + } }, + ) +} + +func (b *Builder) addOTLPExporter(queueSize int) buildComponentFunc { + return b.AddExporter( + formatOTLPExporterID, + func(ctx context.Context, mp *telemetryv1alpha1.MetricPipeline) (any, common.EnvVars, error) { + otlpExporterBuilder := common.NewOTLPExporterConfigBuilder( + b.Reader, + mp.Spec.Output.OTLP, + mp.Name, + queueSize, + common.SignalTypeMetric, + ) + + return otlpExporterBuilder.OTLPExporterConfig(ctx) + }, + ) +} + +// Helper functions + +func shouldFilterByNamespace(namespaceSelector *telemetryv1alpha1.NamespaceSelector) bool { + return namespaceSelector != nil && (len(namespaceSelector.Include) > 0 || len(namespaceSelector.Exclude) > 0) +} + +func filterByNamespaceProcessorConfig(namespaceSelector *telemetryv1alpha1.NamespaceSelector) *common.FilterProcessor { + var filterExpressions []string + + notFromKymaStatsReceiver := common.Not(common.ScopeNameEquals(common.InstrumentationScopeKyma)) + + if len(namespaceSelector.Exclude) > 0 { + namespacesConditions := namespacesConditionsBuilder(namespaceSelector.Exclude) + excludeNamespacesExpr := common.JoinWithAnd(notFromKymaStatsReceiver, common.JoinWithOr(namespacesConditions...)) + filterExpressions = append(filterExpressions, excludeNamespacesExpr) + } + + if len(namespaceSelector.Include) > 0 { + namespacesConditions := namespacesConditionsBuilder(namespaceSelector.Include) + includeNamespacesExpr := common.JoinWithAnd( + notFromKymaStatsReceiver, + common.ResourceAttributeIsNotNil(common.K8sNamespaceName), + common.Not(common.JoinWithOr(namespacesConditions...)), + ) + filterExpressions = append(filterExpressions, includeNamespacesExpr) } + + return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ + Metric: filterExpressions, + }) } -func formatInputMetricServicePipelineID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf("metrics/%s-input", mp.Name) +func namespacesConditionsBuilder(namespaces []string) []string { + var conditions []string + for _, ns := range namespaces { + conditions = append(conditions, common.NamespaceEquals(ns)) + } + + return conditions } -func formatEnrichmentServicePipelineID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf("metrics/%s-attributes-enrichment", mp.Name) +func formatOTLPNamespaceFilterID(pipelineName string) string { + return fmt.Sprintf(common.ComponentIDNamespacePerInputFilterProcessor, pipelineName, common.InputSourceOTLP) } -func formatOutputServicePipelineID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf("metrics/%s-output", mp.Name) +func formatOTLPExporterID(pipeline *telemetryv1alpha1.MetricPipeline) string { + return common.ExporterID(pipeline.Spec.Output.OTLP.Protocol, pipeline.Name) } -func formatRoutingConnectorID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf(common.ComponentIDRoutingConnector, mp.Name) +func formatUserDefinedTransformProcessorID(mp *telemetryv1alpha1.MetricPipeline) string { + return fmt.Sprintf(common.ComponentIDUserDefinedTransformProcessor, mp.Name) } -func formatForwardConnectorID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf(common.ComponentIDForwardConnector, mp.Name) +func formatUserDefinedFilterProcessorID(mp *telemetryv1alpha1.MetricPipeline) string { + return fmt.Sprintf(common.ComponentIDUserDefinedFilterProcessor, mp.Name) +} +func formatOutputServicePipelineID(mp *telemetryv1alpha1.MetricPipeline) string { + return fmt.Sprintf("metrics/%s-output", mp.Name) } diff --git a/internal/otelcollector/config/metricgateway/config_builder_enrichment.go b/internal/otelcollector/config/metricgateway/config_builder_enrichment.go deleted file mode 100644 index 4a342d2157..0000000000 --- a/internal/otelcollector/config/metricgateway/config_builder_enrichment.go +++ /dev/null @@ -1,44 +0,0 @@ -package metricgateway - -import ( - "context" - - telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" - "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/common" -) - -func (b *Builder) addEnrichmentRoutingReceiver() buildComponentFunc { - return b.AddReceiver( - formatRoutingConnectorID, - func(mp *telemetryv1alpha1.MetricPipeline) any { - return enrichmentRoutingConnectorConfig(mp) - }, - ) -} - -func (b *Builder) addK8sAttributesProcessor(opts BuildOptions) buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDK8sAttributesProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return common.K8sAttributesProcessorConfig(opts.Enrichments) - }, - ) -} - -func (b *Builder) addServiceEnrichmentProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDServiceEnrichmentProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return common.ResolveServiceNameConfig() - }, - ) -} - -func (b *Builder) addEnrichmentForwardExporter() buildComponentFunc { - return b.AddExporter( - formatForwardConnectorID, - func(ctx context.Context, mp *telemetryv1alpha1.MetricPipeline) (any, common.EnvVars, error) { - return &common.ForwardConnector{}, nil, nil - }, - ) -} diff --git a/internal/otelcollector/config/metricgateway/config_builder_input.go b/internal/otelcollector/config/metricgateway/config_builder_input.go deleted file mode 100644 index 6e8cd8d4ca..0000000000 --- a/internal/otelcollector/config/metricgateway/config_builder_input.go +++ /dev/null @@ -1,70 +0,0 @@ -package metricgateway - -import ( - "context" - "fmt" - - telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" - "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/common" - "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" -) - -func (b *Builder) addOTLPReceiver() buildComponentFunc { - return b.AddReceiver( - b.StaticComponentID(common.ComponentIDOTLPReceiver), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return &common.OTLPReceiver{ - Protocols: common.ReceiverProtocols{ - HTTP: common.Endpoint{ - Endpoint: fmt.Sprintf("${%s}:%d", common.EnvVarCurrentPodIP, ports.OTLPHTTP), - }, - GRPC: common.Endpoint{ - Endpoint: fmt.Sprintf("${%s}:%d", common.EnvVarCurrentPodIP, ports.OTLPGRPC), - }, - }, - } - }, - ) -} - -func (b *Builder) addKymaStatsReceiver() buildComponentFunc { - return b.AddReceiver( - b.StaticComponentID(common.ComponentIDKymaStatsReceiver), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return &KymaStatsReceiver{ - AuthType: "serviceAccount", - K8sLeaderElector: "k8s_leader_elector", - CollectionInterval: "30s", - Resources: []ModuleGVR{ - {Group: "operator.kyma-project.io", Version: "v1alpha1", Resource: "telemetries"}, - {Group: "telemetry.kyma-project.io", Version: "v1alpha1", Resource: "logpipelines"}, - {Group: "telemetry.kyma-project.io", Version: "v1alpha1", Resource: "tracepipelines"}, - {Group: "telemetry.kyma-project.io", Version: "v1alpha1", Resource: "metricpipelines"}, - }, - } - }, - ) -} - -//nolint:mnd // hardcoded values -func (b *Builder) addMemoryLimiterProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDMemoryLimiterProcessor), - func(lp *telemetryv1alpha1.MetricPipeline) any { - return &common.MemoryLimiter{ - CheckInterval: "1s", - LimitPercentage: 75, - SpikeLimitPercentage: 15, - } - }, - ) -} - -func (b *Builder) addInputRoutingExporter() buildComponentFunc { - return b.AddExporter( - formatRoutingConnectorID, - func(ctx context.Context, mp *telemetryv1alpha1.MetricPipeline) (any, common.EnvVars, error) { - return enrichmentRoutingConnectorConfig(mp), nil, nil - }, - ) -} diff --git a/internal/otelcollector/config/metricgateway/config_builder_output.go b/internal/otelcollector/config/metricgateway/config_builder_output.go deleted file mode 100644 index 11698d89fd..0000000000 --- a/internal/otelcollector/config/metricgateway/config_builder_output.go +++ /dev/null @@ -1,549 +0,0 @@ -package metricgateway - -import ( - "context" - "fmt" - - telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" - "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/common" - metricpipelineutils "github.com/kyma-project/telemetry-manager/internal/utils/metricpipeline" -) - -var diagnosticMetricNames = []string{"up", "scrape_duration_seconds", "scrape_samples_scraped", "scrape_samples_post_metric_relabeling", "scrape_series_added"} - -func (b *Builder) addOutputForwardReceiver() buildComponentFunc { - return b.AddReceiver( - formatForwardConnectorID, - func(mp *telemetryv1alpha1.MetricPipeline) any { - return &common.ForwardConnector{} - }, - ) -} - -func (b *Builder) addOutputRoutingReceiver() buildComponentFunc { - return b.AddReceiver( - formatRoutingConnectorID, - func(mp *telemetryv1alpha1.MetricPipeline) any { - return enrichmentRoutingConnectorConfig(mp) - }, - ) -} - -func (b *Builder) addSetInstrumentationScopeToKymaProcessor(opts BuildOptions) buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDSetInstrumentationScopeKymaProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return common.InstrumentationScopeProcessorConfig(opts.InstrumentationScopeVersion, common.InputSourceKyma) - }, - ) -} - -// Input source filter processors - -func (b *Builder) addDropIfRuntimeInputDisabledProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropIfInputSourceRuntimeProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.ScopeNameEquals(common.InstrumentationScopeRuntime)}, - }) - }, - ) -} - -func (b *Builder) addDropIfPrometheusInputDisabledProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropIfInputSourcePrometheusProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if metricpipelineutils.IsPrometheusInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.ResourceAttributeEquals(common.KymaInputNameAttribute, common.KymaInputPrometheus)}, - }) - }, - ) -} - -func (b *Builder) addDropIfIstioInputDisabledProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropIfInputSourceIstioProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if metricpipelineutils.IsIstioInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.ScopeNameEquals(common.InstrumentationScopeIstio)}, - }) - }, - ) -} - -func (b *Builder) addDropIfOTLPInputDisabledProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropIfInputSourceOTLPProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if metricpipelineutils.IsOTLPInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{ottlUknownInputSource()}, - }) - }, - ) -} - -func (b *Builder) addDropEnvoyMetricsIfDisabledProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropEnvoyMetricsIfDisabledProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if metricpipelineutils.IsIstioInputEnabled(mp.Spec.Input) && metricpipelineutils.IsEnvoyMetricsEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - common.IsMatch("name", "^envoy_.*"), - common.ScopeNameEquals(common.InstrumentationScopeIstio), - )}, - }) - }, - ) -} - -// Namespace filter processors - -func (b *Builder) addRuntimeNamespaceFilterProcessor() buildComponentFunc { - return b.AddProcessor( - func(mp *telemetryv1alpha1.MetricPipeline) string { - return formatNamespaceFilterID(mp.Name, common.InputSourceRuntime) - }, - func(mp *telemetryv1alpha1.MetricPipeline) any { - input := mp.Spec.Input - if !metricpipelineutils.IsRuntimeInputEnabled(input) || !shouldFilterByNamespace(input.Runtime.Namespaces) { - return nil - } - - return filterByNamespaceProcessorConfig(input.Runtime.Namespaces, inputSourceEquals(common.InputSourceRuntime)) - }, - ) -} - -func (b *Builder) addPrometheusNamespaceFilterProcessor() buildComponentFunc { - return b.AddProcessor( - func(mp *telemetryv1alpha1.MetricPipeline) string { - return formatNamespaceFilterID(mp.Name, common.InputSourcePrometheus) - }, - func(mp *telemetryv1alpha1.MetricPipeline) any { - input := mp.Spec.Input - if !metricpipelineutils.IsPrometheusInputEnabled(input) || !shouldFilterByNamespace(input.Prometheus.Namespaces) { - return nil - } - - return filterByNamespaceProcessorConfig(input.Prometheus.Namespaces, common.ResourceAttributeEquals(common.KymaInputNameAttribute, common.KymaInputPrometheus)) - }, - ) -} - -func (b *Builder) addIstioNamespaceFilterProcessor() buildComponentFunc { - return b.AddProcessor( - func(mp *telemetryv1alpha1.MetricPipeline) string { - return formatNamespaceFilterID(mp.Name, common.InputSourceIstio) - }, - func(mp *telemetryv1alpha1.MetricPipeline) any { - input := mp.Spec.Input - if !metricpipelineutils.IsIstioInputEnabled(input) || !shouldFilterByNamespace(input.Istio.Namespaces) { - return nil - } - - return filterByNamespaceProcessorConfig(input.Istio.Namespaces, inputSourceEquals(common.InputSourceIstio)) - }, - ) -} - -func (b *Builder) addOTLPNamespaceFilterProcessor() buildComponentFunc { - return b.AddProcessor( - func(mp *telemetryv1alpha1.MetricPipeline) string { - return formatNamespaceFilterID(mp.Name, common.InputSourceOTLP) - }, - func(mp *telemetryv1alpha1.MetricPipeline) any { - input := mp.Spec.Input - if !metricpipelineutils.IsOTLPInputEnabled(input) || input.OTLP == nil || !shouldFilterByNamespace(input.OTLP.Namespaces) { - return nil - } - - return filterByNamespaceProcessorConfig(input.OTLP.Namespaces, ottlUknownInputSource()) - }, - ) -} - -// Runtime resource filter processors - -func (b *Builder) addDropRuntimePodMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimePodMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimePodInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.pod.*"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeContainerMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeContainerMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeContainerInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "(^k8s.container.*)|(^container.*)"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeNodeMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeNodeMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeNodeInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.node.*"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeVolumeMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeVolumeMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeVolumeInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.volume.*"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeDeploymentMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeDeploymentMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeDeploymentInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.deployment.*"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeDaemonSetMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeDaemonSetMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeDaemonSetInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.daemonset.*"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeStatefulSetMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeStatefulSetMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeStatefulSetInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.statefulset.*"), - )}, - }) - }, - ) -} - -func (b *Builder) addDropRuntimeJobMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropRuntimeJobMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsRuntimeInputEnabled(mp.Spec.Input) || metricpipelineutils.IsRuntimeJobInputEnabled(mp.Spec.Input) { - return nil - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: []string{common.JoinWithAnd( - inputSourceEquals(common.InputSourceRuntime), - common.IsMatch("name", "^k8s.job.*"), - )}, - }) - }, - ) -} - -// Diagnostic metric filter processors - -func (b *Builder) addDropPrometheusDiagnosticMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropPrometheusDiagnosticMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsPrometheusInputEnabled(mp.Spec.Input) || metricpipelineutils.IsPrometheusDiagnosticInputEnabled(mp.Spec.Input) { - return nil - } - - return dropDiagnosticMetricsFilterConfig(inputSourceEquals(common.InputSourcePrometheus)) - }, - ) -} - -func (b *Builder) addDropIstioDiagnosticMetricsProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropIstioDiagnosticMetricsProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - if !metricpipelineutils.IsIstioInputEnabled(mp.Spec.Input) || metricpipelineutils.IsIstioDiagnosticInputEnabled(mp.Spec.Input) { - return nil - } - - return dropDiagnosticMetricsFilterConfig(inputSourceEquals(common.InputSourceIstio)) - }, - ) -} - -// Helper functions - -func shouldFilterByNamespace(namespaceSelector *telemetryv1alpha1.NamespaceSelector) bool { - return namespaceSelector != nil && (len(namespaceSelector.Include) > 0 || len(namespaceSelector.Exclude) > 0) -} - -func inputSourceEquals(inputSourceType common.InputSourceType) string { - return common.ScopeNameEquals(common.InstrumentationScope[inputSourceType]) -} - -// When instrumentation scope is not set to any of the following values -// io.kyma-project.telemetry/runtime, io.kyma-project.telemetry/prometheus, io.kyma-project.telemetry/istio, and io.kyma-project.telemetry/kyma -// we assume the metric is being pushed directly to metrics gateway. -func ottlUknownInputSource() string { - return fmt.Sprintf("not(%s or %s or %s or %s)", - common.ScopeNameEquals(common.InstrumentationScopeRuntime), - common.ResourceAttributeEquals(common.KymaInputNameAttribute, common.KymaInputPrometheus), - common.ScopeNameEquals(common.InstrumentationScopeIstio), - common.ScopeNameEquals(common.InstrumentationScopeKyma), - ) -} - -func filterByNamespaceProcessorConfig(namespaceSelector *telemetryv1alpha1.NamespaceSelector, inputSourceCondition string) *common.FilterProcessor { - var filterExpressions []string - - if len(namespaceSelector.Exclude) > 0 { - namespacesConditions := namespacesConditions(namespaceSelector.Exclude) - excludeNamespacesExpr := common.JoinWithAnd(inputSourceCondition, common.JoinWithOr(namespacesConditions...)) - filterExpressions = append(filterExpressions, excludeNamespacesExpr) - } - - if len(namespaceSelector.Include) > 0 { - namespacesConditions := namespacesConditions(namespaceSelector.Include) - includeNamespacesExpr := common.JoinWithAnd( - inputSourceCondition, - common.ResourceAttributeIsNotNil(common.K8sNamespaceName), - common.Not(common.JoinWithOr(namespacesConditions...)), - ) - filterExpressions = append(filterExpressions, includeNamespacesExpr) - } - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: filterExpressions, - }) -} - -func namespacesConditions(namespaces []string) []string { - var conditions []string - for _, ns := range namespaces { - conditions = append(conditions, common.NamespaceEquals(ns)) - } - - return conditions -} - -// Resource processors - -func (b *Builder) addInsertClusterAttributesProcessor(opts BuildOptions) buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDInsertClusterAttributesProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return common.InsertClusterAttributesProcessorConfig(opts.ClusterName, opts.ClusterUID, opts.CloudProvider) - }, - ) -} - -func (b *Builder) addDeleteSkipEnrichmentAttributeProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropSkipEnrichmentAttributeProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return &common.ResourceProcessor{ - Attributes: []common.AttributeAction{ - { - Action: "delete", - Key: common.SkipEnrichmentAttribute, - }, - }, - } - }, - ) -} - -func (b *Builder) addDropKymaAttributesProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDDropKymaAttributesProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return common.DropKymaAttributesProcessorConfig() - }, - ) -} - -func (b *Builder) addUserDefinedTransformProcessor() buildComponentFunc { - return b.AddProcessor( - formatUserDefinedTransformProcessorID, - func(mp *telemetryv1alpha1.MetricPipeline) any { - if len(mp.Spec.Transforms) == 0 { - return nil // No transforms, no processor needed - } - - transformStatements := common.TransformSpecsToProcessorStatements(mp.Spec.Transforms) - transformProcessor := common.MetricTransformProcessorConfig(transformStatements) - - return transformProcessor - }, - ) -} - -func (b *Builder) addUserDefinedFilterProcessor() buildComponentFunc { - return b.AddProcessor( - formatUserDefinedFilterProcessorID, - func(mp *telemetryv1alpha1.MetricPipeline) any { - if mp.Spec.Filters == nil { - return nil // No filters, no processor needed - } - - return common.FilterSpecsToMetricFilterProcessorConfig(mp.Spec.Filters) - }, - ) -} - -// Batch processor - -//nolint:mnd // hardcoded values -func (b *Builder) addBatchProcessor() buildComponentFunc { - return b.AddProcessor( - b.StaticComponentID(common.ComponentIDBatchProcessor), - func(mp *telemetryv1alpha1.MetricPipeline) any { - return &common.BatchProcessor{ - SendBatchSize: 1024, - Timeout: "10s", - SendBatchMaxSize: 1024, - } - }, - ) -} - -func (b *Builder) addOTLPExporter(queueSize int) buildComponentFunc { - return b.AddExporter( - formatOTLPExporterID, - func(ctx context.Context, mp *telemetryv1alpha1.MetricPipeline) (any, common.EnvVars, error) { - otlpExporterBuilder := common.NewOTLPExporterConfigBuilder( - b.Reader, - mp.Spec.Output.OTLP, - mp.Name, - queueSize, - common.SignalTypeMetric, - ) - - return otlpExporterBuilder.OTLPExporterConfig(ctx) - }, - ) -} - -func dropDiagnosticMetricsFilterConfig(inputSourceCondition string) *common.FilterProcessor { - var filterExpressions []string - - metricNameConditions := nameConditions(diagnosticMetricNames) - excludeScrapeMetricsExpr := common.JoinWithAnd(inputSourceCondition, common.JoinWithOr(metricNameConditions...)) - filterExpressions = append(filterExpressions, excludeScrapeMetricsExpr) - - return common.MetricFilterProcessorConfig(common.FilterProcessorMetrics{ - Metric: filterExpressions, - }) -} - -func nameConditions(names []string) []string { - var nameConditions []string - for _, name := range names { - nameConditions = append(nameConditions, common.NameAttributeEquals(name)) - } - - return nameConditions -} - -func formatNamespaceFilterID(pipelineName string, inputSourceType common.InputSourceType) string { - return fmt.Sprintf(common.ComponentIDNamespacePerInputFilterProcessor, pipelineName, inputSourceType) -} - -func formatOTLPExporterID(pipeline *telemetryv1alpha1.MetricPipeline) string { - return common.ExporterID(pipeline.Spec.Output.OTLP.Protocol, pipeline.Name) -} - -func formatUserDefinedTransformProcessorID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf(common.ComponentIDUserDefinedTransformProcessor, mp.Name) -} - -func formatUserDefinedFilterProcessorID(mp *telemetryv1alpha1.MetricPipeline) string { - return fmt.Sprintf(common.ComponentIDUserDefinedFilterProcessor, mp.Name) -} diff --git a/internal/otelcollector/config/metricgateway/config_builder_test.go b/internal/otelcollector/config/metricgateway/config_builder_test.go index 4f78ab8335..9283e4ce78 100644 --- a/internal/otelcollector/config/metricgateway/config_builder_test.go +++ b/internal/otelcollector/config/metricgateway/config_builder_test.go @@ -57,25 +57,29 @@ func TestMakeConfig(t *testing.T) { }, }, { - name: "complex pipeline with comprehensive configuration", + name: "two pipelines with comprehensive configuration", goldenFileName: "setup-comprehensive.yaml", pipelines: []telemetryv1alpha1.MetricPipeline{ testutils.NewMetricPipelineBuilder(). WithName("cls"). - WithRuntimeInput(true, testutils.IncludeNamespaces("production", "staging")). - WithRuntimeInputPodMetrics(true). - WithRuntimeInputContainerMetrics(true). - WithRuntimeInputNodeMetrics(true). - WithPrometheusInput(true, testutils.ExcludeNamespaces("kube-system")). - WithPrometheusInputDiagnosticMetrics(true). - WithIstioInput(true). - WithIstioInputEnvoyMetrics(true). - WithOTLPInput(true, testutils.IncludeNamespaces("apps")). + WithOTLPInput(true, testutils.IncludeNamespaces("apps-cls")). WithOTLPOutput(testutils.OTLPEndpoint("https://backend.example.com")). WithTransform(telemetryv1alpha1.TransformSpec{ Conditions: []string{"resource.attributes[\"k8s.namespace.name\"] == \"production\""}, Statements: []string{"set(attributes[\"environment\"], \"prod\")"}, - }).Build(), + }).WithFilter(telemetryv1alpha1.FilterSpec{ + Conditions: []string{"metric.type == METRIC_DATA_TYPE_GAUGE"}, + }).Build(), + testutils.NewMetricPipelineBuilder(). + WithName("dynatrace"). + WithOTLPInput(true, testutils.IncludeNamespaces("apps-dynatrace")). + WithOTLPOutput(testutils.OTLPEndpoint("https://backend.example.com")). + WithTransform(telemetryv1alpha1.TransformSpec{ + Conditions: []string{"resource.attributes[\"k8s.namespace.name\"] == \"staging\""}, + Statements: []string{"set(attributes[\"environment\"], \"staging\")"}, + }).WithFilter(telemetryv1alpha1.FilterSpec{ + Conditions: []string{"metric.type == METRIC_DATA_TYPE_SUMMARY"}, + }).Build(), }, }, { @@ -88,45 +92,7 @@ func TestMakeConfig(t *testing.T) { WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), }, }, - { - name: "pipeline with runtime input and namespace filters", - goldenFileName: "runtime-namespace-filters.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithRuntimeInput(true, - testutils.IncludeNamespaces("monitoring", "observability"), - testutils.ExcludeNamespaces("kube-system", "istio-system"), - ). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with prometheus input and namespace filters", - goldenFileName: "prometheus-namespace-filters.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithPrometheusInput(true, - testutils.IncludeNamespaces("monitoring", "observability"), - testutils.ExcludeNamespaces("kube-system", "istio-system"), - ). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with istio input and namespace filters", - goldenFileName: "istio-namespace-filters.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithIstioInput(true, - testutils.IncludeNamespaces("monitoring", "observability"), - testutils.ExcludeNamespaces("kube-system", "istio-system"), - ). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, + { name: "pipeline with OTLP input and namespace filters", goldenFileName: "otlp-namespace-filters.yaml", @@ -140,88 +106,6 @@ func TestMakeConfig(t *testing.T) { WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), }, }, - { - name: "pipeline with multiple input types and mixed configurations", - goldenFileName: "multiple-inputs-mixed.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithRuntimeInput(true, testutils.IncludeNamespaces("default")). - WithPrometheusInput(true, testutils.ExcludeNamespaces("kube-system")). - WithIstioInput(false). - WithOTLPInput(true). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with all runtime input resources desiabled", - goldenFileName: "runtime-resources-all-disabled.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithRuntimeInput(true). - WithRuntimeInputPodMetrics(false). - WithRuntimeInputContainerMetrics(false). - WithRuntimeInputNodeMetrics(false). - WithRuntimeInputVolumeMetrics(false). - WithRuntimeInputDeploymentMetrics(false). - WithRuntimeInputDaemonSetMetrics(false). - WithRuntimeInputStatefulSetMetrics(false). - WithRuntimeInputJobMetrics(false). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with some runtime input resources disabled", - goldenFileName: "runtime-resources-some-disabled.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithRuntimeInput(true). - WithRuntimeInputPodMetrics(true). - WithRuntimeInputContainerMetrics(false). - WithRuntimeInputNodeMetrics(false). - WithRuntimeInputVolumeMetrics(true). - WithRuntimeInputDeploymentMetrics(true). - WithRuntimeInputDaemonSetMetrics(true). - WithRuntimeInputStatefulSetMetrics(true). - WithRuntimeInputJobMetrics(true). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with prometheus diagnostic metrics", - goldenFileName: "prometheus-diagnostic.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithPrometheusInput(true). - WithPrometheusInputDiagnosticMetrics(true). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with istio envoy metrics", - goldenFileName: "istio-envoy.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithIstioInput(true). - WithIstioInputEnvoyMetrics(true). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, - { - name: "pipeline with istio diagnostic metrics", - goldenFileName: "istio-diagnostic.yaml", - pipelines: []telemetryv1alpha1.MetricPipeline{ - testutils.NewMetricPipelineBuilder(). - WithName("cls"). - WithIstioInput(true). - WithIstioInputDiagnosticMetrics(true). - WithOTLPOutput(testutils.OTLPEndpoint("https://localhost")).Build(), - }, - }, { name: "pipeline with all inputs disabled except OTLP", goldenFileName: "otlp-only.yaml", @@ -265,7 +149,7 @@ func TestMakeConfig(t *testing.T) { }, { name: "two pipelines with user-defined Filter", - goldenFileName: "two-pipelines-with-filter.yaml", + goldenFileName: "user-defined-filters.yaml", pipelines: []telemetryv1alpha1.MetricPipeline{ testutils.NewMetricPipelineBuilder(). WithName("cls"). @@ -285,7 +169,7 @@ func TestMakeConfig(t *testing.T) { }, { name: "pipeline with user-defined Transform Filter", - goldenFileName: "pipeline-with-transform-filter.yaml", + goldenFileName: "user-defined-transform-filter.yaml", pipelines: []telemetryv1alpha1.MetricPipeline{ testutils.NewMetricPipelineBuilder(). WithName("cls"). @@ -314,7 +198,6 @@ func TestMakeConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { config, _, err := sut.Build(t.Context(), tt.pipelines, buildOptions) require.NoError(t, err) - configYAML, err := yaml.Marshal(config) require.NoError(t, err, "failed to marshal config") diff --git a/internal/otelcollector/config/metricgateway/testdata/http-protocol-with-custom-path.yaml b/internal/otelcollector/config/metricgateway/testdata/http-protocol-with-custom-path.yaml index a1183d1488..a27b9c97e0 100644 --- a/internal/otelcollector/config/metricgateway/testdata/http-protocol-with-custom-path.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/http-protocol-with-custom-path.yaml @@ -8,35 +8,23 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/test-attributes-enrichment: - receivers: - - routing/test - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/test - metrics/test-input: + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter + - transform/set-instrumentation-scope-kyma + - k8sattributes + - service_enrichment + - resource/insert-cluster-attributes + - resource/drop-kyma-attributes exporters: - - routing/test + - forward metrics/test-output: receivers: - - routing/test - - forward/test + - forward processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - batch exporters: - otlphttp/test @@ -84,26 +72,6 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" k8sattributes: auth_type: serviceAccount passthrough: false @@ -153,10 +121,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -191,12 +155,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/test: {} - routing/test: - default_pipelines: - - metrics/test-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/test-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/http-protocol-without-custom-path.yaml b/internal/otelcollector/config/metricgateway/testdata/http-protocol-without-custom-path.yaml index 93adc09efe..c821be7dab 100644 --- a/internal/otelcollector/config/metricgateway/testdata/http-protocol-without-custom-path.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/http-protocol-without-custom-path.yaml @@ -8,35 +8,23 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/test-attributes-enrichment: - receivers: - - routing/test - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/test - metrics/test-input: + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter + - transform/set-instrumentation-scope-kyma + - k8sattributes + - service_enrichment + - resource/insert-cluster-attributes + - resource/drop-kyma-attributes exporters: - - routing/test + - forward metrics/test-output: receivers: - - routing/test - - forward/test + - forward processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - batch exporters: - otlphttp/test @@ -84,26 +72,6 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" k8sattributes: auth_type: serviceAccount passthrough: false @@ -153,10 +121,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -191,12 +155,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/test: {} - routing/test: - default_pipelines: - - metrics/test-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/test-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/istio-envoy.yaml b/internal/otelcollector/config/metricgateway/testdata/istio-envoy.yaml deleted file mode 100644 index 52916e8f15..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/istio-envoy.yaml +++ /dev/null @@ -1,194 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-diagnostic-metrics-if-input-source-istio - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/drop-diagnostic-metrics-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" and (name == "up" or name == "scrape_duration_seconds" or name == "scrape_samples_scraped" or name == "scrape_samples_post_metric_relabeling" or name == "scrape_series_added") - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/istio-namespace-filters.yaml b/internal/otelcollector/config/metricgateway/testdata/istio-namespace-filters.yaml deleted file mode 100644 index ab8c737492..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/istio-namespace-filters.yaml +++ /dev/null @@ -1,207 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-envoy-metrics-if-disabled - - filter/cls-filter-by-namespace-istio-input - - filter/drop-diagnostic-metrics-if-input-source-istio - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/cls-filter-by-namespace-istio-input: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" and (resource.attributes["k8s.namespace.name"] == "kube-system" or resource.attributes["k8s.namespace.name"] == "istio-system") - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "monitoring" or resource.attributes["k8s.namespace.name"] == "observability") - filter/drop-diagnostic-metrics-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" and (name == "up" or name == "scrape_duration_seconds" or name == "scrape_samples_scraped" or name == "scrape_samples_post_metric_relabeling" or name == "scrape_series_added") - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/multiple-inputs-mixed.yaml b/internal/otelcollector/config/metricgateway/testdata/multiple-inputs-mixed.yaml deleted file mode 100644 index 7c4c05efd9..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/multiple-inputs-mixed.yaml +++ /dev/null @@ -1,206 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - filter/cls-filter-by-namespace-runtime-input - - filter/cls-filter-by-namespace-prometheus-input - - filter/drop-diagnostic-metrics-if-input-source-prometheus - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/cls-filter-by-namespace-prometheus-input: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" and (resource.attributes["k8s.namespace.name"] == "kube-system") - filter/cls-filter-by-namespace-runtime-input: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "default") - filter/drop-diagnostic-metrics-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/prometheus" and (name == "up" or name == "scrape_duration_seconds" or name == "scrape_samples_scraped" or name == "scrape_samples_post_metric_relabeling" or name == "scrape_series_added") - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/otlp-disabled.yaml b/internal/otelcollector/config/metricgateway/testdata/otlp-disabled.yaml index 7979483ce6..b7d02f620c 100644 --- a/internal/otelcollector/config/metricgateway/testdata/otlp-disabled.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/otlp-disabled.yaml @@ -8,36 +8,24 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/test-attributes-enrichment: - receivers: - - routing/test - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/test - metrics/test-input: + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter + - transform/set-instrumentation-scope-kyma + - k8sattributes + - service_enrichment + - resource/insert-cluster-attributes + - resource/drop-kyma-attributes exporters: - - routing/test + - forward metrics/test-output: receivers: - - routing/test - - forward/test + - forward processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - filter/drop-if-input-source-otlp - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - batch exporters: - otlp/test @@ -85,31 +73,11 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" filter/drop-if-input-source-otlp: error_mode: ignore metrics: metric: - - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or resource.attributes["kyma.input.name"] == "prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio" or instrumentation_scope.name == "io.kyma-project.telemetry/kyma") - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" + - not(instrumentation_scope.name == "io.kyma-project.telemetry/kyma") k8sattributes: auth_type: serviceAccount passthrough: false @@ -159,10 +127,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -195,12 +159,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/test: {} - routing/test: - default_pipelines: - - metrics/test-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/test-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/otlp-namespace-filters.yaml b/internal/otelcollector/config/metricgateway/testdata/otlp-namespace-filters.yaml index 7ba68ff854..7f329d6c3b 100644 --- a/internal/otelcollector/config/metricgateway/testdata/otlp-namespace-filters.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/otlp-namespace-filters.yaml @@ -8,39 +8,27 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/cls-attributes-enrichment: + metrics/cls-output: receivers: - - routing/cls + - forward processors: - - k8sattributes - - service_enrichment + - filter/cls-filter-by-namespace-otlp-input + - batch exporters: - - forward/cls - metrics/cls-input: + - otlp/cls + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - filter/cls-filter-by-namespace-otlp-input + - k8sattributes + - service_enrichment - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - resource/drop-kyma-attributes - - batch exporters: - - otlp/cls + - forward telemetry: metrics: readers: @@ -89,28 +77,8 @@ processors: error_mode: ignore metrics: metric: - - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or resource.attributes["kyma.input.name"] == "prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio" or instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and (resource.attributes["k8s.namespace.name"] == "kube-system" or resource.attributes["k8s.namespace.name"] == "istio-system") - - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or resource.attributes["kyma.input.name"] == "prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio" or instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "monitoring" or resource.attributes["k8s.namespace.name"] == "observability") - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" + - not(instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and (resource.attributes["k8s.namespace.name"] == "kube-system" or resource.attributes["k8s.namespace.name"] == "istio-system") + - not(instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "monitoring" or resource.attributes["k8s.namespace.name"] == "observability") k8sattributes: auth_type: serviceAccount passthrough: false @@ -160,10 +128,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -196,12 +160,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/otlp-only.yaml b/internal/otelcollector/config/metricgateway/testdata/otlp-only.yaml index bbff3b1516..e19c3d6e85 100644 --- a/internal/otelcollector/config/metricgateway/testdata/otlp-only.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/otlp-only.yaml @@ -8,38 +8,26 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/cls-attributes-enrichment: + metrics/cls-output: receivers: - - routing/cls + - forward processors: - - k8sattributes - - service_enrichment + - batch exporters: - - forward/cls - metrics/cls-input: + - otlp/cls + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled + - k8sattributes + - service_enrichment - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - resource/drop-kyma-attributes - - batch exporters: - - otlp/cls + - forward telemetry: metrics: readers: @@ -84,26 +72,6 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" k8sattributes: auth_type: serviceAccount passthrough: false @@ -153,10 +121,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -189,12 +153,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/pipeline-with-transform-filter.yaml b/internal/otelcollector/config/metricgateway/testdata/pipeline-with-transform-filter.yaml deleted file mode 100644 index 0980a96156..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/pipeline-with-transform-filter.yaml +++ /dev/null @@ -1,215 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - transform/user-defined-cls - - filter/user-defined-cls - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" - filter/user-defined-cls: - error_mode: ignore - metrics: - datapoint: - - metric.type == METRIC_DATA_TYPE_SUMMARY - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - transform/user-defined-cls: - error_mode: ignore - metric_statements: - - statements: - - set(attributes["log.level"], "error") - - set(body, "transformed2") - conditions: - - IsMatch(body, ".*error.*") -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/prometheus-namespace-filters.yaml b/internal/otelcollector/config/metricgateway/testdata/prometheus-namespace-filters.yaml deleted file mode 100644 index 217d080044..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/prometheus-namespace-filters.yaml +++ /dev/null @@ -1,207 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - filter/cls-filter-by-namespace-prometheus-input - - filter/drop-diagnostic-metrics-if-input-source-prometheus - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/cls-filter-by-namespace-prometheus-input: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" and (resource.attributes["k8s.namespace.name"] == "kube-system" or resource.attributes["k8s.namespace.name"] == "istio-system") - - resource.attributes["kyma.input.name"] == "prometheus" and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "monitoring" or resource.attributes["k8s.namespace.name"] == "observability") - filter/drop-diagnostic-metrics-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/prometheus" and (name == "up" or name == "scrape_duration_seconds" or name == "scrape_samples_scraped" or name == "scrape_samples_post_metric_relabeling" or name == "scrape_series_added") - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/runtime-namespace-filters.yaml b/internal/otelcollector/config/metricgateway/testdata/runtime-namespace-filters.yaml deleted file mode 100644 index 20fb20c4c1..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/runtime-namespace-filters.yaml +++ /dev/null @@ -1,201 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - filter/cls-filter-by-namespace-runtime-input - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/cls-filter-by-namespace-runtime-input: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and (resource.attributes["k8s.namespace.name"] == "kube-system" or resource.attributes["k8s.namespace.name"] == "istio-system") - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "monitoring" or resource.attributes["k8s.namespace.name"] == "observability") - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/runtime-resources-all-disabled.yaml b/internal/otelcollector/config/metricgateway/testdata/runtime-resources-all-disabled.yaml deleted file mode 100644 index a96107b0ac..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/runtime-resources-all-disabled.yaml +++ /dev/null @@ -1,242 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - filter/drop-runtime-pod-metrics - - filter/drop-runtime-container-metrics - - filter/drop-runtime-node-metrics - - filter/drop-runtime-volume-metrics - - filter/drop-runtime-deployment-metrics - - filter/drop-runtime-daemonset-metrics - - filter/drop-runtime-statefulset-metrics - - filter/drop-runtime-job-metrics - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-runtime-container-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "(^k8s.container.*)|(^container.*)") - filter/drop-runtime-daemonset-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.daemonset.*") - filter/drop-runtime-deployment-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.deployment.*") - filter/drop-runtime-job-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.job.*") - filter/drop-runtime-node-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.node.*") - filter/drop-runtime-pod-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.pod.*") - filter/drop-runtime-statefulset-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.statefulset.*") - filter/drop-runtime-volume-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.volume.*") - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/runtime-resources-some-disabled.yaml b/internal/otelcollector/config/metricgateway/testdata/runtime-resources-some-disabled.yaml deleted file mode 100644 index e397b35587..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/runtime-resources-some-disabled.yaml +++ /dev/null @@ -1,206 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - filter/drop-runtime-container-metrics - - filter/drop-runtime-node-metrics - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - batch - exporters: - - otlp/cls - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-runtime-container-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "(^k8s.container.*)|(^container.*)") - filter/drop-runtime-node-metrics: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and IsMatch(name, "^k8s.node.*") - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 256 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output diff --git a/internal/otelcollector/config/metricgateway/testdata/setup-comprehensive.yaml b/internal/otelcollector/config/metricgateway/testdata/setup-comprehensive.yaml index 70399c0d40..4626049e1d 100644 --- a/internal/otelcollector/config/metricgateway/testdata/setup-comprehensive.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/setup-comprehensive.yaml @@ -8,39 +8,39 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/cls-attributes-enrichment: + metrics/cls-output: receivers: - - routing/cls + - forward processors: - - k8sattributes - - service_enrichment + - filter/cls-filter-by-namespace-otlp-input + - transform/user-defined-cls + - filter/user-defined-cls + - batch exporters: - - forward/cls - metrics/cls-input: + - otlp/cls + metrics/dynatrace-output: receivers: - - otlp - - kymastats + - forward processors: - - memory_limiter + - filter/dynatrace-filter-by-namespace-otlp-input + - transform/user-defined-dynatrace + - filter/user-defined-dynatrace + - batch exporters: - - routing/cls - metrics/cls-output: + - otlp/dynatrace + metrics/input: receivers: - - routing/cls - - forward/cls + - otlp + - kymastats processors: + - memory_limiter - transform/set-instrumentation-scope-kyma - - filter/cls-filter-by-namespace-runtime-input - - filter/cls-filter-by-namespace-prometheus-input - - filter/cls-filter-by-namespace-otlp-input - - filter/drop-diagnostic-metrics-if-input-source-istio + - k8sattributes + - service_enrichment - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - resource/drop-kyma-attributes - - transform/user-defined-cls - - batch exporters: - - otlp/cls + - forward telemetry: metrics: readers: @@ -89,22 +89,22 @@ processors: error_mode: ignore metrics: metric: - - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or resource.attributes["kyma.input.name"] == "prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio" or instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "apps") - filter/cls-filter-by-namespace-prometheus-input: + - not(instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "apps-cls") + filter/dynatrace-filter-by-namespace-otlp-input: error_mode: ignore metrics: metric: - - resource.attributes["kyma.input.name"] == "prometheus" and (resource.attributes["k8s.namespace.name"] == "kube-system") - filter/cls-filter-by-namespace-runtime-input: + - not(instrumentation_scope.name == "io.kyma-project.telemetry/kyma") and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "apps-dynatrace") + filter/user-defined-cls: error_mode: ignore metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" and resource.attributes["k8s.namespace.name"] != nil and not(resource.attributes["k8s.namespace.name"] == "production" or resource.attributes["k8s.namespace.name"] == "staging") - filter/drop-diagnostic-metrics-if-input-source-istio: + datapoint: + - metric.type == METRIC_DATA_TYPE_GAUGE + filter/user-defined-dynatrace: error_mode: ignore metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" and (name == "up" or name == "scrape_duration_seconds" or name == "scrape_samples_scraped" or name == "scrape_samples_post_metric_relabeling" or name == "scrape_series_added") + datapoint: + - metric.type == METRIC_DATA_TYPE_SUMMARY k8sattributes: auth_type: serviceAccount passthrough: false @@ -154,10 +154,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -185,24 +181,33 @@ processors: - set(attributes["environment"], "prod") conditions: - resource.attributes["k8s.namespace.name"] == "production" + transform/user-defined-dynatrace: + error_mode: ignore + metric_statements: + - statements: + - set(attributes["environment"], "staging") + conditions: + - resource.attributes["k8s.namespace.name"] == "staging" exporters: otlp/cls: endpoint: ${OTLP_ENDPOINT_CLS} sending_queue: enabled: true - queue_size: 256 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + otlp/dynatrace: + endpoint: ${OTLP_ENDPOINT_DYNATRACE} + sending_queue: + enabled: true + queue_size: 128 retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/setup-simple.yaml b/internal/otelcollector/config/metricgateway/testdata/setup-simple.yaml index 35a8ee659b..0379ba04d0 100644 --- a/internal/otelcollector/config/metricgateway/testdata/setup-simple.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/setup-simple.yaml @@ -8,35 +8,23 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/test-attributes-enrichment: - receivers: - - routing/test - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/test - metrics/test-input: + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter + - transform/set-instrumentation-scope-kyma + - k8sattributes + - service_enrichment + - resource/insert-cluster-attributes + - resource/drop-kyma-attributes exporters: - - routing/test + - forward metrics/test-output: receivers: - - routing/test - - forward/test + - forward processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - batch exporters: - otlp/test @@ -84,26 +72,6 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" k8sattributes: auth_type: serviceAccount passthrough: false @@ -153,10 +121,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -189,12 +153,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/test: {} - routing/test: - default_pipelines: - - metrics/test-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/test-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/two-pipelines-with-filter.yaml b/internal/otelcollector/config/metricgateway/testdata/two-pipelines-with-filter.yaml deleted file mode 100644 index 55c106a967..0000000000 --- a/internal/otelcollector/config/metricgateway/testdata/two-pipelines-with-filter.yaml +++ /dev/null @@ -1,263 +0,0 @@ -extensions: - health_check: - endpoint: ${MY_POD_IP}:13133 - k8s_leader_elector: - auth_type: serviceAccount - lease_name: telemetry-metric-gateway-kymastats - pprof: - endpoint: 127.0.0.1:1777 -service: - pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - filter/user-defined-cls - - batch - exporters: - - otlp/cls - metrics/dynatrace-attributes-enrichment: - receivers: - - routing/dynatrace - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/dynatrace - metrics/dynatrace-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/dynatrace - metrics/dynatrace-output: - receivers: - - routing/dynatrace - - forward/dynatrace - processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - - filter/user-defined-dynatrace - - batch - exporters: - - otlp/dynatrace - telemetry: - metrics: - readers: - - pull: - exporter: - prometheus: - host: ${MY_POD_IP} - port: 8888 - logs: - level: info - encoding: json - extensions: - - health_check - - pprof - - k8s_leader_elector -receivers: - kymastats: - auth_type: serviceAccount - collection_interval: 30s - resources: - - group: operator.kyma-project.io - version: v1alpha1 - resource: telemetries - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: logpipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: tracepipelines - - group: telemetry.kyma-project.io - version: v1alpha1 - resource: metricpipelines - k8s_leader_elector: k8s_leader_elector - otlp: - protocols: - http: - endpoint: ${MY_POD_IP}:4318 - grpc: - endpoint: ${MY_POD_IP}:4317 -processors: - batch: - send_batch_size: 1024 - timeout: 10s - send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" - filter/user-defined-cls: - error_mode: ignore - metrics: - datapoint: - - metric.type == METRIC_DATA_TYPE_SUMMARY - filter/user-defined-dynatrace: - error_mode: ignore - metrics: - datapoint: - - metric.type == METRIC_DATA_TYPE_HISTOGRAM - k8sattributes: - auth_type: serviceAccount - passthrough: false - extract: - metadata: - - k8s.pod.name - - k8s.node.name - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - labels: - - from: pod - key: app.kubernetes.io/name - tag_name: kyma.kubernetes_io_app_name - - from: pod - key: app - tag_name: kyma.app_name - - from: node - key: topology.kubernetes.io/region - tag_name: cloud.region - - from: node - key: topology.kubernetes.io/zone - tag_name: cloud.availability_zone - - from: node - key: node.kubernetes.io/instance-type - tag_name: host.type - - from: node - key: kubernetes.io/arch - tag_name: host.arch - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - memory_limiter: - check_interval: 1s - limit_percentage: 75 - spike_limit_percentage: 15 - resource/drop-kyma-attributes: - attributes: - - action: delete - pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment - resource/insert-cluster-attributes: - attributes: - - action: insert - key: k8s.cluster.name - value: ${KUBERNETES_SERVICE_HOST} - - action: insert - key: k8s.cluster.uid - - action: insert - key: cloud.provider - value: test-cloud-provider - service_enrichment: - resource_attributes: - - kyma.kubernetes_io_app_name - - kyma.app_name - transform/set-instrumentation-scope-kyma: - error_mode: ignore - metric_statements: - - statements: - - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" -exporters: - otlp/cls: - endpoint: ${OTLP_ENDPOINT_CLS} - sending_queue: - enabled: true - queue_size: 128 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s - otlp/dynatrace: - endpoint: ${OTLP_ENDPOINT_DYNATRACE} - sending_queue: - enabled: true - queue_size: 128 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s -connectors: - forward/cls: {} - forward/dynatrace: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output - routing/dynatrace: - default_pipelines: - - metrics/dynatrace-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/dynatrace-output diff --git a/internal/otelcollector/config/metricgateway/testdata/prometheus-diagnostic.yaml b/internal/otelcollector/config/metricgateway/testdata/user-defined-filters.yaml similarity index 77% rename from internal/otelcollector/config/metricgateway/testdata/prometheus-diagnostic.yaml rename to internal/otelcollector/config/metricgateway/testdata/user-defined-filters.yaml index faac55293c..4c4c0846d4 100644 --- a/internal/otelcollector/config/metricgateway/testdata/prometheus-diagnostic.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/user-defined-filters.yaml @@ -8,37 +8,35 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/cls-attributes-enrichment: + metrics/cls-output: receivers: - - routing/cls + - forward processors: - - k8sattributes - - service_enrichment + - filter/user-defined-cls + - batch exporters: - - forward/cls - metrics/cls-input: + - otlp/cls + metrics/dynatrace-output: receivers: - - otlp - - kymastats + - forward processors: - - memory_limiter + - filter/user-defined-dynatrace + - batch exporters: - - routing/cls - metrics/cls-output: + - otlp/dynatrace + metrics/input: receivers: - - routing/cls - - forward/cls + - otlp + - kymastats processors: + - memory_limiter - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled + - k8sattributes + - service_enrichment - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - resource/drop-kyma-attributes - - batch exporters: - - otlp/cls + - forward telemetry: metrics: readers: @@ -83,21 +81,16 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: + filter/user-defined-cls: error_mode: ignore metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-runtime: + datapoint: + - metric.type == METRIC_DATA_TYPE_SUMMARY + filter/user-defined-dynatrace: error_mode: ignore metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" + datapoint: + - metric.type == METRIC_DATA_TYPE_HISTOGRAM k8sattributes: auth_type: serviceAccount passthrough: false @@ -147,10 +140,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -176,19 +165,21 @@ exporters: endpoint: ${OTLP_ENDPOINT_CLS} sending_queue: enabled: true - queue_size: 256 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + otlp/dynatrace: + endpoint: ${OTLP_ENDPOINT_DYNATRACE} + sending_queue: + enabled: true + queue_size: 128 retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/istio-diagnostic.yaml b/internal/otelcollector/config/metricgateway/testdata/user-defined-transform-filter.yaml similarity index 76% rename from internal/otelcollector/config/metricgateway/testdata/istio-diagnostic.yaml rename to internal/otelcollector/config/metricgateway/testdata/user-defined-transform-filter.yaml index e74596a5ed..1c29f9eff3 100644 --- a/internal/otelcollector/config/metricgateway/testdata/istio-diagnostic.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/user-defined-transform-filter.yaml @@ -8,37 +8,28 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/cls-attributes-enrichment: + metrics/cls-output: receivers: - - routing/cls + - forward processors: - - k8sattributes - - service_enrichment + - transform/user-defined-cls + - filter/user-defined-cls + - batch exporters: - - forward/cls - metrics/cls-input: + - otlp/cls + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter - exporters: - - routing/cls - metrics/cls-output: - receivers: - - routing/cls - - forward/cls - processors: - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-envoy-metrics-if-disabled + - k8sattributes + - service_enrichment - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - resource/drop-kyma-attributes - - batch exporters: - - otlp/cls + - forward telemetry: metrics: readers: @@ -83,21 +74,11 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: + filter/user-defined-cls: error_mode: ignore metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" + datapoint: + - metric.type == METRIC_DATA_TYPE_SUMMARY k8sattributes: auth_type: serviceAccount passthrough: false @@ -147,10 +128,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -171,6 +148,14 @@ processors: - statements: - set(scope.version, "") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" - set(scope.name, "io.kyma-project.telemetry/kyma") where scope.name == "github.com/kyma-project/opentelemetry-collector-components/receiver/kymastatsreceiver" + transform/user-defined-cls: + error_mode: ignore + metric_statements: + - statements: + - set(attributes["log.level"], "error") + - set(body, "transformed2") + conditions: + - IsMatch(body, ".*error.*") exporters: otlp/cls: endpoint: ${OTLP_ENDPOINT_CLS} @@ -183,12 +168,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/cls: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output + forward: {} diff --git a/internal/otelcollector/config/metricgateway/testdata/user-defined-transforms.yaml b/internal/otelcollector/config/metricgateway/testdata/user-defined-transforms.yaml index 519e304780..917815de9c 100644 --- a/internal/otelcollector/config/metricgateway/testdata/user-defined-transforms.yaml +++ b/internal/otelcollector/config/metricgateway/testdata/user-defined-transforms.yaml @@ -8,72 +8,35 @@ extensions: endpoint: 127.0.0.1:1777 service: pipelines: - metrics/cls-attributes-enrichment: - receivers: - - routing/cls - processors: - - k8sattributes - - service_enrichment - exporters: - - forward/cls - metrics/cls-input: - receivers: - - otlp - - kymastats - processors: - - memory_limiter - exporters: - - routing/cls metrics/cls-output: receivers: - - routing/cls - - forward/cls + - forward processors: - - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled - - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - - resource/drop-kyma-attributes - transform/user-defined-cls - batch exporters: - otlp/cls - metrics/dynatrace-attributes-enrichment: + metrics/dynatrace-output: receivers: - - routing/dynatrace + - forward processors: - - k8sattributes - - service_enrichment + - transform/user-defined-dynatrace + - batch exporters: - - forward/dynatrace - metrics/dynatrace-input: + - otlp/dynatrace + metrics/input: receivers: - otlp - kymastats processors: - memory_limiter - exporters: - - routing/dynatrace - metrics/dynatrace-output: - receivers: - - routing/dynatrace - - forward/dynatrace - processors: - transform/set-instrumentation-scope-kyma - - filter/drop-if-input-source-runtime - - filter/drop-if-input-source-prometheus - - filter/drop-if-input-source-istio - - filter/drop-envoy-metrics-if-disabled + - k8sattributes + - service_enrichment - resource/insert-cluster-attributes - - resource/drop-skip-enrichment-attribute - resource/drop-kyma-attributes - - transform/user-defined-dynatrace - - batch exporters: - - otlp/dynatrace + - forward telemetry: metrics: readers: @@ -118,26 +81,6 @@ processors: send_batch_size: 1024 timeout: 10s send_batch_max_size: 1024 - filter/drop-envoy-metrics-if-disabled: - error_mode: ignore - metrics: - metric: - - IsMatch(name, "^envoy_.*") and instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-istio: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/istio" - filter/drop-if-input-source-prometheus: - error_mode: ignore - metrics: - metric: - - resource.attributes["kyma.input.name"] == "prometheus" - filter/drop-if-input-source-runtime: - error_mode: ignore - metrics: - metric: - - instrumentation_scope.name == "io.kyma-project.telemetry/runtime" k8sattributes: auth_type: serviceAccount passthrough: false @@ -187,10 +130,6 @@ processors: attributes: - action: delete pattern: kyma.* - resource/drop-skip-enrichment-attribute: - attributes: - - action: delete - key: io.kyma-project.telemetry.skip_enrichment resource/insert-cluster-attributes: attributes: - action: insert @@ -249,21 +188,4 @@ exporters: max_interval: 30s max_elapsed_time: 300s connectors: - forward/cls: {} - forward/dynatrace: {} - routing/cls: - default_pipelines: - - metrics/cls-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/cls-output - routing/dynatrace: - default_pipelines: - - metrics/dynatrace-attributes-enrichment - error_mode: ignore - table: - - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" - pipelines: - - metrics/dynatrace-output + forward: {} diff --git a/internal/otelcollector/config/tracegateway/config_builder.go b/internal/otelcollector/config/tracegateway/config_builder.go index 6ffb2b14ae..031e88e71f 100644 --- a/internal/otelcollector/config/tracegateway/config_builder.go +++ b/internal/otelcollector/config/tracegateway/config_builder.go @@ -43,6 +43,8 @@ func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.Trace b.addIstioNoiseFilterProcessor(), b.addInsertClusterAttributesProcessor(opts), b.addServiceEnrichmentProcessor(), + // Kyma attributes are dropped before user-defined transform and filter processors + // to prevent user access to internal attributes. b.addDropKymaAttributesProcessor(), b.addUserDefinedTransformProcessor(), b.addUserDefinedFilterProcessor(), diff --git a/internal/otelcollector/config/tracegateway/config_builder_test.go b/internal/otelcollector/config/tracegateway/config_builder_test.go index b4a0cd6cf0..498756a4ef 100644 --- a/internal/otelcollector/config/tracegateway/config_builder_test.go +++ b/internal/otelcollector/config/tracegateway/config_builder_test.go @@ -75,7 +75,7 @@ func TestBuildConfig(t *testing.T) { }, }).Build(), }, - goldenFileName: "two-pipelines-with-transforms.yaml", + goldenFileName: "user-defined-transforms.yaml", }, { name: "pipeline with user-defined filters", @@ -91,7 +91,7 @@ func TestBuildConfig(t *testing.T) { Conditions: []string{"IsMatch(spanevent.attributes[\"foo\"], \".*bar.*\")"}, }).Build(), }, - goldenFileName: "two-pipelines-with-filter.yaml", + goldenFileName: "user-defined-filters.yaml", }, { name: "pipeline with user-defined transform and filters", @@ -109,7 +109,7 @@ func TestBuildConfig(t *testing.T) { }, }).Build(), }, - goldenFileName: "pipeline-with-transform-filter.yaml", + goldenFileName: "user-defined-transform-filter.yaml", }, } diff --git a/internal/otelcollector/config/tracegateway/testdata/two-pipelines-with-filter.yaml b/internal/otelcollector/config/tracegateway/testdata/user-defined-filters.yaml similarity index 100% rename from internal/otelcollector/config/tracegateway/testdata/two-pipelines-with-filter.yaml rename to internal/otelcollector/config/tracegateway/testdata/user-defined-filters.yaml diff --git a/internal/otelcollector/config/tracegateway/testdata/pipeline-with-transform-filter.yaml b/internal/otelcollector/config/tracegateway/testdata/user-defined-transform-filter.yaml similarity index 100% rename from internal/otelcollector/config/tracegateway/testdata/pipeline-with-transform-filter.yaml rename to internal/otelcollector/config/tracegateway/testdata/user-defined-transform-filter.yaml diff --git a/internal/otelcollector/config/tracegateway/testdata/two-pipelines-with-transforms.yaml b/internal/otelcollector/config/tracegateway/testdata/user-defined-transforms.yaml similarity index 100% rename from internal/otelcollector/config/tracegateway/testdata/two-pipelines-with-transforms.yaml rename to internal/otelcollector/config/tracegateway/testdata/user-defined-transforms.yaml