Skip to content

Commit

Permalink
Rebuild the AppO11y pipeline so it's defined in a pipeline.tpl file. …
Browse files Browse the repository at this point in the history
…Much easier to modify and understand.

Signed-off-by: Pete Wall <[email protected]>
  • Loading branch information
petewall committed Jan 23, 2025
1 parent 71df87a commit 72a68cf
Show file tree
Hide file tree
Showing 28 changed files with 302 additions and 282 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ otelcol.connector.host_info "{{ .name | default "default" }}" {
host_identifiers = [ "k8s.node.name" ]
output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,4 @@
{{- define "feature.applicationObservability.module" }}
{{- $metricsNext := "" }}
{{- $logsNext := "" }}
{{- $resourceDetection := include "feature.applicationObservability.processor.resourcedetection.alloy.target" dict }}
{{- $k8sAttributes := include "feature.applicationObservability.processor.k8sattributes.alloy.target" dict }}
{{- $grafanaCloudMetrics := include "feature.applicationObservability.connector.host_info.alloy.target" dict }}
{{- $transform := include "feature.applicationObservability.processor.transform.alloy.target" dict }}
{{- $filter := include "feature.applicationObservability.processor.filter.alloy.target" dict }}
{{- $batch := include "feature.applicationObservability.processor.batch.alloy.target" dict }}
{{- $interval := include "feature.applicationObservability.processor.interval.alloy.target" dict }}
{{- $memoryLimiter := include "feature.applicationObservability.processor.memory_limiter.alloy.target" dict }}
declare "application_observability" {
argument "metrics_destinations" {
comment = "Must be a list of metrics destinations where collected metrics should be forwarded to"
Expand All @@ -21,86 +11,24 @@ declare "application_observability" {
argument "traces_destinations" {
comment = "Must be a list of trace destinations where collected trace should be forwarded to"
}

// Receivers --> Resource Detection Processor
{{- $next := printf "[%s]" $resourceDetection }}
{{- include "feature.applicationObservability.receiver.otlp.alloy" (dict "Values" $.Values "metricsOutput" $next "logsOutput" $next "tracesOutput" $next ) | indent 2 }}
{{- include "feature.applicationObservability.receiver.jaeger.alloy" (dict "Values" $.Values "tracesOutput" $next ) | indent 2 }}
{{- include "feature.applicationObservability.receiver.zipkin.alloy" (dict "Values" $.Values "tracesOutput" $next ) | indent 2 }}

// Resource Detection Processor --> K8s Attribute Processor
{{- $next = printf "[%s]" $k8sAttributes }}
{{- include "feature.applicationObservability.processor.resourcedetection.alloy" (dict "Values" $.Values "metricsOutput" $next "logsOutput" $next "tracesOutput" $next ) | indent 2 }}

// K8s Attribute Processor --> Transform Processor
{{- $tracesNext := list $transform }}
{{- $next = printf "[%s]" $transform }}
{{- if (index .Values.processors "grafanaCloudMetrics").enabled | default .Values.connectors.grafanaCloudMetrics.enabled }}
// Resource Detection Processor Traces --> Host Info Connector
{{- $tracesNext = append $tracesNext $grafanaCloudMetrics }}
{{- end -}}
{{- $tracesNext = printf "[%s]" ($tracesNext | join ", ")}}
{{- include "feature.applicationObservability.processor.k8sattributes.alloy" (dict "Values" $.Values "metricsOutput" $next "logsOutput" $next "tracesOutput" $tracesNext ) | indent 2 }}

{{- if (index .Values.processors "grafanaCloudMetrics").enabled | default .Values.connectors.grafanaCloudMetrics.enabled }}
// Host Info Connector --> Batch Processor
{{- $next = printf "[%s]" $batch }}
{{- include "feature.applicationObservability.connector.host_info.alloy" (dict "Values" $.Values "metricsOutput" $next ) | indent 2 }}
{{- end }}

{{ if eq (include "feature.applicationObservability.processor.filter.enabled" .) "true" }}
// Transform Processor --> Filter Processor
{{- $next = printf "[%s]" $filter }}
{{- else }}
// Transform Processor --> Batch Processor
{{- $next = printf "[%s]" $batch }}
{{- end }}
{{- include "feature.applicationObservability.processor.transform.alloy" (dict "Values" $.Values "metricsOutput" $next "logsOutput" $next "tracesOutput" $next ) | indent 2 }}
{{ if eq (include "feature.applicationObservability.processor.filter.enabled" .) "true" }}
// Filter Processor --> Batch Processor
{{- $next = printf "[%s]" $batch }}
{{- include "feature.applicationObservability.processor.filter.alloy" (dict "Values" $.Values "metricsOutput" $next "logsOutput" $next "tracesOutput" $next ) | indent 2 }}
{{- end }}

{{- if .Values.processors.memoryLimiter.enabled }}
// Batch Processor --> Memory Limiter
{{- $metricsNext = printf "[%s]" $memoryLimiter }}
{{- $logsNext = printf "[%s]" $memoryLimiter }}
{{- $tracesNext = printf "[%s]" $memoryLimiter }}
{{- else if .Values.processors.interval.enabled }}
// Batch Processor --> Interval
{{- $metricsNext = printf "[%s]" $interval }}
{{- $logsNext = printf "[%s]" $interval }}
{{- $tracesNext = printf "[%s]" $interval }}
{{- else }}
// Batch Processor --> Destinations
{{- $metricsNext = "argument.metrics_destinations.value" }}
{{- $logsNext = "argument.logs_destinations.value" }}
{{- $tracesNext = "argument.traces_destinations.value" }}
{{- end }}
{{- include "feature.applicationObservability.processor.batch.alloy" (dict "Values" $.Values "metricsOutput" $metricsNext "logsOutput" $logsNext "tracesOutput" $tracesNext ) | indent 2 }}

{{- if .Values.processors.memoryLimiter.enabled }}
{{- if .Values.processors.interval.enabled }}
// Memory Limiter --> Interval
{{- $metricsNext = printf "[%s]" $interval }}
{{- $logsNext = printf "[%s]" $interval }}
{{- $tracesNext = printf "[%s]" $interval }}
{{- else }}
// Memory Limiter --> Destinations
{{- $metricsNext = "argument.metrics_destinations.value" }}
{{- $logsNext = "argument.logs_destinations.value" }}
{{- $tracesNext = "argument.traces_destinations.value" }}
{{- end }}
{{- include "feature.applicationObservability.processor.memory_limiter.alloy" (dict "Values" $.Values "metricsOutput" $metricsNext "logsOutput" $logsNext "tracesOutput" $tracesNext ) | indent 2 }}
{{- end }}

{{- if .Values.processors.interval.enabled }}
// Interval --> Destinations
{{- $metricsNext = "argument.metrics_destinations.value" }}
{{- $logsNext = "argument.logs_destinations.value" }}
{{- $tracesNext = "argument.traces_destinations.value" }}
{{- include "feature.applicationObservability.processor.interval.alloy" (dict "Values" $.Values "metricsOutput" $metricsNext "logsOutput" $logsNext "tracesOutput" $tracesNext ) | indent 2 }}
{{- $pipeline := include "feature.applicationObservability.pipeline" . | fromYamlArray }}
{{- range $component := $pipeline }}
{{- $args := (dict "Values" $.Values "name" $component.name) }}

{{- range $dataType := (list "metrics" "logs" "traces")}}
{{- if kindIs "string" (index $component.targets $dataType) }}
{{- $args = merge $args (dict $dataType (index $component.targets $dataType)) }}
{{- else if kindIs "slice" (index $component.targets $dataType) }}
{{- $targets := list }}
{{- range $target := (index $component.targets $dataType) }}
{{- $targets = append $targets (include (printf "feature.applicationObservability.%s.alloy.target" $target.component) $target) }}
{{- end }}
{{- $args = merge $args (dict $dataType (printf "[%s]" (join ", " $targets))) }}
{{- end }}
{{- end }}

// {{ $component.description }}
{{- include (printf "feature.applicationObservability.%s.alloy" $component.component) $args | indent 2 }}
{{- end }}
}
{{- end }}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{{- define "feature.applicationObservability.pipeline" }}
# Format:
# - name: Alloy component name
# description: Human friendly description of the component
# component: Component slug (used for including "feature.applicationObservability.%s.alloy")
# targets:
# <type>: <list>
# - name: Name of the target
# component: Component slug (used for including "feature.applicationObservability.%s.alloy.target")
# <type>: <string> Raw target string (useful for terminating with argument.<type>_destinations.value)
# <type>: <null> No target defined for this type
{{- if or .Values.receivers.otlp.grpc.enabled .Values.receivers.otlp.http.enabled }}
- name: default
description: OTLP Receiver
component: receiver.otlp
targets:
metrics: [{name: default, component: processor.resourcedetection}]
logs: [{name: default, component: processor.resourcedetection}]
traces: [{name: default, component: processor.resourcedetection}]
{{- end }}
{{- if or .Values.receivers.jaeger.grpc.enabled .Values.receivers.jaeger.thriftBinary.enabled .Values.receivers.jaeger.thriftCompact.enabled .Values.receivers.jaeger.thriftBinary.enabled }}
- name: default
description: Jaeger Receiver
component: receiver.jaeger
targets:
traces: [{name: default, component: processor.resourcedetection}]
{{- end }}
{{- if .Values.receivers.zipkin.enabled }}
- name: default
description: Zipkin Receiver
component: receiver.zipkin
targets:
traces: [{name: default, component: processor.resourcedetection}]
{{- end }}

- name: default
description: Resource Detection Processor
component: processor.resourcedetection
targets:
metrics: [{name: default, component: processor.k8sattributes}]
logs: [{name: default, component: processor.k8sattributes}]
traces: [{name: default, component: processor.k8sattributes}]

- name: default
description: K8s Attributes Processor
component: processor.k8sattributes
targets:
metrics: [{name: default, component: processor.transform}]
logs: [{name: default, component: processor.transform}]
{{- if (index .Values.processors "grafanaCloudMetrics").enabled | default .Values.connectors.grafanaCloudMetrics.enabled }}
traces: [{name: default, component: processor.transform}, {name: default, component: connector.host_info}]

- name: default
description: Host Info Connector
component: connector.host_info
targets:
metrics: [{name: default, component: processor.batch}]
{{- else }}
traces: [{name: default, component: processor.transform}]
{{- end }}

- name: default
description: Transform Processor
component: processor.transform
targets:
{{- if eq (include "feature.applicationObservability.processor.filter.enabled" .) "true" }}
metrics: [{name: default, component: processor.filter}]
logs: [{name: default, component: processor.filter}]
traces: [{name: default, component: processor.filter}]

- name: default
description: Filter Processor
component: processor.filter
targets:
{{- end }}
metrics: [{name: default, component: processor.batch}]
logs: [{name: default, component: processor.batch}]
traces: [{name: default, component: processor.batch}]

- name: default
description: Batch Processor
component: processor.batch
targets:
{{- if .Values.processors.memoryLimiter.enabled }}
metrics: [{name: default, component: processor.memory_limiter}]
logs: [{name: default, component: processor.memory_limiter}]
traces: [{name: default, component: processor.memory_limiter}]

- name: default
description: Memory Limiter
component: processor.memory_limiter
targets:
{{- end }}
{{- if .Values.processors.interval.enabled }}
metrics: [{name: default, component: processor.interval}]
logs: [{name: default, component: processor.interval}]
traces: [{name: default, component: processor.interval}]

- name: default
description: Interval Processor
component: processor.interval
targets:
{{- end }}
metrics: argument.metrics_destinations.value
logs: argument.logs_destinations.value
traces: argument.traces_destinations.value

{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ otelcol.processor.batch {{ .name | default "default" | quote }} {
timeout = {{ .Values.processors.batch.timeout | quote}}

output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,14 @@ otelcol.processor.filter "{{ .name | default "default" }}" {
}
{{- end }}
output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ otelcol.processor.interval {{ .name | default "default" | quote }} {
}

output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ otelcol.processor.k8sattributes "{{ .name | default "default" }}" {
}

output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ otelcol.processor.memory_limiter "{{ .name | default "default" }}" {
limit = {{ .Values.processors.memoryLimiter.limit | quote }}

output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ otelcol.processor.resourcedetection "{{ .name | default "default" }}" {
}

output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,14 +96,14 @@ otelcol.processor.transform "{{ .name | default "default" }}" {
{{- end }}

output {
{{- if and .metricsOutput .Values.metrics.enabled }}
metrics = {{ .metricsOutput }}
{{- if and .metrics .Values.metrics.enabled }}
metrics = {{ .metrics }}
{{- end }}
{{- if and .logsOutput .Values.logs.enabled }}
logs = {{ .logsOutput }}
{{- if and .logs .Values.logs.enabled }}
logs = {{ .logs }}
{{- end }}
{{- if and .tracesOutput .Values.traces.enabled }}
traces = {{ .tracesOutput }}
{{- if and .traces .Values.traces.enabled }}
traces = {{ .traces }}
{{- end }}
}
}
Expand Down
Loading

0 comments on commit 72a68cf

Please sign in to comment.