Filter out log lines not matching list

charts/meta-monitoring/templates/agent/_helpers-agent.tpl

@@ -18,10 +18,10 @@
 {{- end }}
 
 {{- define "agent.loki_process_targets" -}}
-{{- if empty .Values.logs.piiRegexes }}
+{{- if and (empty .Values.logs.piiRegexes) (empty .Values.logs.retain) }}
 {{- include "agent.loki_write_targets" . }}
 {{- else }}
-{{- printf "loki.process.PII.receiver" }}
+{{- printf "loki.process.filter.receiver" }}
 {{- end }}
 {{- end }}
 

charts/meta-monitoring/templates/agent/config.yaml

@@ -45,17 +45,26 @@ data:
       forward_to = [ {{ include "agent.loki_process_targets" . }} ]
     }
 
-    {{- if not (empty .Values.logs.piiRegexes) }}
-    loki.process "PII" {
+    {{- if or (not (empty .Values.logs.retain)) (not (empty .Values.logs.piiRegexes)) }}
+    loki.process "filter" {
       forward_to = [ {{ include "agent.loki_write_targets" . }} ]
 
+      {{- if not (empty .Values.logs.retain) }}
+      stage.match {
+        selector = "{cluster=\"{{- .Values.clusterName -}}\"} !~ \"{{ join "|" .Values.logs.retain }}\""
+        action   = "drop"
+      }
+      {{- end }}
+
+      {{- if not (empty .Values.logs.piiRegexes) }}
       {{- range .Values.logs.piiRegexes }}
       stage.replace {
         expression = "{{ .expression }}"
         source = "{{ .source }}"
         replace = "{{ .replace }}"
       }
       {{- end }}
+      {{- end }}
     }
     {{- end }}
     {{- end }}

charts/meta-monitoring/values.yaml

@@ -35,15 +35,24 @@ local:
   minio:
     enabled: false  # This should be set to true if any of the previous is enabled
 
-# Adding regexes here will add a stage.replace block for logs. For more information see
-# https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block
 logs:
+  # Adding regexes here will add a stage.replace block for logs. For more information see
+  # https://grafana.com/docs/agent/latest/flow/reference/components/loki.process/#stagereplace-block
   piiRegexes:
   # This example replaces the word after password with *****
   # - expression: "password (\\\\S+)"
   #   source: ""         # Empty uses the log message
   #   replace: "*****""
 
+  # The lines matching these will be kept in Loki
+  retain:
+  # This shows the queries
+  - caller=metrics.go
+  # This shows any errors
+  - level=error
+  # This shows the ingest requests and is very noisy. Uncomment to include.
+  # - caller=push.go
+
 # Set enabled = true to add the default logs/metrics/traces dashboards to the local Grafana
 dashboards:
   logs: