[PFMENG-3093] add custom parser variable (#176)

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] add custom parser variable

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] change s3_key_format

* [PFMENG-3093] add modify parameter

* [PFMENG-3093] add modify parameter

* [PFMENG-3093] add modify parameter

* [PFMENG-3093] add k8 api endpoint

* [PFMENG-3093] remove modify filter

* [PFMENG-3093] add k8 api endpoint

* [PFMENG-3093] add k8 api endpoint

Author: zodilib

modules/essentials/fluent_bit.tf

@@ -32,6 +32,7 @@ locals {
     service_account_name = local.service_account_name,
     image_repository     = var.fluent_bit_image_repository,
     image_tag            = var.fluent_bit_image_tag,
+    custom_parser        = var.fluent_bit_custom_parser,
     liveness_probe       = jsonencode(var.fluent_bit_liveness_probe),
     readiness_probe      = jsonencode(var.fluent_bit_readiness_probe),
     resources            = jsonencode(var.fluent_bit_resources),
@@ -40,6 +41,7 @@ locals {
     excluded_namespaces  = var.fluent_bit_excluded_namespaces,
     s3_bucket_name       = var.fluent_bit_enable_s3_output ? module.fluentbit_s3_bucket[0].s3_bucket_id : null,
     cw_enable            = var.fluent_bit_enable_cw_output
+    kube_api_endpoint    = var.fluent_bit_kube_api_endpoint
   })
 
   fluent_bit_helm_config = merge(

modules/essentials/templates/fluent_bit.yaml

@@ -56,12 +56,12 @@ config:
   ## Default parsers are from : https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf
   customParsers: |
     [PARSER]
-        Name custom_apache
+        Name ${custom_parser.name}
         Format regex
-        Regex ^(?<client_ip>[^ ]*) \<(?<x_forwarded_for>[^\"]*)\> (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<latency>[^\"]*)" "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Regex ${custom_parser.regex}
         Time_Keep Off
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
+        Time_Key ${custom_parser.time_key}
+        Time_Format ${custom_parser.time_format}
 
   ## https://docs.fluentbit.io/manual/pipeline/inputs
   inputs: |
@@ -92,18 +92,19 @@ config:
     [FILTER]
         Name kubernetes
         Match kube.*
-        Kube_URL https://kubernetes.default.svc.cluster.local:443
+        Kube_URL ${kube_api_endpoint}
         Merge_Log On
         Keep_Log Off
         K8S-Logging.Parser On
         K8S-Logging.Exclude On
         Buffer_Size 31k
+        Kube_Tag_Prefix kube.var.log.containers.
 
     [FILTER]
         Name parser
         Match kube.*
         Key_Name log
-        Parser custom_apache
+        Parser ${custom_parser.name}
         Reserve_Data true
 
 %{if length(excluded_namespaces) > 0}
@@ -119,18 +120,20 @@ config:
     [FILTER]
         Name kubernetes
         Match kube_s3.*
-        Kube_URL https://kubernetes.default.svc.cluster.local:443
+        Kube_URL ${kube_api_endpoint}
         Merge_Log On
         Keep_Log Off
         K8S-Logging.Parser On
         Buffer_Size 31k
+        Kube_Tag_Prefix kube_s3.var.log.containers.
 
     [FILTER]
         Name parser
         Match kube_s3.*
         Key_Name log
-        Parser custom_apache
+        Parser ${custom_parser.name}
         Reserve_Data true
+
 %{ endif }
 
   ## https://docs.fluentbit.io/manual/pipeline/outputs
@@ -152,9 +155,9 @@ config:
         Match kube_s3.*
         region ap-southeast-1
         bucket ${s3_bucket_name}
-        compression gzip
         storage_class REDUCED_REDUNDANCY
         retry_limit 2
+        s3_key_format /fluent-bit-logs/%Y/%m/%d/%H/%M/$TAG
 %{ endif }
 
 #  extraFiles: {}

modules/essentials/variables.tf

@@ -1487,6 +1487,24 @@ variable "fluent_bit_log_group_retention" {
   default     = 30
 }
 
+variable "fluent_bit_custom_parser" {
+  description = "Custom parser for Fluent Bit"
+  type = object({
+    name        = string
+    format      = string
+    regex       = optional(string)
+    time_key    = string
+    time_format = string
+  })
+  default = {
+    name        = "custom_apache"
+    format      = "regex"
+    regex       = "^(?<client_ip>[^ ]*) \\<(?<x_forwarded_for>[^\\\"]*)\\> (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \"(?<latency>[^\\\"]*)\" \"(?<method>\\S+)(?: +(?<path>[^ ]*) +\\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>[^\\\"]*)\" \"(?<agent>[^\\\"]*)\")?$"
+    time_key    = "time"
+    time_format = "%d/%b/%Y:%H:%M:%S %z"
+  }
+}
+
 variable "resolve_conflicts_on_update" {
   description = "value for resolve_conflicts_on_update for aws_eks_addon resource"
   type        = string
@@ -1553,6 +1571,12 @@ variable "fluent_bit_tolerations" {
   ]
 }
 
+variable "fluent_bit_kube_api_endpoint" {
+  description = "Kube API endpoint for fluent-bit"
+  type        = string
+  default     = "https://kubernetes.default.svc.cluster.local:443"
+}
+
 variable "ip_dual_stack_enabled" {
   description = "Enable essentials to support EKS dual stack cluster"
   type        = bool