From f39de789b4c70c9d9eb4da257e1377926a3b5f7e Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 19 Dec 2024 12:34:30 -0500 Subject: [PATCH 1/9] gotta catch then all --- .../src/logstash-filters/snippets/syslog_standard.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index 529d7e65..3e574ce8 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -68,7 +68,7 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { # structured-data if [syslog_sd] { grok { - match => [ "syslog_sd", "\[%{DATA:syslog_sd_id} (?[^\]]+)\]" ] + match => [ "syslog_sd", "\[%{DATA:syslog_sd_id} (?[^\]]+)\][^\]]+)\]" ] remove_field => [ "syslog_sd" ] From 8017629d23bdf259fd0119f63ad672cb78bbf1e8 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 19 Dec 2024 15:10:31 -0500 Subject: [PATCH 2/9] gsub out brackets --- .../src/logstash-filters/snippets/syslog_standard.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index 3e574ce8..813f96b9 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -68,7 +68,7 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { # structured-data if [syslog_sd] { grok { - match => [ "syslog_sd", "\[%{DATA:syslog_sd_id} (?[^\]]+)\][^\]]+)\]" ] + match => [ "syslog_sd", "\[%{DATA:syslog_sd_id} (?[^\]]+)\]" ] remove_field => [ "syslog_sd" ] @@ -76,8 +76,14 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { tag_on_timeout => [ "fail/syslog_standard/_grokparsetimeout-syslog_standard-5424/sds" ] } + if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs + mutate{ + gsub => [ + "syslog_sd_params_raw", "[]", "" + ] + } kv { source => "syslog_sd_params_raw" target => "syslog_sd_params" From 7bb832466af437408c4120a80f767d9b605aa5f5 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 19 Dec 2024 15:31:42 -0500 Subject: [PATCH 3/9] gsub out brackets --- .../src/logstash-filters/snippets/syslog_standard.conf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index 813f96b9..ed9bd180 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -79,10 +79,8 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs - mutate{ - gsub => [ - "syslog_sd_params_raw", "[]", "" - ] + mutate { + gsub => [ "syslog_sd_params_raw", '[]', ""] } kv { source => "syslog_sd_params_raw" From 5ac667f0332833cf1fb12754ebfce6a1a2c4f157 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 19 Dec 2024 15:54:59 -0500 Subject: [PATCH 4/9] gsub out brackets --- .../src/logstash-filters/snippets/syslog_standard.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index ed9bd180..bdcd62f2 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -80,7 +80,7 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs mutate { - gsub => [ "syslog_sd_params_raw", '[]', ""] + gsub => [ "syslog_sd_params_raw", '\[\]', ""] } kv { source => "syslog_sd_params_raw" From 9883774cd04b09a5b955dfa4564536fcad897478 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 19 Dec 2024 16:53:35 -0500 Subject: [PATCH 5/9] replacing these values --- .../src/logstash-filters/snippets/syslog_standard.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index bdcd62f2..80f213e0 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -80,7 +80,7 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs mutate { - gsub => [ "syslog_sd_params_raw", '\[\]', ""] + gsub => [ "syslog_sd_params_raw", "[]", ""] } kv { source => "syslog_sd_params_raw" From ff5e228832dc053058b8050dd07db3191f8b2c4d Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 20 Dec 2024 11:41:31 -0500 Subject: [PATCH 6/9] replacing these values --- .../src/logstash-filters/snippets/syslog_standard.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index 80f213e0..79059f43 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -80,7 +80,8 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs mutate { - gsub => [ "syslog_sd_params_raw", "[]", ""] + gsub => [ "syslog_sd_params_raw", "\[", "" ] + gsub => [ "syslog_sd_params_raw", "\]", "" ] } kv { source => "syslog_sd_params_raw" From 174ebb445b8bc5e7a7a6833423ae62086fd862c6 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 20 Dec 2024 15:35:47 -0500 Subject: [PATCH 7/9] better organization --- .../templates/config/input_and_output.conf.erb | 12 ++++-------- .../logstash-filters/snippets/syslog_standard.conf | 5 +---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index dd824cd2..5b3d31f5 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -21,28 +21,25 @@ filter rename => {"[cloudwatch_logs][tags][OrganizationGUID]"=>"[@cf][org_id]"} rename => {"[cloudwatch_logs][tags][Organization GUID]"=>"[@cf][org_id]"} + rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} + rename => {"[cloudwatch_logs][tags][Organization name]"=>"[@cf][org]"} rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"} rename => {"[cloudwatch_logs][tags][Space GUID]"=>"[@cf][space_id]"} - rename => {"[cloudwatch_logs][tags][Spacename]"=>"[@cf][space]"} rename => {"[cloudwatch_logs][tags][Space name]"=>"[@cf][space]"} - rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} - rename => {"[cloudwatch_logs][tags][Organization name]"=>"[@cf][org]"} - rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"[@cf][service_instance_id]"} rename => {"[cloudwatch_logs][tags][Instance GUID]"=>"[@cf][service_instance_id]"} - rename => {"[cloudwatch_logs][tags][Instance name]"=>"[@cf][service]"} rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"[@cf][service_offering]"} rename => {"[cloudwatch_logs][tags][Service offering name]"=>"[@cf][service_offering]"} - rename => {"[cloudwatch_logs][tags][Service plan name]"=>"[@cf][service_plan]"} - rename => {"[cloudwatch_logs][tags][service]"=>"broker"} + rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} + rename => {"message" => "@message" } remove_field => ["[cloudwatch_logs][tags][Createdat]"] remove_field => ["[cloudwatch_logs][tags][Created at]"] @@ -54,7 +51,6 @@ filter remove_field => ["[cloudwatch_logs][tags][ServiceGUID]"] remove_field => ["[cloudwatch_logs][tags][Service GUID]"] - rename => [ "message" => "@message" ] } truncate { fields => ["@message"] diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index 79059f43..124cfc01 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -79,10 +79,7 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs - mutate { - gsub => [ "syslog_sd_params_raw", "\[", "" ] - gsub => [ "syslog_sd_params_raw", "\]", "" ] - } + kv { source => "syslog_sd_params_raw" target => "syslog_sd_params" From 8cfeb72d68cb70cae19cf6dbef62f06442aa3155 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 20 Dec 2024 15:37:43 -0500 Subject: [PATCH 8/9] more cleanup --- .../ingestor_cloudwatch/templates/bin/ingestor_cloudwatch | 8 -------- 1 file changed, 8 deletions(-) diff --git a/jobs/ingestor_cloudwatch/templates/bin/ingestor_cloudwatch b/jobs/ingestor_cloudwatch/templates/bin/ingestor_cloudwatch index 3a1e3c83..a4205480 100644 --- a/jobs/ingestor_cloudwatch/templates/bin/ingestor_cloudwatch +++ b/jobs/ingestor_cloudwatch/templates/bin/ingestor_cloudwatch @@ -46,8 +46,6 @@ export LOGSTASH_WORKERS=`grep -c ^processor /proc/cpuinfo` <% else %> export LOGSTASH_WORKERS=<%= p('logstash_parser.workers') %> <% end %> -export TIMECOP_REJECT_GREATER_THAN_HOURS=<%= p('logstash_parser.timecop.reject_greater_than_hours') %> -export TIMECOP_REJECT_LESS_THAN_HOURS=<%= p('logstash_parser.timecop.reject_less_than_hours') %> export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * <%= p("logstash.heap_percentage") %> ) / 100 ))K <% if_p('logstash.heap_size') do |heap_size| %> HEAP_SIZE=<%= heap_size %> @@ -56,12 +54,6 @@ HEAP_SIZE=<%= heap_size %> export <%= env.keys[0] %>="<%= env.values[0] %>" <% end %> - -# These are what changes between ingestors -<% p("logstash_parser.wait_for_templates").each do |template| %> -wait_for_template "<%= template %>" -<% end %> - export LS_JAVA_OPTS="-Xms$HEAP_SIZE -Xmx$HEAP_SIZE -DPID=$$" # construct a complete config file from all the fragments From 2a76660698780e2482d2f6f778c23a1fce22a44a Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 20 Dec 2024 15:38:33 -0500 Subject: [PATCH 9/9] more cleanup --- .../src/logstash-filters/snippets/syslog_standard.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf index 124cfc01..529d7e65 100644 --- a/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf +++ b/src/base-logstash-filters/src/logstash-filters/snippets/syslog_standard.conf @@ -76,10 +76,8 @@ if !("fail/syslog_standard/_grokparsefailure" in [tags]) { tag_on_timeout => [ "fail/syslog_standard/_grokparsetimeout-syslog_standard-5424/sds" ] } - if !("fail/syslog_standard/_grokparsefailure-syslog_standard-5424/sd" in [tags]) { # convert the the key-value pairs - kv { source => "syslog_sd_params_raw" target => "syslog_sd_params"