[HOT FIX] v0.1.7-alpha02262019 - Logstash Pipeline

helk-logstash
+ Added offline plugins file
+ Updated win security conversion
+ cleaned process-name filter & process-name-split configs
+ cleaned process-id filter & proces-id conversion configs
+ set kafka max poll records to 500
+ updated SOURCE_ & TARGET_ field names from process entity to be renamed process_source_ and process_target. Following the basic `entity_context_property` from OSSEM CIM

Author: Cyb3rWard0g

docker/helk-logstash/Dockerfile

@@ -11,4 +11,5 @@ LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
 LABEL description="Dockerfile base for the HELK Logstash."
 
 RUN mv /usr/share/logstash/config/logstash.yml /usr/share/logstash/config/logstash.yml.backup
-COPY --chown=logstash:logstash config/logstash.yml /usr/share/logstash/config/logstash.yml
+COPY --chown=logstash:logstash config/logstash.yml /usr/share/logstash/config/logstash.yml
+COPY --chown=logstash:logstash plugins/logstash-offline-plugins-6.6.1.zip /usr/share/logstash/logstash-offline-plugins-6.6.1.zip

docker/helk-logstash/pipeline/0002-kafka-input.conf

@@ -20,6 +20,6 @@ input {
     session_timeout_ms => "30000"
     max_poll_interval_ms => "300000"
     #############################
-    max_poll_records => "250"
+    max_poll_records => "500"
   }
 }

docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf

@@ -0,0 +1,44 @@
+# HELK process-name-filter filter conf
+# HELK build Stage: Alpha
+# Author: Roberto Rodriguez (@Cyb3rWard0g)
+# License: GPL-3.0
+
+filter {
+    if [event_id] {
+        if [Image] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_1" } }
+            mutate { rename => { "Image" => "process_path" } }
+        }
+        if [Application] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_2" } }
+            mutate { rename => { "Application" => "process_path" } }
+        }
+        if [NewProcessName] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_3" } }
+            mutate { rename => { "NewProcessName" => "process_path" } }
+        }
+        if [ProcessName] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_4" } }
+            mutate { rename => { "ProcessName" => "process_path" }}
+        }
+        if [ParentProcessName] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_5" } }
+            mutate { rename => { "ParentProcessName" => "process_parent_path" } }
+        }
+        if [ParentImage] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_6" } }
+            mutate { rename => { "ParentImage" => "process_parent_path" } }
+        }
+        if [TargetImage] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_7" } }
+            mutate { rename => { "TargetImage" => "process_target_path" } }
+        }
+        if [SourceImage] {
+            mutate { add_field => { "z_logstash_pipeline" => "1523_8" } }
+            mutate { rename => { "SourceImage" => "process_path" } }
+        }
+        if [ProdessName] {
+            mutate { rename => { "ProdessName" => "process_path" } }
+        }
+    }
+}

docker/helk-logstash/pipeline/1523-winevent-process-name-split.conf

@@ -0,0 +1,53 @@
+# HELK process-ids filter conf
+# HELK build Stage: Alpha
+# Author: Roberto Rodriguez (@Cyb3rWard0g)
+# License: GPL-3.0
+
+filter {
+    if [event_id] {
+        if [ProcessId] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_2" } }
+            mutate { rename => { "ProcessId" => "process_id" } }
+        }
+        if [NewProcessId] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_3" } }
+            mutate { rename => { "NewProcessId" => "process_id" } }
+        }
+        if [ParentProcessId] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_5" } }
+            mutate { rename => { "ParentProcessId" => "process_parent_id" } }
+        }
+        if [ProcessGuid] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_6" } }
+            mutate { rename => { "ProcessGuid" => "process_guid" } }
+        }
+        if [ParentProcessGuid] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_7" } }
+            mutate { rename => { "ParentProcessGuid" => "process_parent_guid" } }
+        }
+        if [SourceProcessGuid] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_8" } }
+            mutate { rename => { "SourceProcessGuid" => "process_guid" } }   
+        }
+        if [SourceProcessGUID] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_9" } }
+            mutate { rename => { "SourceProcessGUID" => "process_guid" } }   
+        }
+        if [SourceProcessId] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_11" } }
+            mutate { rename => { "SourceProcessId" => "process_id" } }   
+        }
+        if [TargetProcessGuid] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_12" } }
+            mutate { rename => { "TargetProcessGuid" => "process_target_guid" } }   
+        }
+        if [TargetProcessGUID] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_13" } }
+            mutate { rename => { "TargetProcessGUID" => "process_target_guid" } }   
+        }
+        if [TargetProcessId] {
+            mutate { add_field => { "z_logstash_pipeline" => "1524_15" } }
+            mutate { rename => { "TargetProcessId" => "process_target_id" } }   
+        }
+    }
+}

docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf

@@ -20,19 +20,10 @@ filter {
         source => "RuleName"
         field_split => ","
         value_split => "="
-        prefix => "mitre_"
+        prefix => "rule_"
         transform_key => "lowercase"
       }
     }
-    if [Image] {
-      mutate {
-        rename => {
-          "Image" => "process_path"
-          "ProcessGuid" => "process_guid"
-          "ProcessId" => "process_id"
-        }
-      }
-    }
     if [Hashes] {
       kv {
         source => "Hashes"
@@ -42,18 +33,6 @@ filter {
         transform_key => "lowercase"
       }
     }
-    if [TargetImage] {
-      mutate {
-        rename => {
-          "SourceImage" => "process_path"
-          "SourceProcessGUID" => "process_guid"
-          "SourceProcessId" => "process_id"
-          "TargetImage" => "target_process_path"
-          "TargetProcessGUID" => "target_process_guid"
-          "TargetProcessId" => "target_process_id"
-        }
-      }
-    }
     if [User] {
       grok {
         match => { "User" => "%{GREEDYDATA:user_domain}\\%{GREEDYDATA:user_name}" }
@@ -66,13 +45,10 @@ filter {
         rename => {
           "CommandLine" => "process_command_line"
           "CurrentDirectory" => "process_current_directory"
-          "ParentImage" => "process_parent_path"
           "ParentCommandLine" => "process_parent_command_line"
           "IntegrityLevel" => "process_integrity_level"
           "LogonGuid" => "user_logon_guid"
           "LogonId" => "user_logon_id"
-          "ParentProcessGuid" => "process_parent_guid"
-          "ParentProcessId" => "process_parent_id"
           "TerminalSessionId" => "user_session_id"
           "FileVersion" => "file_version"
           "Description" => "file_description"
@@ -238,7 +214,7 @@ filter {
     date {
         timezone => "UTC"
         match => [ "CreationUtcTime", "YYYY-MM-dd HH:mm:ss.SSS" ]
-        target => "@date_creation"
+        target => "@file_date_creation"
         tag_on_failure => [ "_sysmon_CreationUtcTime_datefailure", "_sysmon_datefailure", "_dateparsefailure" ]
     }
     date {