cyclone-logging/logstash.conf at master · cyclone-project/cyclone-logging · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
input {
    tcp {
        mode => "server"
        port => 9400
        codec => json_lines
        add_field => { "connectionType" => "tcp" }
    }
    log4j {
        mode => "server"
        port => 9500
        add_field => { "connectionType" => "log4j" }
    }
    tcp {
        mode => "server"
        port => 9600
        codec => json
        add_field => { "connectionType" => "tcp2" }
    }
    syslog {
        port => 9514
        add_field => { "connectionType" => "syslog" }
    }
}

filter {
    # check for error, try to fix it
    if "_jsonparsefailure" in [tags] {
        mutate { gsub => [ "message", "[\n\t]", " "] }
        json { source => "message" }
    }

    if [connectionType] == "tcp" {
        mutate {
            # rename timestamp field for better differentiation
            rename => [ "timestamp", "timestamp-remote" ]
            # undo any possible escaping
            gsub => [
                "message", "\n", " ",
                "message", "&amp;", "&",
                "message", "&lt;", "<",
                "message", "&gt;", ">",
                "message", "&quot;", "\"",
                "message", "&apos;", "'",
                "message", "&#x2F;", "/"
            ]
        }
        # parse remote unix timestamp into human readable format
        date {
            match => [ "timestamp-remote", "UNIX_MS" ]
            target => "timestamp-remote"
        }
    }

    mutate { remove_field => [ "connectionType", "tags" ] }
}

output {
    elasticsearch {
        host => "elasticsearch"
        protocol => "transport"
        index => "%{client-id}-%{+YYYY.MM.dd}"
    }
}