From ec03a559e5a556e4bea8f1799aa7763cf072da85 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 17:18:03 +0700 Subject: [PATCH 01/26] Refactoring flow & structures --- internal/runner/metrics.go | 60 ++++++++++++++++++++++++++++++++++++ internal/runner/runner.go | 39 +++++------------------ internal/runner/utils.go | 8 +++++ internal/runner/validator.go | 22 ------------- pkg/metrics/metrics.go | 8 ++--- pkg/metrics/send.go | 1 + pkg/parsers/config.go | 50 ++++++------------------------ pkg/parsers/get.go | 28 +++++++++++++++++ pkg/parsers/logs.go | 9 ++++++ pkg/parsers/metrics.go | 8 +++++ pkg/parsers/notifications.go | 12 ++++++++ pkg/parsers/threat.go | 6 ++++ pkg/parsers/yaml.go | 10 ------ pkg/teler/teler.go | 20 ------------ pkg/teler/utils.go | 20 ++++++++++++ 15 files changed, 173 insertions(+), 128 deletions(-) create mode 100644 internal/runner/metrics.go create mode 100644 internal/runner/utils.go create mode 100644 pkg/parsers/get.go create mode 100644 pkg/parsers/logs.go create mode 100644 pkg/parsers/metrics.go create mode 100644 pkg/parsers/notifications.go create mode 100644 pkg/parsers/threat.go delete mode 100644 pkg/parsers/yaml.go diff --git a/internal/runner/metrics.go b/internal/runner/metrics.go new file mode 100644 index 00000000..e6adee28 --- /dev/null +++ b/internal/runner/metrics.go @@ -0,0 +1,60 @@ +package runner + +import ( + "fmt" + "net/http" + "reflect" + "strconv" + + "github.com/projectdiscovery/gologger" + "github.com/prometheus/client_golang/prometheus/promhttp" + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/errors" + "ktbs.dev/teler/pkg/metrics" +) + +func metric(options *common.Options) { + m := options.Configs.Metrics + v := reflect.ValueOf(m) + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + switch t.Field(i).Name { + case "Prometheus": + if v.Field(i).FieldByName("Active").Bool() { + startPrometheus(options) + } + } + } +} + +func startPrometheus(options *common.Options) { + p := options.Configs.Metrics.Prometheus + + if p.Host == "" { + p.Host = "127.0.0.1" + } + + if p.Port == 0 { + p.Port = 9090 + } + + if p.Endpoint == "" { + p.Endpoint = "/metrics" + } + + s := fmt.Sprint(p.Host, ":", strconv.Itoa(p.Port)) + e := p.Endpoint + + go func() { + http.Handle(e, promhttp.Handler()) + + err := http.ListenAndServe(s, nil) + if err != nil { + errors.Exit(err.Error()) + } + }() + + metrics.Prometheus() + gologger.Info().Msgf(fmt.Sprint("Listening metrics on http://", s, e)) +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 24bc5739..1416c695 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -4,15 +4,12 @@ import ( "encoding/json" "fmt" "io" - "net/http" "os" "os/signal" - "regexp" "github.com/acarl005/stripansi" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/remeh/sizedwaitgroup" "github.com/satyrius/gonx" "ktbs.dev/teler/common" @@ -22,31 +19,15 @@ import ( "ktbs.dev/teler/pkg/teler" ) -func removeLBR(s string) string { - re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`) - return re.ReplaceAllString(s, ``) -} - // New read & pass stdin log func New(options *common.Options) { - var input *os.File - var out string - var pass int - - metric, promserve, promendpoint := prometheus(options) - if metric { - go func() { - http.Handle(promendpoint, promhttp.Handler()) - - err := http.ListenAndServe(promserve, nil) - if err != nil { - errors.Exit(err.Error()) - } - }() - - metrics.Init() - gologger.Info().Msgf("Listening metrics on http://" + promserve + promendpoint) - } + var ( + input *os.File + out string + pass int + ) + + go metric(options) jobs := make(chan *gonx.Entry) gologger.Info().Msg("Analyzing...") @@ -70,12 +51,7 @@ func New(options *common.Options) { defer swg.Done() threat, obj := teler.Analyze(options, line) - if threat { - if metric { - metrics.GetThreatTotal.WithLabelValues(obj["category"]).Inc() - } - if options.JSON { json, err := json.Marshal(obj) if err != nil { @@ -104,6 +80,7 @@ func New(options *common.Options) { } alert.New(options, common.Version, obj) + metrics.Send(obj) } }(log) } diff --git a/internal/runner/utils.go b/internal/runner/utils.go new file mode 100644 index 00000000..52af18f7 --- /dev/null +++ b/internal/runner/utils.go @@ -0,0 +1,8 @@ +package runner + +import "regexp" + +func removeLBR(s string) string { + re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`) + return re.ReplaceAllString(s, ``) +} diff --git a/internal/runner/validator.go b/internal/runner/validator.go index 9c2f5ca0..0cae8b68 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -3,7 +3,6 @@ package runner import ( "os" "reflect" - "strconv" "strings" "gopkg.in/validator.v2" @@ -55,27 +54,6 @@ func validate(options *common.Options) { } } -func prometheus(options *common.Options) (bool, string, string) { - config := options.Configs - if config.Prometheus.Active { - if config.Prometheus.Host == "" { - config.Prometheus.Host = "127.0.0.1" - } - - if config.Prometheus.Port == 0 { - config.Prometheus.Port = 9090 - } - - if config.Prometheus.Endpoint == "" { - config.Prometheus.Endpoint = "/metrics" - } - } - - server := config.Prometheus.Host + ":" + strconv.Itoa(config.Prometheus.Port) - - return config.Prometheus.Active, server, config.Prometheus.Endpoint -} - func notification(options *common.Options) { config := options.Configs diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index f54005dc..761b7c11 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -52,7 +52,7 @@ var ( []string{"http_referer"}, ) - GetThreatTotal = prometheus.NewCounterVec( + getThreatTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "teler_threats_count_total", Help: "Total number of detected threats", @@ -61,10 +61,10 @@ var ( ) ) -// Init will register a Prometheus metrics with the specified variables -func Init() { +// Prometheus will register a metrics with the specified variables +func Prometheus() { prometheus.MustRegister( getBadCrawler, getDirBruteforce, getBadIP, - getCWA, getCVE, getBadReferrer, GetThreatTotal, + getCWA, getCVE, getBadReferrer, getThreatTotal, ) } diff --git a/pkg/metrics/send.go b/pkg/metrics/send.go index ac9ba036..b161612b 100644 --- a/pkg/metrics/send.go +++ b/pkg/metrics/send.go @@ -48,4 +48,5 @@ func Send(log map[string]string) { } counter.Inc() + getThreatTotal.WithLabelValues(log["category"]).Inc() } diff --git a/pkg/parsers/config.go b/pkg/parsers/config.go index ef264761..ab797006 100644 --- a/pkg/parsers/config.go +++ b/pkg/parsers/config.go @@ -1,24 +1,6 @@ package parsers -import "io/ioutil" - -type options struct { - Excludes []string `yaml:"excludes"` - Whitelists []string `yaml:"whitelists"` -} - -type general struct { - Token string `yaml:"token"` - Color string `yaml:"color"` - Channel string `yaml:"channel"` -} - -type telegram struct { - Token string `yaml:"token"` - ChatID string `yaml:"chat_id"` -} - -// Configs default structure for config +// Configs default structure for configurations type Configs struct { Logformat string `yaml:"log_format" validate:"nonzero"` @@ -27,12 +9,13 @@ type Configs struct { Threat options `yaml:"threat" validate:"nonzero"` } `yaml:"rules" validate:"nonzero"` - Prometheus struct { - Active bool `yaml:"active"` - Host string `yaml:"host"` - Port int `yaml:"port"` - Endpoint string `yaml:"endpoint"` - } `yaml:"prometheus" validate:"nonzero"` + Metrics struct { + Prometheus prometheus `yaml:"prometheus"` + } `yaml:"metrics" validate:"nonzero"` + + Logs struct { + Zinc zinc `yaml:"zinc"` + } `yaml:"logs" validate:"nonzero"` Alert struct { Active bool `yaml:"active"` @@ -40,23 +23,8 @@ type Configs struct { } `yaml:"alert" validate:"nonzero"` Notifications struct { - Slack general `yaml:"slack"` + Slack general `yaml:"slack"` Telegram telegram `yaml:"telegram"` Discord general `yaml:"discord"` } `yaml:"notifications"` } - -// GetConfig will parse the config file -func GetConfig(f string) (*Configs, error) { - config := &Configs{} - file, err := ioutil.ReadFile(f) - if err != nil { - return nil, err - } - err = GetYaml(file, config) - if err != nil { - return nil, err - } - - return config, nil -} diff --git a/pkg/parsers/get.go b/pkg/parsers/get.go new file mode 100644 index 00000000..b63d884a --- /dev/null +++ b/pkg/parsers/get.go @@ -0,0 +1,28 @@ +package parsers + +import ( + "io/ioutil" + + "gopkg.in/yaml.v2" +) + +// GetConfig will parse the config file +func GetConfig(f string) (*Configs, error) { + config := &Configs{} + file, err := ioutil.ReadFile(f) + if err != nil { + return nil, err + } + err = GetYaml(file, config) + if err != nil { + return nil, err + } + + return config, nil +} + +// GetYaml file configuration +func GetYaml(f []byte, s interface{}) error { + y := yaml.Unmarshal(f, s) + return y +} diff --git a/pkg/parsers/logs.go b/pkg/parsers/logs.go new file mode 100644 index 00000000..441563c0 --- /dev/null +++ b/pkg/parsers/logs.go @@ -0,0 +1,9 @@ +package parsers + +type zinc struct { + Active bool `yaml:"active"` + SSL bool `yaml:"ssl"` + User string `yaml:"user"` + Pass string `yaml:"pass"` + Index string `yaml:"index"` +} diff --git a/pkg/parsers/metrics.go b/pkg/parsers/metrics.go new file mode 100644 index 00000000..b6a3f10a --- /dev/null +++ b/pkg/parsers/metrics.go @@ -0,0 +1,8 @@ +package parsers + +type prometheus struct { + Active bool `yaml:"active"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Endpoint string `yaml:"endpoint"` +} diff --git a/pkg/parsers/notifications.go b/pkg/parsers/notifications.go new file mode 100644 index 00000000..23547493 --- /dev/null +++ b/pkg/parsers/notifications.go @@ -0,0 +1,12 @@ +package parsers + +type general struct { + Token string `yaml:"token"` + Color string `yaml:"color"` + Channel string `yaml:"channel"` +} + +type telegram struct { + Token string `yaml:"token"` + ChatID string `yaml:"chat_id"` +} diff --git a/pkg/parsers/threat.go b/pkg/parsers/threat.go new file mode 100644 index 00000000..9f44adff --- /dev/null +++ b/pkg/parsers/threat.go @@ -0,0 +1,6 @@ +package parsers + +type options struct { + Excludes []string `yaml:"excludes"` + Whitelists []string `yaml:"whitelists"` +} diff --git a/pkg/parsers/yaml.go b/pkg/parsers/yaml.go deleted file mode 100644 index ac5437f4..00000000 --- a/pkg/parsers/yaml.go +++ /dev/null @@ -1,10 +0,0 @@ -package parsers - -import ( - "gopkg.in/yaml.v2" -) - -func GetYaml(f []byte, s interface{}) error { - y := yaml.Unmarshal(f, s) - return y -} diff --git a/pkg/teler/teler.go b/pkg/teler/teler.go index 779d22b0..a3125da0 100644 --- a/pkg/teler/teler.go +++ b/pkg/teler/teler.go @@ -7,13 +7,11 @@ import ( "reflect" "regexp" "strings" - "unicode/utf8" "github.com/satyrius/gonx" "github.com/valyala/fastjson" "ktbs.dev/teler/common" "ktbs.dev/teler/pkg/matchers" - "ktbs.dev/teler/pkg/metrics" ) // Analyze logs from threat resources @@ -223,27 +221,9 @@ func Analyze(options *common.Options, logs *gonx.Entry) (bool, map[string]string } if match { - metrics.Send(log) return match, log } } return match, log } - -func trimFirst(s string) string { - _, i := utf8.DecodeRuneInString(s) - return s[i:] -} - -func isWhitelist(options *common.Options, find string) bool { - whitelist := options.Configs.Rules.Threat.Whitelists - for i := 0; i < len(whitelist); i++ { - match := matchers.IsMatch(whitelist[i], find) - if match { - return true - } - } - - return false -} diff --git a/pkg/teler/utils.go b/pkg/teler/utils.go index a35b61ce..264029f8 100644 --- a/pkg/teler/utils.go +++ b/pkg/teler/utils.go @@ -2,7 +2,10 @@ package teler import ( "reflect" + "unicode/utf8" + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/matchers" "ktbs.dev/teler/resource" ) @@ -22,3 +25,20 @@ func getDatasets() { datasets[cat]["content"] = con } } + +func trimFirst(s string) string { + _, i := utf8.DecodeRuneInString(s) + return s[i:] +} + +func isWhitelist(options *common.Options, find string) bool { + whitelist := options.Configs.Rules.Threat.Whitelists + for i := 0; i < len(whitelist); i++ { + match := matchers.IsMatch(whitelist[i], find) + if match { + return true + } + } + + return false +} From 90bfcb73be21e54586847c20b43110f181c73f4c Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 17:51:48 +0700 Subject: [PATCH 02/26] Refactoring file output logs --- common/options.go | 3 +-- internal/runner/options.go | 5 ----- internal/runner/runner.go | 34 ++++++++-------------------------- internal/runner/validator.go | 20 ++++++++++++-------- pkg/errors/messages.go | 1 + pkg/logs/file.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/parsers/config.go | 1 + pkg/parsers/logs.go | 6 ++++++ 8 files changed, 65 insertions(+), 41 deletions(-) create mode 100644 pkg/logs/file.go diff --git a/common/options.go b/common/options.go index 4aa980dc..440b1686 100644 --- a/common/options.go +++ b/common/options.go @@ -13,8 +13,7 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process Version bool // Version check of teler flag Input string // Parse log from data persistence rather than buffer stream - Output string // Save detected threats to file - OutFile *os.File // Write log output into file + Output *os.File // Write log output into file Configs *parsers.Configs // Get teler configuration interface JSON bool // Display threats in the terminal as JSON format RmCache bool // To remove all cached resources on local diff --git a/internal/runner/options.go b/internal/runner/options.go index d87232fb..f9d5e211 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,9 +25,6 @@ func ParseOptions() *common.Options { flag.IntVar(&options.Concurrency, "x", 20, "") flag.IntVar(&options.Concurrency, "concurrent", 20, "") - flag.StringVar(&options.Output, "o", "", "") - flag.StringVar(&options.Output, "output", "", "") - flag.BoolVar(&options.Version, "v", false, "") flag.BoolVar(&options.Version, "version", false, "") @@ -47,8 +44,6 @@ func ParseOptions() *common.Options { " -c, --config teler configuration file", " -i, --input Analyze logs from data persistence rather than buffer stream", " -x, --concurrent Set the concurrency level to analyze logs (default: 20)", - " -o, --output Save detected threats to file", - " --json Display threats in the terminal as JSON format", " --rm-cache Removes all cached resources", " -v, --version Show current teler version", "", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1416c695..e4299afa 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -1,13 +1,11 @@ package runner import ( - "encoding/json" "fmt" "io" "os" "os/signal" - "github.com/acarl005/stripansi" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/remeh/sizedwaitgroup" @@ -15,6 +13,7 @@ import ( "ktbs.dev/teler/common" "ktbs.dev/teler/internal/alert" "ktbs.dev/teler/pkg/errors" + "ktbs.dev/teler/pkg/logs" "ktbs.dev/teler/pkg/metrics" "ktbs.dev/teler/pkg/teler" ) @@ -52,34 +51,17 @@ func New(options *common.Options) { threat, obj := teler.Analyze(options, line) if threat { - if options.JSON { - json, err := json.Marshal(obj) - if err != nil { - errors.Exit(err.Error()) - } - out = fmt.Sprintf("%s\n", json) - } else { - out = fmt.Sprintf("[%s] [%s] [%s] %s\n", - aurora.Cyan(obj["time_local"]), - aurora.Green(obj["remote_addr"]), - aurora.Yellow(obj["category"]), - aurora.Red(obj[obj["element"]]), - ) - } + out = fmt.Sprintf("[%s] [%s] [%s] %s\n", + aurora.Cyan(obj["time_local"]), + aurora.Green(obj["remote_addr"]), + aurora.Yellow(obj["category"]), + aurora.Red(obj[obj["element"]]), + ) fmt.Print(out) - if options.Output != "" { - if !options.JSON { - out = stripansi.Strip(out) - } - - if _, write := options.OutFile.WriteString(out); write != nil { - errors.Show(write.Error()) - } - } - alert.New(options, common.Version, obj) + logs.File(options, obj) metrics.Send(obj) } }(log) diff --git a/internal/runner/validator.go b/internal/runner/validator.go index 0cae8b68..134e3079 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -28,18 +28,22 @@ func validate(options *common.Options) { } } - if options.Output != "" { - f, errOutput := os.OpenFile(options.Output, + config, errConfig := parsers.GetConfig(options.ConfigFile) + if errConfig != nil { + errors.Exit(errConfig.Error()) + } + + if config.Logs.File.Active { + if config.Logs.File.Path == "" { + errors.Exit(errors.ErrNoFilePath) + } + + f, errOutput := os.OpenFile(config.Logs.File.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if errOutput != nil { errors.Exit(errOutput.Error()) } - options.OutFile = f - } - - config, errConfig := parsers.GetConfig(options.ConfigFile) - if errConfig != nil { - errors.Exit(errConfig.Error()) + options.Output = f } // Validates log format diff --git a/pkg/errors/messages.go b/pkg/errors/messages.go index f3ac7a10..f98b089d 100644 --- a/pkg/errors/messages.go +++ b/pkg/errors/messages.go @@ -6,4 +6,5 @@ const ( ErrAlertProvider = "Provider \":platform\" not available; " + ErrCheckConfig ErrNoInputLog = "No input logs provided" ErrNoInputConfig = "No config file specified" + ErrNoFilePath = "No file path specified" ) diff --git a/pkg/logs/file.go b/pkg/logs/file.go new file mode 100644 index 00000000..1abe1b8d --- /dev/null +++ b/pkg/logs/file.go @@ -0,0 +1,36 @@ +package logs + +import ( + "encoding/json" + "fmt" + + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/errors" +) + +// File write detected threats into it +func File(options *common.Options, log map[string]string) { + var out string + file := options.Configs.Logs.File + + if options.Output != nil { + if file.JSON { + logJSON, err := json.Marshal(log) + if err != nil { + errors.Exit(err.Error()) + } + out = fmt.Sprintf("%s\n", logJSON) + } else { + out = fmt.Sprintf("[%s] [%s] [%s] %s\n", + log["time_local"], + log["remote_addr"], + log["category"], + log[log["element"]], + ) + } + + if _, write := options.Output.WriteString(out); write != nil { + errors.Show(write.Error()) + } + } +} diff --git a/pkg/parsers/config.go b/pkg/parsers/config.go index ab797006..116e7d5b 100644 --- a/pkg/parsers/config.go +++ b/pkg/parsers/config.go @@ -14,6 +14,7 @@ type Configs struct { } `yaml:"metrics" validate:"nonzero"` Logs struct { + File file `yaml:"file"` Zinc zinc `yaml:"zinc"` } `yaml:"logs" validate:"nonzero"` diff --git a/pkg/parsers/logs.go b/pkg/parsers/logs.go index 441563c0..f06f375f 100644 --- a/pkg/parsers/logs.go +++ b/pkg/parsers/logs.go @@ -7,3 +7,9 @@ type zinc struct { Pass string `yaml:"pass"` Index string `yaml:"index"` } + +type file struct { + Active bool `yaml:"active"` + JSON bool `yaml:"json"` + Path string `yaml:"path"` +} From 2addd620db065ff6799f2b553d204e930902ac24 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 17:54:14 +0700 Subject: [PATCH 03/26] Reverse metrics expression & conditions --- internal/runner/metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/runner/metrics.go b/internal/runner/metrics.go index e6adee28..82071527 100644 --- a/internal/runner/metrics.go +++ b/internal/runner/metrics.go @@ -19,9 +19,9 @@ func metric(options *common.Options) { t := v.Type() for i := 0; i < v.NumField(); i++ { - switch t.Field(i).Name { - case "Prometheus": - if v.Field(i).FieldByName("Active").Bool() { + if v.Field(i).FieldByName("Active").Bool() { + switch t.Field(i).Name { + case "Prometheus": startPrometheus(options) } } From a371740dfc5cdc7ce54c02d339189d6ee7f75491 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 18:29:16 +0700 Subject: [PATCH 04/26] Create internal logs handler --- internal/runner/logs.go | 33 +++++++++++++++++++++++++++++++++ internal/runner/runner.go | 7 +++---- pkg/logs/file.go | 19 ++++++++++--------- 3 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 internal/runner/logs.go diff --git a/internal/runner/logs.go b/internal/runner/logs.go new file mode 100644 index 00000000..cd5360a7 --- /dev/null +++ b/internal/runner/logs.go @@ -0,0 +1,33 @@ +package runner + +import ( + "reflect" + + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/errors" + "ktbs.dev/teler/pkg/logs" +) + +func log(options *common.Options, data map[string]string) { + m := options.Configs.Logs + v := reflect.ValueOf(m) + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + if v.Field(i).FieldByName("Active").Bool() { + switch t.Field(i).Name { + case "File": + toFile(options, data) + // case "Zinc": // TODO + // toZinc(options, data) + } + } + } +} + +func toFile(options *common.Options, data map[string]string) { + err := logs.File(options, data) + if err != nil { + errors.Show(err.Error()) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e4299afa..035f6cfc 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -13,7 +13,6 @@ import ( "ktbs.dev/teler/common" "ktbs.dev/teler/internal/alert" "ktbs.dev/teler/pkg/errors" - "ktbs.dev/teler/pkg/logs" "ktbs.dev/teler/pkg/metrics" "ktbs.dev/teler/pkg/teler" ) @@ -44,7 +43,7 @@ func New(options *common.Options) { con := options.Concurrency swg := sizedwaitgroup.New(con) go func() { - for log := range jobs { + for job := range jobs { swg.Add() go func(line *gonx.Entry) { defer swg.Done() @@ -61,10 +60,10 @@ func New(options *common.Options) { fmt.Print(out) alert.New(options, common.Version, obj) - logs.File(options, obj) + log(options, obj) metrics.Send(obj) } - }(log) + }(job) } }() diff --git a/pkg/logs/file.go b/pkg/logs/file.go index 1abe1b8d..00dcb218 100644 --- a/pkg/logs/file.go +++ b/pkg/logs/file.go @@ -5,32 +5,33 @@ import ( "fmt" "ktbs.dev/teler/common" - "ktbs.dev/teler/pkg/errors" ) // File write detected threats into it -func File(options *common.Options, log map[string]string) { +func File(options *common.Options, data map[string]string) error { var out string file := options.Configs.Logs.File if options.Output != nil { if file.JSON { - logJSON, err := json.Marshal(log) + logJSON, err := json.Marshal(data) if err != nil { - errors.Exit(err.Error()) + return err } out = fmt.Sprintf("%s\n", logJSON) } else { out = fmt.Sprintf("[%s] [%s] [%s] %s\n", - log["time_local"], - log["remote_addr"], - log["category"], - log[log["element"]], + data["time_local"], + data["remote_addr"], + data["category"], + data[data["element"]], ) } if _, write := options.Output.WriteString(out); write != nil { - errors.Show(write.Error()) + return write } } + + return nil } From 9dc7b19d1ab035d3b224c55e0cddb02634eaf9a4 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 18:33:42 +0700 Subject: [PATCH 05/26] misc: File names --- pkg/errors/{messages.go => constants.go} | 0 pkg/metrics/prometheus.go | 52 ++++++++++++++++++++++++ pkg/metrics/send.go | 52 ------------------------ 3 files changed, 52 insertions(+), 52 deletions(-) rename pkg/errors/{messages.go => constants.go} (100%) create mode 100644 pkg/metrics/prometheus.go delete mode 100644 pkg/metrics/send.go diff --git a/pkg/errors/messages.go b/pkg/errors/constants.go similarity index 100% rename from pkg/errors/messages.go rename to pkg/errors/constants.go diff --git a/pkg/metrics/prometheus.go b/pkg/metrics/prometheus.go new file mode 100644 index 00000000..ccdf491f --- /dev/null +++ b/pkg/metrics/prometheus.go @@ -0,0 +1,52 @@ +package metrics + +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +// Prometheus metric logs insertion +func Prometheus(data map[string]string) { + var counter prometheus.Counter + + switch { + case strings.HasPrefix(data["category"], "Common Web Attack"): + counter = getCWA.WithLabelValues( + data["category"], + data["remote_addr"], + data["request_uri"], + data["status"], + ) + case strings.HasPrefix(data["category"], "CVE-"): + counter = getCVE.WithLabelValues( + data["category"], + data["remote_addr"], + data["request_uri"], + data["status"], + ) + case data["category"] == "Bad Crawler": + counter = getBadCrawler.WithLabelValues( + data["remote_addr"], + data["http_user_agent"], + data["status"], + ) + case data["category"] == "Bad IP Address": + counter = getBadIP.WithLabelValues( + data["remote_addr"], + ) + case data["category"] == "Bad Referrer": + counter = getBadReferrer.WithLabelValues( + data["http_referer"], + ) + case data["category"] == "Directory Bruteforce": + counter = getDirBruteforce.WithLabelValues( + data["remote_addr"], + data["request_uri"], + data["status"], + ) + } + + counter.Inc() + getThreatTotal.WithLabelValues(data["category"]).Inc() +} diff --git a/pkg/metrics/send.go b/pkg/metrics/send.go deleted file mode 100644 index b161612b..00000000 --- a/pkg/metrics/send.go +++ /dev/null @@ -1,52 +0,0 @@ -package metrics - -import ( - "strings" - - "github.com/prometheus/client_golang/prometheus" -) - -// Send logs to metrics -func Send(log map[string]string) { - var counter prometheus.Counter - - switch { - case strings.HasPrefix(log["category"], "Common Web Attack"): - counter = getCWA.WithLabelValues( - log["category"], - log["remote_addr"], - log["request_uri"], - log["status"], - ) - case strings.HasPrefix(log["category"], "CVE-"): - counter = getCVE.WithLabelValues( - log["category"], - log["remote_addr"], - log["request_uri"], - log["status"], - ) - case log["category"] == "Bad Crawler": - counter = getBadCrawler.WithLabelValues( - log["remote_addr"], - log["http_user_agent"], - log["status"], - ) - case log["category"] == "Bad IP Address": - counter = getBadIP.WithLabelValues( - log["remote_addr"], - ) - case log["category"] == "Bad Referrer": - counter = getBadReferrer.WithLabelValues( - log["http_referer"], - ) - case log["category"] == "Directory Bruteforce": - counter = getDirBruteforce.WithLabelValues( - log["remote_addr"], - log["request_uri"], - log["status"], - ) - } - - counter.Inc() - getThreatTotal.WithLabelValues(log["category"]).Inc() -} From 9e018dea7ec0972fa8870b96636f78f573c02716 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 18:40:56 +0700 Subject: [PATCH 06/26] Simplify print-statement --- internal/runner/runner.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 035f6cfc..e409e60e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -21,7 +21,6 @@ import ( func New(options *common.Options) { var ( input *os.File - out string pass int ) @@ -50,15 +49,13 @@ func New(options *common.Options) { threat, obj := teler.Analyze(options, line) if threat { - out = fmt.Sprintf("[%s] [%s] [%s] %s\n", + fmt.Printf("[%s] [%s] [%s] %s\n", aurora.Cyan(obj["time_local"]), aurora.Green(obj["remote_addr"]), aurora.Yellow(obj["category"]), aurora.Red(obj[obj["element"]]), ) - fmt.Print(out) - alert.New(options, common.Version, obj) log(options, obj) metrics.Send(obj) From 8f6d109d73c3207db082e88e45699233d02d4729 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 14 Dec 2021 21:48:56 +0700 Subject: [PATCH 07/26] misc: Rename methods --- internal/runner/runner.go | 2 +- pkg/metrics/prometheus.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e409e60e..b9321244 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -58,7 +58,7 @@ func New(options *common.Options) { alert.New(options, common.Version, obj) log(options, obj) - metrics.Send(obj) + metrics.PrometheusInsert(obj) } }(job) } diff --git a/pkg/metrics/prometheus.go b/pkg/metrics/prometheus.go index ccdf491f..f4ba640b 100644 --- a/pkg/metrics/prometheus.go +++ b/pkg/metrics/prometheus.go @@ -6,8 +6,8 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// Prometheus metric logs insertion -func Prometheus(data map[string]string) { +// PrometheusInsert logs into metrics +func PrometheusInsert(data map[string]string) { var counter prometheus.Counter switch { From 55962aacadfd50f80310dc798a3afd5a0c14f4a0 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 18:55:54 +0700 Subject: [PATCH 08/26] Supports Zinc logs engine --- internal/runner/logs.go | 19 ++++++++- internal/runner/validator.go | 75 ++++++++++++++++++++++++++++++++++++ pkg/errors/constants.go | 8 +++- pkg/logs/zinc.go | 51 ++++++++++++++++++++++++ pkg/parsers/logs.go | 13 ++++--- 5 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 pkg/logs/zinc.go diff --git a/internal/runner/logs.go b/internal/runner/logs.go index cd5360a7..45f84966 100644 --- a/internal/runner/logs.go +++ b/internal/runner/logs.go @@ -1,6 +1,7 @@ package runner import ( + "fmt" "reflect" "ktbs.dev/teler/common" @@ -18,8 +19,8 @@ func log(options *common.Options, data map[string]string) { switch t.Field(i).Name { case "File": toFile(options, data) - // case "Zinc": // TODO - // toZinc(options, data) + case "Zinc": + toZinc(options, data) } } } @@ -31,3 +32,17 @@ func toFile(options *common.Options, data map[string]string) { errors.Show(err.Error()) } } + +func toZinc(options *common.Options, data map[string]string) { + zinc := options.Configs.Logs.Zinc + base := "http" + if zinc.SSL { + base += "s" + } + base += fmt.Sprint("://", zinc.Host, ":", zinc.Port) + + err := logs.Zinc(base, zinc.Index, zinc.Base64Auth, data) + if err != nil { + errors.Show(err.Error()) + } +} diff --git a/internal/runner/validator.go b/internal/runner/validator.go index 134e3079..12c2774f 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -1,6 +1,12 @@ package runner import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" "os" "reflect" "strings" @@ -53,11 +59,80 @@ func validate(options *common.Options) { // Validates notification parts on configuration files notification(options) + // Do Zinc health check, validate & set credentials + if config.Logs.Zinc.Active { + options.Configs.Logs.Zinc.Base64Auth = zinc(options) + } + if errVal := validator.Validate(options); errVal != nil { errors.Exit(errVal.Error()) } } +func zinc(options *common.Options) string { + var health, auth map[string]interface{} + + zinc := options.Configs.Logs.Zinc + base := "http" + if zinc.SSL { + base += "s" + } + base += fmt.Sprint("://", zinc.Host, ":", zinc.Port) + + resp, err := http.Get(fmt.Sprint(base, "/healthz")) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrHealthZinc, ": ", err.Error())) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrHealthZinc, ": ", err.Error())) + } + + if err = json.Unmarshal(body, &health); err != nil { + errors.Exit(fmt.Sprint(errors.ErrHealthZinc, ": ", err.Error())) + } + + if health["status"] != "ok" { + errors.Exit(errors.ErrHealthZinc) + } + + b64auth := base64.StdEncoding.EncodeToString([]byte( + fmt.Sprint(zinc.Username, ":", zinc.Password), + )) + data, _ := json.Marshal(map[string]string{ + "_id": zinc.Username, + "base64encoded": b64auth, + "password": zinc.Password, + }) + + resp, err = http.Post( + fmt.Sprint(base, "/api/login"), + "application/json", + bytes.NewBuffer(data), + ) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrAuthZinc, ": ", err.Error())) + } + defer resp.Body.Close() + + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrAuthZinc, ": ", err.Error())) + } + + if err = json.Unmarshal(body, &auth); err != nil { + errors.Exit(fmt.Sprint(errors.ErrAuthZinc, ": ", err.Error())) + } + + if auth["validated"] == false { + errors.Exit(errors.ErrAuthZinc) + } + + return b64auth +} + func notification(options *common.Options) { config := options.Configs diff --git a/pkg/errors/constants.go b/pkg/errors/constants.go index f98b089d..80662961 100644 --- a/pkg/errors/constants.go +++ b/pkg/errors/constants.go @@ -6,5 +6,11 @@ const ( ErrAlertProvider = "Provider \":platform\" not available; " + ErrCheckConfig ErrNoInputLog = "No input logs provided" ErrNoInputConfig = "No config file specified" - ErrNoFilePath = "No file path specified" + ErrNoFilePath = "No file path specified; " + ErrCheckConfig + ErrNoUserZinc = "No username provided for Zinc log server; " + ErrCheckConfig + ErrNoPassZinc = "No password provided for Zinc log server; " + ErrCheckConfig + ErrNoIndexZinc = "No index provided for Zinc log server; " + ErrCheckConfig + ErrAuthZinc = "Invalid Zinc credentials" + ErrHealthZinc = "Zinc log server is not running" + ErrInsertLogZinc = "Failed to insert logs to Zinc" ) diff --git a/pkg/logs/zinc.go b/pkg/logs/zinc.go new file mode 100644 index 00000000..b86f35a0 --- /dev/null +++ b/pkg/logs/zinc.go @@ -0,0 +1,51 @@ +package logs + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + + e "ktbs.dev/teler/pkg/errors" +) + +// Zinc logs insertion +func Zinc(base string, index string, auth string, log map[string]string) error { + var res map[string]string + client := &http.Client{} + + data, err := json.Marshal(log) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", fmt.Sprint(base, "/api/", index, "/document"), bytes.NewBuffer(data)) + if err != nil { + panic(err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Basic "+auth) + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + if err = json.Unmarshal(body, &res); err != nil { + return err + } + + if res["id"] != "" { + return nil + } + + return errors.New(e.ErrInsertLogZinc) +} diff --git a/pkg/parsers/logs.go b/pkg/parsers/logs.go index f06f375f..f80627d1 100644 --- a/pkg/parsers/logs.go +++ b/pkg/parsers/logs.go @@ -1,11 +1,14 @@ package parsers type zinc struct { - Active bool `yaml:"active"` - SSL bool `yaml:"ssl"` - User string `yaml:"user"` - Pass string `yaml:"pass"` - Index string `yaml:"index"` + Active bool `yaml:"active"` + Host string `yaml:"host"` + Port int `yaml:"port"` + SSL bool `yaml:"ssl"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Index string `yaml:"index"` + Base64Auth string } type file struct { From bafe3c0b90b3b7ec4916c3de5a99e7b482b3ee3a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 19:43:53 +0700 Subject: [PATCH 09/26] Update formatting on error details --- internal/runner/validator.go | 2 +- pkg/errors/constants.go | 1 + pkg/errors/errors.go | 19 +++++++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/runner/validator.go b/internal/runner/validator.go index 12c2774f..643bc01c 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -36,7 +36,7 @@ func validate(options *common.Options) { config, errConfig := parsers.GetConfig(options.ConfigFile) if errConfig != nil { - errors.Exit(errConfig.Error()) + errors.Exit(errors.ErrParseConfig + errConfig.Error()) } if config.Logs.File.Active { diff --git a/pkg/errors/constants.go b/pkg/errors/constants.go index 80662961..12d9ba6c 100644 --- a/pkg/errors/constants.go +++ b/pkg/errors/constants.go @@ -1,6 +1,7 @@ package errors const ( + ErrParseConfig = "Can't parse config file: " ErrCheckConfig = "please check your config file" ErrConfigValidate = "Only validates :key; " + ErrCheckConfig ErrAlertProvider = "Provider \":platform\" not available; " + ErrCheckConfig diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 342237c7..b76fdc1f 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -1,6 +1,7 @@ package errors import ( + "bufio" "strings" "github.com/projectdiscovery/gologger" @@ -8,12 +9,22 @@ import ( // Exit will show error details and stop the program func Exit(err string) { - msg := "Error! " if err != "" { - for _, e := range strings.Split(strings.TrimSuffix(err, "\n"), "\n") { - msg += e + count := 0 + lines := bufio.NewScanner(strings.NewReader(err)) + + for lines.Scan() { + var msg string + + if count == 0 { + msg = "Error! " + } + msg += strings.TrimSpace(lines.Text()) + Show(msg) + count++ } + gologger.Info().Msgf("Use \"-h\" flag for more info about command.") Abort(9) } @@ -21,5 +32,5 @@ func Exit(err string) { // Show error message func Show(msg string) { - gologger.Error().Msgf("%s\n", msg) + gologger.Error().Msg(msg) } From 8f6590ad7c49476b78892583dd7a767d9940afc9 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 19:59:44 +0700 Subject: [PATCH 10/26] Fix assignment to entry in nil map for datasets --- pkg/teler/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/teler/utils.go b/pkg/teler/utils.go index 264029f8..9f3c675d 100644 --- a/pkg/teler/utils.go +++ b/pkg/teler/utils.go @@ -21,7 +21,7 @@ func getDatasets() { continue } - datasets[cat] = map[string]string{} + datasets[cat] = make(map[string]string) datasets[cat]["content"] = con } } From f0a6f2e8516f5d78d5a989c694a324db099c0a79 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 20:29:45 +0700 Subject: [PATCH 11/26] workflow: Ignore branches pattern --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ca33df11..9f511a66 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,6 +15,8 @@ on: - master - development pull_request: + branches-ignore: + - dependabot/** jobs: checks: From b663f281fcf16498c16862f75d04237a6ea91fb3 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 20:30:38 +0700 Subject: [PATCH 12/26] workflow: Add Semgrep action --- .github/workflows/review.yaml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index 96eeb27a..da1bec08 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -1,8 +1,14 @@ name: Review on: + push: + branches: + - master pull_request: branches: - - development + - master + - development + branches-ignore: + - dependabot/** jobs: review: @@ -16,4 +22,15 @@ jobs: uses: kitabisa/sonarqube-action@development with: host: ${{ secrets.SONARQUBE_HOST }} - login: ${{ secrets.SONARQUBE_TOKEN }} \ No newline at end of file + login: ${{ secrets.SONARQUBE_TOKEN }} + + - name: Run Semgrep + uses: returntocorp/semgrep-action@v1 + with: + config: | + p/ci + p/owasp-top-ten + p/golang + p/command-injection + p/security-audit + p/secrets \ No newline at end of file From 8b7d7f85443fe188bfe765dadb30295ca6105876 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 20:31:54 +0700 Subject: [PATCH 13/26] Ignore semgrep rule:go.lang.security.audit.net.use-tls.use-tls --- internal/runner/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runner/metrics.go b/internal/runner/metrics.go index 82071527..091fb8e4 100644 --- a/internal/runner/metrics.go +++ b/internal/runner/metrics.go @@ -49,7 +49,7 @@ func startPrometheus(options *common.Options) { go func() { http.Handle(e, promhttp.Handler()) - err := http.ListenAndServe(s, nil) + err := http.ListenAndServe(s, nil) // nosemgrep: go.lang.security.audit.net.use-tls.use-tls if err != nil { errors.Exit(err.Error()) } From 738335a04df950760a55d5ebca7ddaab4ea1c747 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 20:32:11 +0700 Subject: [PATCH 14/26] Use html/template instead --- internal/alert/telegram.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/alert/telegram.go b/internal/alert/telegram.go index 80e2d91f..9bacdd23 100644 --- a/internal/alert/telegram.go +++ b/internal/alert/telegram.go @@ -2,8 +2,8 @@ package alert import ( "bytes" + "html/template" "strconv" - "text/template" telegramBot "github.com/go-telegram-bot-api/telegram-bot-api" "ktbs.dev/teler/pkg/errors" @@ -31,12 +31,12 @@ func toTelegram(token string, chatID string, log map[string]string) { func telegramMessage(log map[string]string) string { var buffer bytes.Buffer - template, err := template.ParseFiles("internal/alert/template/telegram.tmpl") + tpl, err := template.ParseFiles("internal/alert/template/telegram.tmpl") if err != nil { errors.Exit(err.Error()) } - err = template.Execute(&buffer, log) + err = tpl.Execute(&buffer, log) if err != nil { errors.Exit(err.Error()) } From c3ba0d52ae197ce6cc5c9547cd62f5e2e74373e4 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 21:10:22 +0700 Subject: [PATCH 15/26] Update example configuration file --- teler.example.yaml | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/teler.example.yaml b/teler.example.yaml index bd998b9f..cc25a9df 100644 --- a/teler.example.yaml +++ b/teler.example.yaml @@ -1,4 +1,4 @@ -# To write log format, see https://github.com/kitabisa/teler#configuration +# To write log format, see https://www.notion.so/kitabisa/Configuration-d7c8fab40366406591875bac631bef3f log_format: | $remote_addr - [$remote_addr] $remote_user - [$time_local] "$request_method $request_uri $request_protocol" $status $body_bytes_sent @@ -16,18 +16,35 @@ rules: # - "Bad Crawler" # - "Directory Bruteforce" - # It can be user-agent, request path, HTTP referrer, IP address and/or request query values parsed in regExp + # It can be user-agent, request path, HTTP referrer, + # IP address and/or request query values parsed in regExp whitelists: # - "(curl|Go-http-client|okhttp)/*" # - "^/wp-login\\.php" # - "https://www\\.facebook\\.com" # - "192\\.168\\.0\\.1" -prometheus: - active: false - host: "localhost" - port: 9099 - endpoint: "/metrics" +metrics: + prometheus: + active: false + host: "localhost" + port: 9099 + endpoint: "/metrics" + +logs: + file: + active: false + json: false + path: "/path/to/output.log" + + zinc: + active: false + host: "localhost" + port: 4080 + ssl: false + username: "admin" + password: "Complexpass#123" + index: "lorem-ipsum-index" alert: active: false From d4646f21029ceded6c7cbac8fc1fae1e31980d9a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 21:16:23 +0700 Subject: [PATCH 16/26] Update changes --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b020046..c869554b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project should be documented in this file. +### v2.0.0-beta + +- Add supporting RAW requests from CVEs templates +- Add Zinc logs engine +- Refactoring configuration structures +- Remove `-o/--output` & `--json` flags + ### v1.2.2 - Add utility for get datasets From a4b5d23d0984e8c65744ef2a0b81aebd72c0ac17 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 15 Dec 2021 21:16:31 +0700 Subject: [PATCH 17/26] Add important notes --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index e96f5f7c..c48070d1 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ [![teler](https://user-images.githubusercontent.com/25837540/97096468-f8ccaa00-1696-11eb-8830-0d3a7be45a2d.gif)](#) +> ### :warning: Important notes +> If you upgrade from prior to v2 frontwards there will be some **break changes** that affect the configuration files. +> Appropriate adaptations can refer to [c3ba0d52](https://github.com/kitabisa/teler/commit/c3ba0d52ae197ce6cc5c9547cd62f5e2e74373e4) commit. +> +> See the exact changes in the [CHANGELOG.md](#changes) file. + ## Table of Contents - [Features](#features) - [Why teler?](#why-teler) From 05aaf5ffa31ef68c5272e1a52fff2cb37a2abc98 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 02:30:11 +0700 Subject: [PATCH 18/26] workflow: Remove push event for code review --- .github/workflows/review.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index da1bec08..e868da49 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -1,8 +1,5 @@ name: Review on: - push: - branches: - - master pull_request: branches: - master From 2a03a28f5485156ead7f9b327401c2f7c8b2501b Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 02:21:05 +0700 Subject: [PATCH 19/26] Add custom threat rules --- pkg/parsers/threat.go | 15 ++++++++++-- pkg/teler/teler.go | 55 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/pkg/parsers/threat.go b/pkg/parsers/threat.go index 9f44adff..5a9273cc 100644 --- a/pkg/parsers/threat.go +++ b/pkg/parsers/threat.go @@ -1,6 +1,17 @@ package parsers +type customs struct { + Name string `yaml:"name"` + Condition string `yaml:"condition"` + Rules []struct { + Element string `yaml:"element"` + Pattern string `yaml:"pattern"` + Selector bool `yaml:"selector"` + } `yaml:"rules"` +} + type options struct { - Excludes []string `yaml:"excludes"` - Whitelists []string `yaml:"whitelists"` + Excludes []string `yaml:"excludes"` + Whitelists []string `yaml:"whitelists"` + Customs []customs `yaml:"customs"` } diff --git a/pkg/teler/teler.go b/pkg/teler/teler.go index a3125da0..1556ae7a 100644 --- a/pkg/teler/teler.go +++ b/pkg/teler/teler.go @@ -16,8 +16,12 @@ import ( // Analyze logs from threat resources func Analyze(options *common.Options, logs *gonx.Entry) (bool, map[string]string) { - var match bool + var ( + match bool + selector string + ) + cfg := options.Configs log := make(map[string]string) fields := reflect.ValueOf(logs).Elem().FieldByName("fields") @@ -225,5 +229,54 @@ func Analyze(options *common.Options, logs *gonx.Entry) (bool, map[string]string } } + log["element"] = "" + customs := cfg.Rules.Threat.Customs + + for i := 0; i < len(customs); i++ { + log["category"] = customs[i].Name + + cond := strings.ToLower(customs[i].Condition) + if cond == "" { + cond = "or" + } + + rules := customs[i].Rules + rulesCount := len(customs[i].Rules) + matchCount := 0 + + if rulesCount < 1 { + continue + } + + for j := 0; j < rulesCount; j++ { + if matchers.IsMatch(rules[j].Pattern, log[rules[j].Element]) { + if rules[j].Selector { + log["element"] = rules[j].Element + } + selector = rules[j].Element + + matchCount++ + if cond == "or" { + break + } + } + } + + if log["element"] == "" { + log["element"] = selector + } + + switch { + case cond == "and" && matchCount == rulesCount: + match = true + case cond == "or" && matchCount > 0: + match = true + } + + if match { + break + } + } + return match, log } From d547be5a0896db5fc8213005933a3aeb55456982 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 02:22:14 +0700 Subject: [PATCH 20/26] Fix undefined counter for category from custom rules --- pkg/metrics/prometheus.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/metrics/prometheus.go b/pkg/metrics/prometheus.go index f4ba640b..a407bec3 100644 --- a/pkg/metrics/prometheus.go +++ b/pkg/metrics/prometheus.go @@ -45,6 +45,8 @@ func PrometheusInsert(data map[string]string) { data["request_uri"], data["status"], ) + default: + return } counter.Inc() From 58f3d677366dc59cd798d687ce753857bcdc7ce1 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 02:35:05 +0700 Subject: [PATCH 21/26] Add examples for custom threat rules --- teler.example.yaml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/teler.example.yaml b/teler.example.yaml index cc25a9df..5e2d1a5a 100644 --- a/teler.example.yaml +++ b/teler.example.yaml @@ -1,9 +1,7 @@ # To write log format, see https://www.notion.so/kitabisa/Configuration-d7c8fab40366406591875bac631bef3f log_format: | - $remote_addr - [$remote_addr] $remote_user - [$time_local] - "$request_method $request_uri $request_protocol" $status $body_bytes_sent - "$http_referer" "$http_user_agent" $request_length $request_time - [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id + $remote_addr $remote_user - [$time_local] "$request_method $request_uri $request_protocol" + $status $body_bytes_sent "$http_referer" "$http_user_agent" rules: cache: true @@ -17,13 +15,26 @@ rules: # - "Directory Bruteforce" # It can be user-agent, request path, HTTP referrer, - # IP address and/or request query values parsed in regExp + # IP address and/or request query values parsed in regExp. + # This list applies only to engine defined threats, not to custom threat rules. whitelists: # - "(curl|Go-http-client|okhttp)/*" # - "^/wp-login\\.php" # - "https://www\\.facebook\\.com" # - "192\\.168\\.0\\.1" + customs: + # - name: Large File Upload + # condition: and # and/or + # rules: + # - element: body_bytes_sent + # selector: true + # pattern: \d{6,} + + # - element: request_method + # pattern: P(OST|UT) + + metrics: prometheus: active: false From 25e91a8e9ba04adf6f3e2438de0df2a56bc5a943 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 03:34:59 +0700 Subject: [PATCH 22/26] workflow: Ignore by job expression instead --- .github/workflows/review.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index e868da49..d972a72d 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -1,16 +1,18 @@ name: Review on: + push: + branches: + - master pull_request: branches: - master - development - branches-ignore: - - dependabot/** jobs: review: name: Code Review - runs-on: ubuntu-latest + runs-on: ubuntu-latest + if: startsWith(github.head_ref, 'dependabot/') == false steps: - name: Check out code uses: actions/checkout@v2.3.4 From 6c28dd1435ec1925029de44b47c6553919c624ac Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 04:37:27 +0700 Subject: [PATCH 23/26] docs: Update notes & example configs --- README.md | 4 ++-- teler.example.yaml | 28 ++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c48070d1..13e19a52 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ [![teler](https://user-images.githubusercontent.com/25837540/97096468-f8ccaa00-1696-11eb-8830-0d3a7be45a2d.gif)](#) > ### :warning: Important notes -> If you upgrade from prior to v2 frontwards there will be some **break changes** that affect the configuration files. -> Appropriate adaptations can refer to [c3ba0d52](https://github.com/kitabisa/teler/commit/c3ba0d52ae197ce6cc5c9547cd62f5e2e74373e4) commit. +> If you upgrade from prior to v2 frontwards there will be some **break changes** that affect configuration files. +> Appropriate adaptations can refer to [teler.example.yaml](https://github.com/kitabisa/teler/blob/master/teler.example.yaml) file. > > See the exact changes in the [CHANGELOG.md](#changes) file. diff --git a/teler.example.yaml b/teler.example.yaml index 5e2d1a5a..3b7f157d 100644 --- a/teler.example.yaml +++ b/teler.example.yaml @@ -18,20 +18,32 @@ rules: # IP address and/or request query values parsed in regExp. # This list applies only to engine defined threats, not to custom threat rules. whitelists: - # - "(curl|Go-http-client|okhttp)/*" - # - "^/wp-login\\.php" - # - "https://www\\.facebook\\.com" - # - "192\\.168\\.0\\.1" + # - (curl|Go-http-client|okhttp)/* + # - ^/wp-login\.php + # - https?:\/\/www\.facebook\.com + # - 192\.168\.0\.1 customs: - # - name: Large File Upload - # condition: and # and/or + # - name: "Log4j Attack" + # condition: or # rules: - # - element: body_bytes_sent + # - element: "request_uri" + # pattern: \$\{.*:\/\/.*\/?\w+?\} + + # - element: "http_referer" + # pattern: \$\{.*:\/\/.*\/?\w+?\} + + # - element: "http_user_agent" + # pattern: \$\{.*:\/\/.*\/?\w+?\} + + # - name: "Large File Upload" + # condition: and + # rules: + # - element: "body_bytes_sent" # selector: true # pattern: \d{6,} - # - element: request_method + # - element: "request_method" # pattern: P(OST|UT) From 56b3643c31b710a690be6593f7370d6a1c3bfbb8 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 04:52:58 +0700 Subject: [PATCH 24/26] docs: Update changes & readme file [ci skip] --- CHANGELOG.md | 1 + README.md | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c869554b..697fe624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project should be documented in this file. - Add Zinc logs engine - Refactoring configuration structures - Remove `-o/--output` & `--json` flags +- Add custom threat rules ### v1.2.2 diff --git a/README.md b/README.md index 13e19a52..f53854ab 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ * **Monitoring**: We've our own metrics if you want to monitor threats easily, and we use Prometheus for that. +* **Logging**: is also provided in file form or sends detected threats to the Zinc logs search engine. + * **Latest resources**: Collections is continuously up-to-date. * **Minimal configuration**: You can just run it against your log file, write the log format and let @@ -64,6 +66,8 @@ * **Flexible log formats**: teler allows any custom log format string! It all depends on how you write the log format in configuration file. +* **Custom threat rules**: Want to reach a wider range of threats instead of engine-based _(default)_ rules? You can customize threat rules! + * **Incremental log processing**: Need data persistence rather than [buffer stream](https://linux.die.net/man/1/stdbuf)? teler has the ability to process logs incrementally through the on-disk persistence options. @@ -134,7 +138,7 @@ All external resources used in this teler are **NOT** provided by us. See all pe ## Pronunciation -/télér/ bagaimana bisa seorang pemuda itu teler hanya dengan meminum 1 sloki ciu _(?)_ +[`jv_id`](https://www.localeplanet.com/java/jv-ID/index.html) • **/télér/** — bagaimana bisa seorang pemuda itu teler hanya dengan meminum sloki ciu _(?)_ ## Changes From 9184a440eb34b71b852b7d5ee2d6ecd363062f0c Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 18:29:39 +0700 Subject: [PATCH 25/26] Add validation for custom threat rules --- internal/runner/validator.go | 38 +++++++++++++++++++++++++++++++++++- pkg/errors/constants.go | 21 +++++++++++--------- pkg/matchers/config.go | 18 +++++++++++++++++ 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/internal/runner/validator.go b/internal/runner/validator.go index 643bc01c..fe925dba 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -56,7 +56,8 @@ func validate(options *common.Options) { matchers.IsLogformat(config.Logformat) options.Configs = config - // Validates notification parts on configuration files + // Validates custom threat rules & notification parts configuration + customs(options) notification(options) // Do Zinc health check, validate & set credentials @@ -69,6 +70,41 @@ func validate(options *common.Options) { } } +func customs(options *common.Options) { + var err string + + cfg := options.Configs + cat := make(map[string]bool) + + custom := cfg.Rules.Threat.Customs + for i := 0; i < len(custom); i++ { + cond := strings.ToLower(custom[i].Condition) + matchers.IsCondition(cond) + matchers.IsBlank(custom[i].Name, "Custom threat category") + + if cat[custom[i].Name] { + err = strings.Replace(errors.ErrDupeCategory, ":category", custom[i].Name, -1) + errors.Exit(err) + } + cat[custom[i].Name] = true + + rules := custom[i].Rules + for j := 0; j < len(rules); j++ { + matchers.IsBlank(rules[j].Element, "Custom threat rules element") + elm := fmt.Sprint("$", rules[j].Element) + + if !matchers.IsMatch(fmt.Sprint(`\`, elm), cfg.Logformat) { + err = strings.Replace(errors.ErrNoElement, ":element", elm, -1) + err = strings.Replace(err, ":category", custom[i].Name, -1) + + errors.Exit(err) + } + + matchers.IsBlank(rules[j].Pattern, "Custom threat rules pattern") + } + } +} + func zinc(options *common.Options) string { var health, auth map[string]interface{} diff --git a/pkg/errors/constants.go b/pkg/errors/constants.go index 12d9ba6c..1e017a19 100644 --- a/pkg/errors/constants.go +++ b/pkg/errors/constants.go @@ -1,17 +1,20 @@ package errors const ( - ErrParseConfig = "Can't parse config file: " - ErrCheckConfig = "please check your config file" - ErrConfigValidate = "Only validates :key; " + ErrCheckConfig ErrAlertProvider = "Provider \":platform\" not available; " + ErrCheckConfig - ErrNoInputLog = "No input logs provided" - ErrNoInputConfig = "No config file specified" - ErrNoFilePath = "No file path specified; " + ErrCheckConfig - ErrNoUserZinc = "No username provided for Zinc log server; " + ErrCheckConfig - ErrNoPassZinc = "No password provided for Zinc log server; " + ErrCheckConfig - ErrNoIndexZinc = "No index provided for Zinc log server; " + ErrCheckConfig ErrAuthZinc = "Invalid Zinc credentials" + ErrBlankField = ":field can't be blank; " + ErrCheckConfig + ErrCheckConfig = "please check your config file" + ErrConfigValidate = "Only validates :key; " + ErrCheckConfig + ErrDupeCategory = "Duplicated name for ':category' threat category; " + ErrCheckConfig ErrHealthZinc = "Zinc log server is not running" ErrInsertLogZinc = "Failed to insert logs to Zinc" + ErrNoElement = "Can't find ':element' on log format for ':category' threat category; " + ErrCheckConfig + ErrNoFilePath = "No file path specified; " + ErrCheckConfig + ErrNoIndexZinc = "No index provided for Zinc log server; " + ErrCheckConfig + ErrNoInputConfig = "No config file specified" + ErrNoInputLog = "No input logs provided" + ErrNoPassZinc = "No password provided for Zinc log server; " + ErrCheckConfig + ErrNoUserZinc = "No username provided for Zinc log server; " + ErrCheckConfig + ErrParseConfig = "Can't parse config file: " ) diff --git a/pkg/matchers/config.go b/pkg/matchers/config.go index ffecab2d..b9647a11 100644 --- a/pkg/matchers/config.go +++ b/pkg/matchers/config.go @@ -46,3 +46,21 @@ func IsChatID(s string) { errValidate("chat_id") } } + +// IsCondition validates custom threat rules condition +func IsCondition(s string) { + switch s { + case "or", "and": + default: + errValidate("AND/OR for condition") + } +} + +// IsBlank validates nil field value +func IsBlank(s string, field string) { + s = strings.TrimSpace(s) + if s == "" { + err := strings.Replace(errors.ErrBlankField, ":field", field, -1) + errors.Exit(err) + } +} From 46209b58c82a1a90908de7f31972a9df89be364f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Dec 2021 21:16:25 +0700 Subject: [PATCH 26/26] Add handler for no rules threat --- internal/runner/validator.go | 5 +++++ pkg/errors/constants.go | 1 + 2 files changed, 6 insertions(+) diff --git a/internal/runner/validator.go b/internal/runner/validator.go index fe925dba..aba3e3b2 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -89,6 +89,11 @@ func customs(options *common.Options) { cat[custom[i].Name] = true rules := custom[i].Rules + if len(rules) < 1 { + err = strings.Replace(errors.ErrNoThreatRules, ":category", custom[i].Name, -1) + errors.Exit(err) + } + for j := 0; j < len(rules); j++ { matchers.IsBlank(rules[j].Element, "Custom threat rules element") elm := fmt.Sprint("$", rules[j].Element) diff --git a/pkg/errors/constants.go b/pkg/errors/constants.go index 1e017a19..fb80fcf1 100644 --- a/pkg/errors/constants.go +++ b/pkg/errors/constants.go @@ -15,6 +15,7 @@ const ( ErrNoInputConfig = "No config file specified" ErrNoInputLog = "No input logs provided" ErrNoPassZinc = "No password provided for Zinc log server; " + ErrCheckConfig + ErrNoThreatRules = "No rules for ':category' threat category; " + ErrCheckConfig ErrNoUserZinc = "No username provided for Zinc log server; " + ErrCheckConfig ErrParseConfig = "Can't parse config file: " )