diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 180fe6e..6c36715 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -510,6 +510,27 @@ Roadmap: [godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/addcid) [examples using this segment](https://github.com/search?q=%22segment%3A+addcid%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code) +#### aslookup +The `aslookup` segment can add AS numbers to flows using route collector dumps. +Dumps can be obtained from your RIR in the `.mrt` format and can be converted to +lookup databases using the `asnlookup-util` from the `asnlookup` package. These +databases contain a mapping from IP ranges to AS number in binary format. + +By default the type is set to `db`. It is possible to directly parse `.mrt` files, +however this is not recommended since this will significantly slow down lookup times. + +```yaml +- segment: aslookup + config: + filename: ./lookup.db + # the lines below are optional and set to default + type: db # can be either db or mrt +``` +[MRT specification](https://datatracker.ietf.org/doc/html/rfc6396) +[asnlookup](https://github.com/banviktor/asnlookup) +[godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/aslookup) +[examples using this segment](https://github.com/search?q=%22segment%3A+aslookup%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code) + #### bgp The `bgp` segment can add a information from BGP to flows. By default, this information is retrieved from a session with the router specified by a flow's diff --git a/examples/enricher/config.yml b/examples/enricher/config.yml index 75029a7..4b0e876 100644 --- a/examples/enricher/config.yml +++ b/examples/enricher/config.yml @@ -37,6 +37,14 @@ filename: customer_subnets.csv dropunmatched: 1 # limit to flows belonging to customers +############################################################################### +# Looks up AS numbers from a local database which can be generated from +# an mrt route dump. This database can be generated using asnlookup. +- segment: aslookup + config: + filename: lookup.db + type: db + ############################################################################### # Add a geolocation field to all flows. As this works on remote addresses (the # assumption being that your own addresses are national), we need to run after diff --git a/examples/enricher/lookup.db b/examples/enricher/lookup.db new file mode 100644 index 0000000..f976fc2 Binary files /dev/null and b/examples/enricher/lookup.db differ diff --git a/go.mod b/go.mod index 2e9b8b7..36c0085 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab github.com/asecurityteam/rolling v2.0.4+incompatible + github.com/banviktor/asnlookup v0.1.0 github.com/bwNetFlow/bpf_flowexport v0.0.0-20220515112212-cd8128615c05 github.com/bwNetFlow/flowfilter v0.0.0-20221025122858-60746fa15915 github.com/bwNetFlow/ip_prefix_trie v0.0.0-20210830112018-b360b7b65c04 @@ -61,6 +62,7 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/k-sone/critbitgo v1.4.0 // indirect + github.com/kaorimatz/go-mrt v0.0.0-20210326003454-aa11f3646f93 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/libp2p/go-reuseport v0.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -101,3 +103,5 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/kaorimatz/go-mrt => github.com/TheFireMike/go-mrt v0.0.0-20220205210421-b3040c1c0b7e diff --git a/go.sum b/go.sum index 2c054fe..c2fdd9a 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/Shopify/sarama v1.37.2/go.mod h1:Nxye/E+YPru//Bpaorfhc3JsSGYwCaDDj+R4 github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= +github.com/TheFireMike/go-mrt v0.0.0-20220205210421-b3040c1c0b7e h1:C/wsiPbYFVzVp1NEWf2z0qa+/L3adlcoE95mZMQlolg= +github.com/TheFireMike/go-mrt v0.0.0-20220205210421-b3040c1c0b7e/go.mod h1:b11Of1G6DQ01qWlTnAVDxSycYpzU6Bxz43DGIjqE+VM= github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe h1:SKdmPMOww/faIbffys2UgnZHlQJETCw7N18AaYUYf2M= github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe/go.mod h1:tGK+sH41V0mnyFBVWQoRyj7neHPwQwPM1KJ3PfS6dTI= github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg= @@ -80,6 +82,8 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/asecurityteam/rolling v2.0.4+incompatible h1:WOSeokINZT0IDzYGc5BVcjLlR9vPol08RvI2GAsmB0s= github.com/asecurityteam/rolling v2.0.4+incompatible/go.mod h1:2D4ba5ZfYCWrIMleUgTvc8pmLExEuvu3PDwl+vnG58Q= +github.com/banviktor/asnlookup v0.1.0 h1:rErAtlKFoqkkYxySad3BNBmdOksfbMFTdQCp7WaY1Ok= +github.com/banviktor/asnlookup v0.1.0/go.mod h1:BtQkoiOTh7jgtnk4QpjEpowoVY9O4I4qGw5NKCcDpZE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -110,6 +114,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -443,10 +448,12 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -490,6 +497,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= @@ -895,6 +903,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index bdf81ad..1a00b16 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,7 @@ import ( _ "github.com/bwNetFlow/flowpipeline/segments/modify/addcid" _ "github.com/bwNetFlow/flowpipeline/segments/modify/anonymize" + _ "github.com/bwNetFlow/flowpipeline/segments/modify/aslookup" _ "github.com/bwNetFlow/flowpipeline/segments/modify/bgp" _ "github.com/bwNetFlow/flowpipeline/segments/modify/dropfields" _ "github.com/bwNetFlow/flowpipeline/segments/modify/geolocation" diff --git a/segments/modify/aslookup/aslookup.go b/segments/modify/aslookup/aslookup.go new file mode 100644 index 0000000..a3ddb37 --- /dev/null +++ b/segments/modify/aslookup/aslookup.go @@ -0,0 +1,110 @@ +package aslookup + +import ( + "os" + "log" + "net" + "sync" + + "github.com/banviktor/asnlookup/pkg/database" + "github.com/bwNetFlow/flowpipeline/segments" +) + +type AsLookup struct { + segments.BaseSegment + FileName string + Type string + + asDatabase database.Database +} + +func (segment AsLookup) New(config map[string]string) segments.Segment { + + newSegment := &AsLookup{} + + // parse options + if config["filename"] == "" { + log.Println("[error] AsLookup: This segment requires a 'filename' parameter.") + return nil + } + newSegment.FileName = config["filename"] + + if config["type"] == "db" { + newSegment.Type = "db" + } else if config["type"] == "mrt" { + newSegment.Type = "mrt" + } else { + log.Println("[info] AsLookup: 'type' set to default 'db'.") + newSegment.Type = "db" + } + + // open lookup file + lookupfile, err := os.OpenFile(config["filename"], os.O_RDONLY, 0) + if err != nil { + log.Printf("[error] AsLookup: Error opening lookup file: %s", err) + return nil + } + defer lookupfile.Close() + + // lookup file can either be an MRT file or a lookup database generated with asnlookup + // see: https://github.com/banviktor/asnlookup + if newSegment.Type == "db" { + // open lookup db + db, err := database.NewFromDump(lookupfile) + if err != nil { + log.Printf("[error] AsLookup: Error parsing database file: %s", err) + } + newSegment.asDatabase = db + } else { + // parse with asnlookup + builder := database.NewBuilder() + if err = builder.ImportMRT(lookupfile); err != nil { + log.Printf("[error] AsLookup: Error parsing MRT file: %s", err) + } + + // build lookup database + db, err := builder.Build() + if err != nil { + log.Printf("[error] AsLookup: Error building lookup database: %s", err) + } + newSegment.asDatabase = db + } + + + return newSegment +} + +func (segment *AsLookup) Run(wg *sync.WaitGroup) { + defer func() { + close(segment.Out) + wg.Done() + }() + for msg := range segment.In { + // Look up destination AS + dstIp := net.ParseIP(msg.DstAddrObj().String()) + dstAs, err := segment.asDatabase.Lookup(dstIp) + if err != nil { + log.Printf("[warning] AsLookup: Failed to look up ASN for %s: %s", msg.DstAddrObj().String(), err) + segment.Out <- msg + continue + } + msg.DstAS = dstAs.Number + + // Look up source AS + srcIp := net.ParseIP(msg.SrcAddrObj().String()) + srcAs, err := segment.asDatabase.Lookup(srcIp) + if err != nil { + log.Printf("[warning] AsLookup: Failed to look up ASN for %s: %s", msg.SrcAddrObj().String(), err) + segment.Out <- msg + continue + } + msg.SrcAS = srcAs.Number + + segment.Out <- msg + } +} + +func init() { + segment := &AsLookup{} + segments.RegisterSegment("aslookup", segment) +} diff --git a/segments/modify/aslookup/aslookup_test.go b/segments/modify/aslookup/aslookup_test.go new file mode 100644 index 0000000..80b3f57 --- /dev/null +++ b/segments/modify/aslookup/aslookup_test.go @@ -0,0 +1,31 @@ +package aslookup + +import ( + "testing" + + "github.com/bwNetFlow/flowpipeline/pb" + "github.com/bwNetFlow/flowpipeline/segments" +) + +// TODO: write tests for this +func TestSegment_AsLookup_existingIp(t *testing.T) { + result := segments.TestSegment("aslookup", map[string]string{"filename": "../../../examples/enricher/lookup.db", "type": "db"}, + &pb.EnrichedFlow{SrcAddr: []byte{192, 168, 1, 10}, DstAddr: []byte{192, 168, 1, 10}}) + if result.SrcAS != 65015 { + t.Error("Segment AsLookup is not setting the source AS when the corresponding IP exists in the lookup database") + } + if result.DstAS != 65015 { + t.Error("Segment AsLookup is not setting the destination AS when the corresponding IP exists in the lookup database") + } +} + +func TestSegment_AsLookup_nonexistingIp(t *testing.T) { + result := segments.TestSegment("aslookup", map[string]string{"filename": "../../../examples/enricher/lookup.db", "type": "db"}, + &pb.EnrichedFlow{SrcAddr: []byte{2, 125, 160, 218}, DstAddr: []byte{2, 125, 160, 218}}) + if result.SrcAS != 0 { + t.Error("Segment AsLookup is setting the source AS when the corresponding IP does not exist in the lookup database.") + } + if result.DstAS != 0 { + t.Error("Segment AsLookup is setting the destination AS when the corresponding IP does not exist in the lookup database.") + } +}