From 2c558ab70b708dbca840156f6313bbe8251dae30 Mon Sep 17 00:00:00 2001 From: Sebastian <82781994+sebinbash@users.noreply.github.com> Date: Mon, 15 May 2023 11:54:05 +0200 Subject: [PATCH] new segment: modify/aslookup (#70) * Add aslookup segment * Clarify documentation * Add config sanity checking, make db default type, add tests * Add example config * Add example lookup file, use correct file path --------- Co-authored-by: Sebastian Schnorbus --- CONFIGURATION.md | 21 +++++ examples/enricher/config.yml | 8 ++ examples/enricher/lookup.db | Bin 0 -> 435 bytes go.mod | 4 + go.sum | 9 ++ main.go | 1 + segments/modify/aslookup/aslookup.go | 110 ++++++++++++++++++++++ segments/modify/aslookup/aslookup_test.go | 31 ++++++ 8 files changed, 184 insertions(+) create mode 100644 examples/enricher/lookup.db create mode 100644 segments/modify/aslookup/aslookup.go create mode 100644 segments/modify/aslookup/aslookup_test.go diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 180fe6e..6c36715 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -510,6 +510,27 @@ Roadmap: [godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/addcid) [examples using this segment](https://github.com/search?q=%22segment%3A+addcid%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code) +#### aslookup +The `aslookup` segment can add AS numbers to flows using route collector dumps. +Dumps can be obtained from your RIR in the `.mrt` format and can be converted to +lookup databases using the `asnlookup-util` from the `asnlookup` package. These +databases contain a mapping from IP ranges to AS number in binary format. + +By default the type is set to `db`. It is possible to directly parse `.mrt` files, +however this is not recommended since this will significantly slow down lookup times. + +```yaml +- segment: aslookup + config: + filename: ./lookup.db + # the lines below are optional and set to default + type: db # can be either db or mrt +``` +[MRT specification](https://datatracker.ietf.org/doc/html/rfc6396) +[asnlookup](https://github.com/banviktor/asnlookup) +[godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/aslookup) +[examples using this segment](https://github.com/search?q=%22segment%3A+aslookup%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code) + #### bgp The `bgp` segment can add a information from BGP to flows. By default, this information is retrieved from a session with the router specified by a flow's diff --git a/examples/enricher/config.yml b/examples/enricher/config.yml index 75029a7..4b0e876 100644 --- a/examples/enricher/config.yml +++ b/examples/enricher/config.yml @@ -37,6 +37,14 @@ filename: customer_subnets.csv dropunmatched: 1 # limit to flows belonging to customers +############################################################################### +# Looks up AS numbers from a local database which can be generated from +# an mrt route dump. This database can be generated using asnlookup. +- segment: aslookup + config: + filename: lookup.db + type: db + ############################################################################### # Add a geolocation field to all flows. As this works on remote addresses (the # assumption being that your own addresses are national), we need to run after diff --git a/examples/enricher/lookup.db b/examples/enricher/lookup.db new file mode 100644 index 0000000000000000000000000000000000000000..f976fc2e465db1c65334ef35da0b004690eeb40d GIT binary patch literal 435 zcmYe#EXgQM(o4?I)lW*yE6dC-$uH7REY8cx&(AI`&@aeN*H6mKODw7^DauS`a4afH ztPBCM7zG(1fJuQ7&SHVG@Ka0*M5^N;P94Oy?|<>z%D^B1Rlo(M|NQ4?;DxfafHWTx zn;FUmQLIR89w-~6mmNu*8%dl4Nt_c&oC!%B*=PJvagZBeZUnJm;R=#w0})UF00 github.com/TheFireMike/go-mrt v0.0.0-20220205210421-b3040c1c0b7e diff --git a/go.sum b/go.sum index 2c054fe..c2fdd9a 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/Shopify/sarama v1.37.2/go.mod h1:Nxye/E+YPru//Bpaorfhc3JsSGYwCaDDj+R4 github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= +github.com/TheFireMike/go-mrt v0.0.0-20220205210421-b3040c1c0b7e h1:C/wsiPbYFVzVp1NEWf2z0qa+/L3adlcoE95mZMQlolg= +github.com/TheFireMike/go-mrt v0.0.0-20220205210421-b3040c1c0b7e/go.mod h1:b11Of1G6DQ01qWlTnAVDxSycYpzU6Bxz43DGIjqE+VM= github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe h1:SKdmPMOww/faIbffys2UgnZHlQJETCw7N18AaYUYf2M= github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe/go.mod h1:tGK+sH41V0mnyFBVWQoRyj7neHPwQwPM1KJ3PfS6dTI= github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg= @@ -80,6 +82,8 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/asecurityteam/rolling v2.0.4+incompatible h1:WOSeokINZT0IDzYGc5BVcjLlR9vPol08RvI2GAsmB0s= github.com/asecurityteam/rolling v2.0.4+incompatible/go.mod h1:2D4ba5ZfYCWrIMleUgTvc8pmLExEuvu3PDwl+vnG58Q= +github.com/banviktor/asnlookup v0.1.0 h1:rErAtlKFoqkkYxySad3BNBmdOksfbMFTdQCp7WaY1Ok= +github.com/banviktor/asnlookup v0.1.0/go.mod h1:BtQkoiOTh7jgtnk4QpjEpowoVY9O4I4qGw5NKCcDpZE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -110,6 +114,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -443,10 +448,12 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -490,6 +497,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= @@ -895,6 +903,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index bdf81ad..1a00b16 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,7 @@ import ( _ "github.com/bwNetFlow/flowpipeline/segments/modify/addcid" _ "github.com/bwNetFlow/flowpipeline/segments/modify/anonymize" + _ "github.com/bwNetFlow/flowpipeline/segments/modify/aslookup" _ "github.com/bwNetFlow/flowpipeline/segments/modify/bgp" _ "github.com/bwNetFlow/flowpipeline/segments/modify/dropfields" _ "github.com/bwNetFlow/flowpipeline/segments/modify/geolocation" diff --git a/segments/modify/aslookup/aslookup.go b/segments/modify/aslookup/aslookup.go new file mode 100644 index 0000000..a3ddb37 --- /dev/null +++ b/segments/modify/aslookup/aslookup.go @@ -0,0 +1,110 @@ +package aslookup + +import ( + "os" + "log" + "net" + "sync" + + "github.com/banviktor/asnlookup/pkg/database" + "github.com/bwNetFlow/flowpipeline/segments" +) + +type AsLookup struct { + segments.BaseSegment + FileName string + Type string + + asDatabase database.Database +} + +func (segment AsLookup) New(config map[string]string) segments.Segment { + + newSegment := &AsLookup{} + + // parse options + if config["filename"] == "" { + log.Println("[error] AsLookup: This segment requires a 'filename' parameter.") + return nil + } + newSegment.FileName = config["filename"] + + if config["type"] == "db" { + newSegment.Type = "db" + } else if config["type"] == "mrt" { + newSegment.Type = "mrt" + } else { + log.Println("[info] AsLookup: 'type' set to default 'db'.") + newSegment.Type = "db" + } + + // open lookup file + lookupfile, err := os.OpenFile(config["filename"], os.O_RDONLY, 0) + if err != nil { + log.Printf("[error] AsLookup: Error opening lookup file: %s", err) + return nil + } + defer lookupfile.Close() + + // lookup file can either be an MRT file or a lookup database generated with asnlookup + // see: https://github.com/banviktor/asnlookup + if newSegment.Type == "db" { + // open lookup db + db, err := database.NewFromDump(lookupfile) + if err != nil { + log.Printf("[error] AsLookup: Error parsing database file: %s", err) + } + newSegment.asDatabase = db + } else { + // parse with asnlookup + builder := database.NewBuilder() + if err = builder.ImportMRT(lookupfile); err != nil { + log.Printf("[error] AsLookup: Error parsing MRT file: %s", err) + } + + // build lookup database + db, err := builder.Build() + if err != nil { + log.Printf("[error] AsLookup: Error building lookup database: %s", err) + } + newSegment.asDatabase = db + } + + + return newSegment +} + +func (segment *AsLookup) Run(wg *sync.WaitGroup) { + defer func() { + close(segment.Out) + wg.Done() + }() + for msg := range segment.In { + // Look up destination AS + dstIp := net.ParseIP(msg.DstAddrObj().String()) + dstAs, err := segment.asDatabase.Lookup(dstIp) + if err != nil { + log.Printf("[warning] AsLookup: Failed to look up ASN for %s: %s", msg.DstAddrObj().String(), err) + segment.Out <- msg + continue + } + msg.DstAS = dstAs.Number + + // Look up source AS + srcIp := net.ParseIP(msg.SrcAddrObj().String()) + srcAs, err := segment.asDatabase.Lookup(srcIp) + if err != nil { + log.Printf("[warning] AsLookup: Failed to look up ASN for %s: %s", msg.SrcAddrObj().String(), err) + segment.Out <- msg + continue + } + msg.SrcAS = srcAs.Number + + segment.Out <- msg + } +} + +func init() { + segment := &AsLookup{} + segments.RegisterSegment("aslookup", segment) +} diff --git a/segments/modify/aslookup/aslookup_test.go b/segments/modify/aslookup/aslookup_test.go new file mode 100644 index 0000000..80b3f57 --- /dev/null +++ b/segments/modify/aslookup/aslookup_test.go @@ -0,0 +1,31 @@ +package aslookup + +import ( + "testing" + + "github.com/bwNetFlow/flowpipeline/pb" + "github.com/bwNetFlow/flowpipeline/segments" +) + +// TODO: write tests for this +func TestSegment_AsLookup_existingIp(t *testing.T) { + result := segments.TestSegment("aslookup", map[string]string{"filename": "../../../examples/enricher/lookup.db", "type": "db"}, + &pb.EnrichedFlow{SrcAddr: []byte{192, 168, 1, 10}, DstAddr: []byte{192, 168, 1, 10}}) + if result.SrcAS != 65015 { + t.Error("Segment AsLookup is not setting the source AS when the corresponding IP exists in the lookup database") + } + if result.DstAS != 65015 { + t.Error("Segment AsLookup is not setting the destination AS when the corresponding IP exists in the lookup database") + } +} + +func TestSegment_AsLookup_nonexistingIp(t *testing.T) { + result := segments.TestSegment("aslookup", map[string]string{"filename": "../../../examples/enricher/lookup.db", "type": "db"}, + &pb.EnrichedFlow{SrcAddr: []byte{2, 125, 160, 218}, DstAddr: []byte{2, 125, 160, 218}}) + if result.SrcAS != 0 { + t.Error("Segment AsLookup is setting the source AS when the corresponding IP does not exist in the lookup database.") + } + if result.DstAS != 0 { + t.Error("Segment AsLookup is setting the destination AS when the corresponding IP does not exist in the lookup database.") + } +}