implement reversedns segment (#54)

* revdns: initial working version, no cache, no threads, no nothing

* add working reverse dns impl, but this needs to be non-blocking for cache population

* finish reversedns segment and document it

CONFIGURATION.md

@@ -638,6 +638,23 @@ Any optional parameters relate to the `cidr` policy only and behave as in the
 [godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/remoteaddress)
 [examples using this segment](https://github.com/search?q=%22segment%3A+remoteaddress%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code)
 
+#### reversedns
+The `reversedns` segment looks up DNS PTR records for Src, Dst, Sampler and NextHopAddr and adds
+them to our flows. The results are also written to a internal cache which works well for ad-hoc
+usage, but it's recommended to use an actual caching resolver in real deployment scenarios. The
+refresh interval setting pertains to the internal cache only.
+
+```
+- segment: reversedns
+  config:
+    # the lines below are optional and set to default
+    cache: true
+    refreshinterval: 5m
+```
+
+[godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/reversedns)
+[examples using this segment](https://github.com/search?q=%22segment%3A+reversedns%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code)
+
 #### snmpinterface
 The `snmpinterface` segment annotates flows with interface information learned
 directly from routers using SNMP. This is a potentially perfomance impacting

go.mod

@@ -3,6 +3,8 @@ module github.com/bwNetFlow/flowpipeline
 go 1.18
 
 require (
+	github.com/BelWue/bgp_routeinfo v0.0.0-20221004100427-d8095fc566dd
+	github.com/ClickHouse/clickhouse-go/v2 v2.6.1
 	github.com/Shopify/sarama v1.37.2
 	github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe
 	github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab
@@ -11,23 +13,22 @@ require (
 	github.com/bwNetFlow/flowfilter v0.0.0-20221025122858-60746fa15915
 	github.com/bwNetFlow/ip_prefix_trie v0.0.0-20210830112018-b360b7b65c04
 	github.com/bwNetFlow/protobuf/go v0.0.0-20211004083441-61e193b4b342
-	github.com/google/gopacket v1.1.19
 	github.com/dustin/go-humanize v1.0.1
+	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/logutils v1.0.0
 	github.com/influxdata/influxdb-client-go/v2 v2.11.0
 	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/netsampler/goflow2 v1.1.1
 	github.com/oschwald/maxminddb-golang v1.10.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/prometheus/client_golang v1.13.0
+	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	github.com/BelWue/bgp_routeinfo v0.0.0-20221004100427-d8095fc566dd // indirect
 	github.com/ClickHouse/ch-go v0.51.2 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.6.1 // indirect
 	github.com/alecthomas/participle/v2 v2.0.0-beta.1 // indirect
 	github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -88,6 +89,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.11.2 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect
 	golang.org/x/sys v0.1.0 // indirect
 	golang.org/x/text v0.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect

main.go

@@ -45,6 +45,7 @@ import (
 	_ "github.com/bwNetFlow/flowpipeline/segments/modify/normalize"
 	_ "github.com/bwNetFlow/flowpipeline/segments/modify/protomap"
 	_ "github.com/bwNetFlow/flowpipeline/segments/modify/remoteaddress"
+	_ "github.com/bwNetFlow/flowpipeline/segments/modify/reversedns"
 	_ "github.com/bwNetFlow/flowpipeline/segments/modify/snmp"
 
 	_ "github.com/bwNetFlow/flowpipeline/segments/pass"

pb/flow.go

@@ -268,14 +268,14 @@ func (flow *EnrichedFlow) DstAddrObj() net.IP {
 	return net.IP(flow.DstAddr)
 }
 
-func (flow *EnrichedFlow) SamplerAddressObj() net.IP {
-	return net.IP(flow.SamplerAddress)
-}
-
 func (flow *EnrichedFlow) NextHopObj() net.IP {
 	return net.IP(flow.NextHop)
 }
 
+func (flow *EnrichedFlow) SamplerAddressObj() net.IP {
+	return net.IP(flow.SamplerAddress)
+}
+
 func (flow *EnrichedFlow) GetBps() uint64 {
 	duration := flow.TimeFlowEnd - flow.TimeFlowStart
 	if duration == 0 {

pb/flow.proto

@@ -196,8 +196,13 @@ message EnrichedFlow {
   RemoteAddrType RemoteAddr = 1011; // TODO: figure out a better system? applicable only to service providers right now...
 
   // modify/reversedns
-  string SrcAddrName = 1180; // new
-  string DstAddrName = 1181; // new
+  string SrcHostName = 1180;
+  string DstHostName = 1181;
+  string NextHopHostName = 1182;
+  string SrcASName = 1183;
+  string DstASName = 1184;
+  string NextHopASName = 1185;
+  string SamplerHostName = 1186;
 
   // modify/snmp
   string SrcIfName = 1003;  // TODO: rename to match InIf and OutIf