Handle export segment labels/tags dynamically (#55)

* make prometheus segment accept any label sets

* make influx segment accept any tag set

* use dynamic field list learnings in csv segment

* fixup changes to influx segment

* create documentation for influx, fixup prometheus docs

* update deps to latest versions

* BROKEN: this revision contains a broken import of Participle for demo purposes

* fix participle dependency

* update deps once again

* finish dep update, make gh-action use correct go version

* change default fiels for influx segment

* adjust docs according to previous commit

.github/workflows/master.yml

@@ -23,7 +23,7 @@ jobs:
     - name: setup go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.17
+        go-version: 1.18
 
     - name: test
       run: go test ./...

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
     - name: setup go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.17
+        go-version: 1.18
 
     - name: test before release
       run: go test ./...

CONFIGURATION.md

@@ -158,13 +158,37 @@ i.e. some fields might be lost. For instance, the `prometheus` segment as a
 metric provider does not export any information about flow timing or duration,
 among others.
 
+#### influx
+The `influx` segment provides a way to write into an Influxdb instance.
+The `tags` parameter allows any field to be used as a tag and takes a comma-separated list from any
+field available in the [protobuf definition](https://github.com/bwNetFlow/flowpipeline/blob/master/pb/flow.proto).
+The `fields` works in the exact same way, except that these protobuf fields won't be indexed by InfluxDB.
+
+Note that some of the above fields might not be present depending on the method
+of flow export, the input segment used in this pipeline, or the modify segments
+in front of this export segment.
+
+```
+- segment: influx
+  config:
+    org: my-org
+    bucket: my-bucket
+    token: $AUTH_TOKEN_ENVVAR
+    # the lines below are optional and set to default
+    address: http://127.0.0.1:8086
+    tags: "ProtoName"
+    fields: "Bytes,Packets"
+```
+
+[godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/export/prometheus)
+[examples using this segment](https://github.com/search?q=%22segment%3A+prometheus%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code)
+
+
 #### prometheus
 The `prometheus` segment provides a standard prometheus exporter, exporting its
 own monitoring info at `:8080/metrics` and its flow data at `:8080/flowdata` by
 default. The label set included with each metric is freely configurable with a
-comma-separated list from any of the follwing valid fields: `router`,
-`ipversion`, `application`, `protoname`, `direction`, `peer`, `remoteas`,
-`remotecountry`, `src_port`, `dst_port`, `src_addr`, `dst_addr`.
+comma-separated list from any field available in the [protobuf definition](https://github.com/bwNetFlow/flowpipeline/blob/master/pb/flow.proto).
 
 Note that some of the above fields might not be present depending on the method
 of flow export, the input segment used in this pipeline, or the modify segments
@@ -173,9 +197,9 @@ in front of this export segment.
 ```
 - segment: prometheus
   config:
-    labels: "router,ipversion,application,protoname,direction,peer,remoteas,remotecountry"
     # the lines below are optional and set to default
     endpoint: ":8080"
+    labels: "Etype,Proto"
     metricspath: "/metrics"
     flowdatapath: "/flowdata"
 ```

go.mod

@@ -3,41 +3,41 @@ module github.com/bwNetFlow/flowpipeline
 go 1.18
 
 require (
-	github.com/BelWue/bgp_routeinfo v0.0.0-20220510102019-664808e55ca7
-	github.com/Shopify/sarama v1.34.0
+	github.com/Shopify/sarama v1.37.2
 	github.com/Yawning/cryptopan v0.0.0-20170504040949-65bca51288fe
 	github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab
 	github.com/asecurityteam/rolling v2.0.4+incompatible
 	github.com/bwNetFlow/bpf_flowexport v0.0.0-20220515112212-cd8128615c05
-	github.com/bwNetFlow/flowfilter v0.0.0-20220602133712-054b7de2c4d6
+	github.com/bwNetFlow/flowfilter v0.0.0-20221025122858-60746fa15915
 	github.com/bwNetFlow/ip_prefix_trie v0.0.0-20210830112018-b360b7b65c04
 	github.com/bwNetFlow/protobuf/go v0.0.0-20211004083441-61e193b4b342
 	github.com/dustin/go-humanize v1.0.0
 	github.com/hashicorp/logutils v1.0.0
-	github.com/influxdata/influxdb-client-go/v2 v2.9.0
-	github.com/mattn/go-sqlite3 v1.14.13
-	github.com/netsampler/goflow2 v1.1.0
-	github.com/oschwald/maxminddb-golang v1.9.0
+	github.com/influxdata/influxdb-client-go/v2 v2.11.0
+	github.com/mattn/go-sqlite3 v1.14.15
+	github.com/netsampler/goflow2 v1.1.1
+	github.com/oschwald/maxminddb-golang v1.10.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/prometheus/client_golang v1.12.2
-	google.golang.org/protobuf v1.28.0
+	github.com/prometheus/client_golang v1.13.0
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	github.com/alecthomas/participle/v2 v2.0.0-alpha9 // indirect
+	github.com/BelWue/bgp_routeinfo v0.0.0-20221004100427-d8095fc566dd // indirect
+	github.com/alecthomas/participle/v2 v2.0.0-beta.1 // indirect
 	github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cilium/ebpf v0.9.0 // indirect
+	github.com/cilium/ebpf v0.9.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deepmap/oapi-codegen v1.11.0 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/eapache/channels v1.1.0 // indirect
-	github.com/eapache/go-resiliency v1.2.0 // indirect
+	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.3.0 // indirect
@@ -48,39 +48,39 @@ require (
 	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
-	github.com/jcmturner/gofork v1.0.0 // indirect
-	github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/k-sone/critbitgo v1.4.0 // indirect
-	github.com/klauspost/compress v1.15.5 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/libp2p/go-reuseport v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/osrg/gobgp/v3 v3.2.0 // indirect
+	github.com/osrg/gobgp/v3 v3.7.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.14 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.34.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/spf13/afero v1.8.2 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.12.0 // indirect
-	github.com/subosito/gotenv v1.3.0 // indirect
-	github.com/vishvananda/netlink v1.2.0-beta // indirect
-	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
-	golang.org/x/net v0.0.0-20220531201128-c960675eff93 // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	gopkg.in/ini.v1 v1.66.6 // indirect
+	github.com/spf13/viper v1.13.0 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
+	github.com/vishvananda/netns v0.0.0-20220913150850-18c4f4234207 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
+	google.golang.org/grpc v1.50.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )