Add pelican metrics

.github/workflows/test.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:

amqp.go

@@ -2,7 +2,6 @@ package shoveler
 
 import (
 	"errors"
-	"io/ioutil"
 	"math/rand"
 	"net/url"
 	"os"
@@ -139,7 +138,7 @@ func readMsg(messagesQueue chan<- []byte, queue *ConfirmationQueue) {
 func readToken(tokenLocation string) (string, error) {
 	// Get the token password
 	// Read in the token file
-	tokenContents, err := ioutil.ReadFile(tokenLocation)
+	tokenContents, err := os.ReadFile(tokenLocation)
 	if err != nil {
 		log.Errorln("Unable to read file:", tokenLocation)
 		return "", err

cmd/createtoken/main.go

@@ -5,7 +5,6 @@ import (
 	"encoding/pem"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"time"
 
@@ -25,7 +24,7 @@ func main() {
 	}
 
 	// Read in the private key
-	pemString, err := ioutil.ReadFile(flag.Args()[0])
+	pemString, err := os.ReadFile(flag.Args()[0])
 	if err != nil {
 		fmt.Println("Failed to read in private key:", os.Args[1], ":", err)
 		os.Exit(1)

cmd/shoveler/main.go

@@ -115,6 +115,11 @@ func main() {
 			continue
 		}
 
+		// Send to the metrics processor, if enabled
+		if config.Metrics {
+			shoveler.ProcessMetricsPacket(buf[:rlen])
+		}
+
 		msg := shoveler.PackageUdp(buf[:rlen], remote)
 
 		// Send the message to the queue

go.mod

@@ -1,19 +1,58 @@
 module github.com/opensciencegrid/xrootd-monitoring-shoveler
 
-go 1.16
+go 1.20
 
 require (
 	github.com/go-stomp/stomp/v3 v3.0.3
 	github.com/golang-jwt/jwt v3.2.1+incompatible
 	github.com/golang-jwt/jwt/v4 v4.4.2
+	github.com/jellydator/ttlcache/v3 v3.1.0
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/joncrlsn/dque v0.0.0-20211108142734-c2ef48c5192a
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/pterm/pterm v0.12.49
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/viper v1.10.0
 	github.com/streadway/amqp v1.0.0
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.8.2
+)
+
+require (
+	atomicgo.dev/cursor v0.1.1 // indirect
+	atomicgo.dev/keyboard v0.2.8 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/containerd/console v1.0.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/gofrs/flock v0.7.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/gookit/color v1.5.2 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/lithammer/fuzzysearch v1.1.5 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

metrics.go

@@ -1,12 +1,14 @@
 package shoveler
 
 import (
+	"net/http"
+	"strconv"
+
+	"github.com/opensciencegrid/xrootd-monitoring-shoveler/metrics"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
-	"net/http"
-	"strconv"
 )
 
 var (
@@ -29,6 +31,11 @@ var (
 		Name: "shoveler_queue_size",
 		Help: "The number of messages in the queue",
 	})
+
+	MetricsProcessFailures = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "metrics_process_failure",
+		Help: "The number of packets that failed to be processed by the metrics library",
+	})
 )
 
 func StartMetrics(metricsPort int) {
@@ -46,3 +53,12 @@ func StartMetrics(metricsPort int) {
 	}()
 
 }
+
+func ProcessMetricsPacket(bytes []byte) {
+	err := metrics.HandlePacket(bytes)
+	if err != nil {
+		log.Errorln("Failed to process metrics packet:", err)
+		MetricsProcessFailures.Inc()
+		return
+	}
+}