diff --git a/cmd/crowdsec/lpmetrics.go b/cmd/crowdsec/lpmetrics.go index fe5ec00c922..e5544571af2 100644 --- a/cmd/crowdsec/lpmetrics.go +++ b/cmd/crowdsec/lpmetrics.go @@ -48,6 +48,7 @@ type MetricsProvider struct { type staticMetrics struct { osName string + osFamily string osVersion string startupTS int64 featureFlags []string @@ -71,9 +72,11 @@ func getHubState(hub *cwhub.Hub) models.HubItems { if item.State.IsLocal() { status = "custom" } + if item.State.Tainted { status = "tainted" } + ret[itemType] = append(ret[itemType], models.HubItem{ Name: item.Name, Status: status, @@ -93,10 +96,11 @@ func newStaticMetrics(consoleOptions []string, datasources []acquisition.DataSou datasourceMap[ds.GetName()] += 1 } - osName, osVersion := version.DetectOS() + osName, osFamily, osVersion := version.DetectOS() return staticMetrics{ osName: osName, + osFamily: osFamily, osVersion: osVersion, startupTS: time.Now().UTC().Unix(), featureFlags: fflag.Crowdsec.GetEnabledFeatures(), @@ -106,14 +110,23 @@ func newStaticMetrics(consoleOptions []string, datasources []acquisition.DataSou } } -func NewMetricsProvider(apic *apiclient.ApiClient, interval time.Duration, logger *logrus.Entry, - consoleOptions []string, datasources []acquisition.DataSource, hub *cwhub.Hub, +func NewMetricsProvider( + apic *apiclient.ApiClient, + interval time.Duration, + logger *logrus.Entry, + consoleOptions []string, + datasources []acquisition.DataSource, + hub *cwhub.Hub, ) *MetricsProvider { + static := newStaticMetrics(consoleOptions, datasources, hub) + + logger.Debugf("Detected %s %s (family: %s)", static.osName, static.osVersion, static.osFamily) + return &MetricsProvider{ apic: apic, interval: interval, logger: logger, - static: newStaticMetrics(consoleOptions, datasources, hub), + static: static, } } @@ -123,6 +136,7 @@ func getLabelValue(labels []*io_prometheus_client.LabelPair, key string) string return label.GetValue() } } + return "" } @@ -135,9 +149,11 @@ func getDeltaKey(metricName string, labels []*io_prometheus_client.LabelPair) st slices.SortFunc(sortedLabels, func(a, b *io_prometheus_client.LabelPair) int { return strings.Compare(a.GetName(), b.GetName()) }) + for _, label := range sortedLabels { parts = append(parts, label.GetName()+label.GetValue()) } + return strings.Join(parts, "") } @@ -147,10 +163,12 @@ func shouldIgnoreMetric(exclude map[string]*regexp.Regexp, promLabels []*io_prom if labelValue == "" { continue } + if regex.MatchString(labelValue) { return true } } + return false } @@ -167,6 +185,7 @@ func (m *MetricsProvider) gatherPromMetrics(metricsName []string, labelsMap labe if !slices.Contains(metricsName, metricFamily.GetName()) { continue } + for _, metric := range metricFamily.GetMetric() { promLabels := metric.GetLabel() @@ -191,6 +210,7 @@ func (m *MetricsProvider) gatherPromMetrics(metricsName []string, labelsMap labe value = 0 } } + metricsLastValues[deltaKey] = currentValue if value == 0 { @@ -312,12 +332,12 @@ func (m *MetricsProvider) metricsPayload() *models.AllMetrics { Items: make([]*models.MetricsDetailItem, 0), }) - /* Acquisition metrics */ - /*{"name": "read", "value": 10, "unit": "line", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ - /* Parser metrics */ - /*{"name": "parsed", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ - /*{"name": "unparsed", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ - /*{"name": "whitelisted", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ + // Acquisition metrics + // {"name": "read", "value": 10, "unit": "line", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}} + // Parser metrics + // {"name": "parsed", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}} + // {"name": "unparsed", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}} + // {"name": "whitelisted", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}} acquisitionMetrics := m.getAcquisitionMetrics() if len(acquisitionMetrics) > 0 { diff --git a/go.mod b/go.mod index 3a636a2ae68..7092ba87238 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.3 github.com/beevik/etree v1.4.1 - github.com/blackfireio/osinfo v1.1.0 // indirect github.com/bluele/gcache v0.0.2 github.com/buger/jsonparser v1.1.1 github.com/cenkalti/backoff/v5 v5.0.2 @@ -26,7 +25,7 @@ require ( github.com/corazawaf/coraza/v3 v3.3.3 github.com/corazawaf/libinjection-go v0.2.2 github.com/crowdsecurity/dlog v0.0.2 - github.com/crowdsecurity/go-cs-lib v0.0.21 + github.com/crowdsecurity/go-cs-lib v0.0.23 github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.3 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -74,7 +73,7 @@ require ( github.com/r3labs/diff/v2 v2.15.1 github.com/sanity-io/litter v1.5.8 github.com/segmentio/kafka-go v0.4.48 - github.com/shirou/gopsutil/v4 v4.25.7 + github.com/shirou/gopsutil/v4 v4.25.8 github.com/sirupsen/logrus v1.9.3 github.com/slack-go/slack v0.16.0 github.com/spf13/cobra v1.10.1 @@ -195,7 +194,7 @@ require ( github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/go.sum b/go.sum index c1fc9a56b1a..30609a2c6dc 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blackfireio/osinfo v1.1.0 h1:1LMkMiFL42+Brx7r3MKuf7UTlXBRgebFLJQAfoFafj8= -github.com/blackfireio/osinfo v1.1.0/go.mod h1:Pd987poVNmd5Wsx6PRPw4+w7kLlf9iJxoRKPtPAjOrA= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= @@ -142,8 +140,8 @@ github.com/crowdsecurity/coraza/v3 v3.3.3-crowdsec.20250609 h1:t0fgUIJ7FnDKQSigu github.com/crowdsecurity/coraza/v3 v3.3.3-crowdsec.20250609/go.mod h1:q/LGNBRelJdzJZK08U1Rm5cNHv9DKp98p0esMDhJ5tE= github.com/crowdsecurity/dlog v0.0.2 h1:nj/7jLKO0o8tYn79O+g51ASeGLr5oOVahSoJ6Umq51g= github.com/crowdsecurity/dlog v0.0.2/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.21 h1:LUq3QIwICRaxsSJjV5KADP5HbY4GLcnkqS0IX8QzLxI= -github.com/crowdsecurity/go-cs-lib v0.0.21/go.mod h1:HLUGOlXGjkXTpONmQACOE+YayoiudaGjRsE89dSa8So= +github.com/crowdsecurity/go-cs-lib v0.0.23 h1:9YPJG97uXZh95uwvPWeFeAZPXjZ7HaUdtcyFCLSYxu8= +github.com/crowdsecurity/go-cs-lib v0.0.23/go.mod h1:X0GMJY2CxdA1S09SpuqIKaWQsvRGxXmecUp9cP599dE= github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4= github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.3 h1:mgd//PhMJqyA1EdRTgwRvafwbzNjoktdJyEgZGVCD2Q= @@ -495,8 +493,8 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= @@ -540,8 +538,8 @@ github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApI github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shirou/gopsutil/v4 v4.25.7 h1:bNb2JuqKuAu3tRlPv5piSmBZyMfecwQ+t/ILq+1JqVM= -github.com/shirou/gopsutil/v4 v4.25.7/go.mod h1:XV/egmwJtd3ZQjBpJVY5kndsiOO4IRqy9TQnmm6VP7U= +github.com/shirou/gopsutil/v4 v4.25.8 h1:NnAsw9lN7587WHxjJA9ryDnqhJpFH6A+wagYWTOH970= +github.com/shirou/gopsutil/v4 v4.25.8/go.mod h1:q9QdMmfAOVIw7a+eF86P7ISEU6ka+NLgkUxlopV4RwI= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= diff --git a/pkg/apiserver/apic_metrics.go b/pkg/apiserver/apic_metrics.go index ede66f6b954..576eaacc145 100644 --- a/pkg/apiserver/apic_metrics.go +++ b/pkg/apiserver/apic_metrics.go @@ -25,7 +25,7 @@ type dbPayload struct { func (a *apic) GetUsageMetrics(ctx context.Context) (*models.AllMetrics, []int, error) { allMetrics := &models.AllMetrics{} - metricsIds := make([]int, 0) + metricsIDs := make([]int, 0) lps, err := a.dbClient.ListMachines(ctx) if err != nil { @@ -66,7 +66,7 @@ func (a *apic) GetUsageMetrics(ctx context.Context) (*models.AllMetrics, []int, for _, dbMetric := range dbMetrics { dbPayload := &dbPayload{} // Append no matter what, if we cannot unmarshal, there's no way we'll be able to fix it automatically - metricsIds = append(metricsIds, dbMetric.ID) + metricsIDs = append(metricsIDs, dbMetric.ID) err := json.Unmarshal([]byte(dbMetric.Payload), dbPayload) if err != nil { @@ -128,7 +128,7 @@ func (a *apic) GetUsageMetrics(ctx context.Context) (*models.AllMetrics, []int, for _, dbMetric := range dbMetrics { dbPayload := &dbPayload{} // Append no matter what, if we cannot unmarshal, there's no way we'll be able to fix it automatically - metricsIds = append(metricsIds, dbMetric.ID) + metricsIDs = append(metricsIDs, dbMetric.ID) err := json.Unmarshal([]byte(dbMetric.Payload), dbPayload) if err != nil { @@ -150,10 +150,11 @@ func (a *apic) GetUsageMetrics(ctx context.Context) (*models.AllMetrics, []int, }, } - osName, osVersion := version.DetectOS() + osName, osFamily, osVersion := version.DetectOS() allMetrics.Lapi.Os = &models.OSversion{ Name: ptr.Of(osName), + Family: osFamily, Version: ptr.Of(osVersion), } allMetrics.Lapi.Version = ptr.Of(version.String()) @@ -178,7 +179,7 @@ func (a *apic) GetUsageMetrics(ctx context.Context) (*models.AllMetrics, []int, allMetrics.LogProcessors = make([]*models.LogProcessorsMetrics, 0) } - return allMetrics, metricsIds, nil + return allMetrics, metricsIDs, nil } func (a *apic) MarkUsageMetricsAsSent(ctx context.Context, ids []int) error { @@ -358,7 +359,7 @@ func (a *apic) SendUsageMetrics(ctx context.Context) { ticker.Reset(a.usageMetricsInterval) } - metrics, metricsId, err := a.GetUsageMetrics(ctx) + metrics, metricsID, err := a.GetUsageMetrics(ctx) if err != nil { log.Errorf("unable to get usage metrics: %s", err) continue @@ -380,13 +381,13 @@ func (a *apic) SendUsageMetrics(ctx context.Context) { } } - err = a.MarkUsageMetricsAsSent(ctx, metricsId) + err = a.MarkUsageMetricsAsSent(ctx, metricsID) if err != nil { log.Errorf("unable to mark usage metrics as sent: %s", err) continue } - log.Infof("Sent %d usage metrics", len(metricsId)) + log.Infof("Sent %d usage metrics", len(metricsID)) } } } diff --git a/pkg/models/localapi_swagger.yaml b/pkg/models/localapi_swagger.yaml index fda6bce31e8..30ddb7f89fd 100644 --- a/pkg/models/localapi_swagger.yaml +++ b/pkg/models/localapi_swagger.yaml @@ -1290,6 +1290,10 @@ definitions: type: string description: name of the OS maxLength: 255 + family: + type: string + description: family of the OS + maxLength: 255 version: type: string description: version of the OS diff --git a/pkg/models/o_sversion.go b/pkg/models/o_sversion.go index 8f1f43ea9cc..49dd9fc5e27 100644 --- a/pkg/models/o_sversion.go +++ b/pkg/models/o_sversion.go @@ -19,6 +19,10 @@ import ( // swagger:model OSversion type OSversion struct { + // family of the OS + // Max Length: 255 + Family string `json:"family,omitempty"` + // name of the OS // Required: true // Max Length: 255 @@ -34,6 +38,10 @@ type OSversion struct { func (m *OSversion) Validate(formats strfmt.Registry) error { var res []error + if err := m.validateFamily(formats); err != nil { + res = append(res, err) + } + if err := m.validateName(formats); err != nil { res = append(res, err) } @@ -48,6 +56,18 @@ func (m *OSversion) Validate(formats strfmt.Registry) error { return nil } +func (m *OSversion) validateFamily(formats strfmt.Registry) error { + if swag.IsZero(m.Family) { // not required + return nil + } + + if err := validate.MaxLength("family", "body", m.Family, 255); err != nil { + return err + } + + return nil +} + func (m *OSversion) validateName(formats strfmt.Registry) error { if err := validate.Required("name", "body", m.Name); err != nil {