From ed4232f662a56da78be4ac2a930017ffac7247e2 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 15:30:21 +0000 Subject: [PATCH 01/77] create mib package --- main.go | 25 ++++++++----------------- mib/mib.go | 12 ++++++++++++ 2 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 mib/mib.go diff --git a/main.go b/main.go index 003439e..6802336 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,8 @@ import ( "github.com/gosnmp/gosnmp" "github.com/sirupsen/logrus" + + "github.com/yseto/switch-traffic-to-mackerel/mib" ) const ( @@ -22,17 +24,6 @@ const ( MIBifOperStatus = "1.3.6.1.2.1.2.2.1.8" ) -var mibOidmapping = map[string]string{ - "ifInOctets": "1.3.6.1.2.1.2.2.1.10", - "ifOutOctets": "1.3.6.1.2.1.2.2.1.16", - "ifHCInOctets": "1.3.6.1.2.1.31.1.1.1.6", - "ifHCOutOctets": "1.3.6.1.2.1.31.1.1.1.10", - "ifInDiscards": "1.3.6.1.2.1.2.2.1.13", - "ifOutDiscards": "1.3.6.1.2.1.2.2.1.19", - "ifInErrors": "1.3.6.1.2.1.2.2.1.14", - "ifOutErrors": "1.3.6.1.2.1.2.2.1.20", -} - type MetricsDutum struct { IfIndex uint64 `json:"ifIndex"` Mib string `json:"mib"` @@ -162,8 +153,8 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { metrics := make([]MetricsDutum, 0) - for _, mib := range c.mibs { - values, err := bulkWalk(mibOidmapping[mib], ifNumber) + for _, mibName := range c.mibs { + values, err := bulkWalk(mib.Oidmapping[mibName], ifNumber) if err != nil { return nil, err } @@ -185,12 +176,12 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { log.WithFields(logrus.Fields{ "IfIndex": ifIndex, - "Mib": mib, + "Mib": mibName, "IfName": ifName, "Value": value, }).Debug() - metrics = append(metrics, MetricsDutum{IfIndex: ifIndex, Mib: mib, IfName: ifName, Value: value}) + metrics = append(metrics, MetricsDutum{IfIndex: ifIndex, Mib: mibName, IfName: ifName, Value: value}) } } return metrics, nil @@ -200,7 +191,7 @@ func mibsValidate(rawMibs *string) ([]string, error) { var parseMibs []string switch *rawMibs { case "all": - for key := range mibOidmapping { + for key := range mib.Oidmapping { // skipped 32 bit octets. if key == "ifInOctets" || key == "ifOutOctets" { continue @@ -210,7 +201,7 @@ func mibsValidate(rawMibs *string) ([]string, error) { case "": default: for _, name := range strings.Split(*rawMibs, ",") { - if _, exists := mibOidmapping[name]; !exists { + if _, exists := mib.Oidmapping[name]; !exists { return nil, fmt.Errorf("mib %s is not supported.", name) } parseMibs = append(parseMibs, name) diff --git a/mib/mib.go b/mib/mib.go new file mode 100644 index 0000000..b816ea2 --- /dev/null +++ b/mib/mib.go @@ -0,0 +1,12 @@ +package mib + +var Oidmapping = map[string]string{ + "ifInOctets": "1.3.6.1.2.1.2.2.1.10", + "ifOutOctets": "1.3.6.1.2.1.2.2.1.16", + "ifHCInOctets": "1.3.6.1.2.1.31.1.1.1.6", + "ifHCOutOctets": "1.3.6.1.2.1.31.1.1.1.10", + "ifInDiscards": "1.3.6.1.2.1.2.2.1.13", + "ifOutDiscards": "1.3.6.1.2.1.2.2.1.19", + "ifInErrors": "1.3.6.1.2.1.2.2.1.14", + "ifOutErrors": "1.3.6.1.2.1.2.2.1.20", +} From 6c6f72737434b8ce28441bfe1ccd96546224404d Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 15:35:06 +0000 Subject: [PATCH 02/77] move mibsValidate --- main.go | 25 +------------------------ mib/mib.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index 6802336..5dc8ce6 100644 --- a/main.go +++ b/main.go @@ -76,7 +76,7 @@ func parseFlags() (*CollectParams, error) { return nil, err } - mibs, err := mibsValidate(rawMibs) + mibs, err := mib.Validate(rawMibs) if err != nil { return nil, err } @@ -187,29 +187,6 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { return metrics, nil } -func mibsValidate(rawMibs *string) ([]string, error) { - var parseMibs []string - switch *rawMibs { - case "all": - for key := range mib.Oidmapping { - // skipped 32 bit octets. - if key == "ifInOctets" || key == "ifOutOctets" { - continue - } - parseMibs = append(parseMibs, key) - } - case "": - default: - for _, name := range strings.Split(*rawMibs, ",") { - if _, exists := mib.Oidmapping[name]; !exists { - return nil, fmt.Errorf("mib %s is not supported.", name) - } - parseMibs = append(parseMibs, name) - } - } - return parseMibs, nil -} - func captureIfIndex(oid, name string) (uint64, error) { indexStr := strings.Replace(name, "."+oid+".", "", 1) return strconv.ParseUint(indexStr, 10, 64) diff --git a/mib/mib.go b/mib/mib.go index b816ea2..552593f 100644 --- a/mib/mib.go +++ b/mib/mib.go @@ -1,5 +1,10 @@ package mib +import ( + "fmt" + "strings" +) + var Oidmapping = map[string]string{ "ifInOctets": "1.3.6.1.2.1.2.2.1.10", "ifOutOctets": "1.3.6.1.2.1.2.2.1.16", @@ -10,3 +15,26 @@ var Oidmapping = map[string]string{ "ifInErrors": "1.3.6.1.2.1.2.2.1.14", "ifOutErrors": "1.3.6.1.2.1.2.2.1.20", } + +func Validate(rawMibs *string) ([]string, error) { + var parseMibs []string + switch *rawMibs { + case "all": + for key := range Oidmapping { + // skipped 32 bit octets. + if key == "ifInOctets" || key == "ifOutOctets" { + continue + } + parseMibs = append(parseMibs, key) + } + case "": + default: + for _, name := range strings.Split(*rawMibs, ",") { + if _, exists := Oidmapping[name]; !exists { + return nil, fmt.Errorf("mib %s is not supported.", name) + } + parseMibs = append(parseMibs, name) + } + } + return parseMibs, nil +} From 1dfe1a00192dc3c3f2c32d846aee6a486db2cdfd Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 15:45:39 +0000 Subject: [PATCH 03/77] create snmp package --- main.go | 19 ++++++++----------- snmp/snmp.go | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 snmp/snmp.go diff --git a/main.go b/main.go index 5dc8ce6..3e8d77e 100644 --- a/main.go +++ b/main.go @@ -10,12 +10,12 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/gosnmp/gosnmp" "github.com/sirupsen/logrus" "github.com/yseto/switch-traffic-to-mackerel/mib" + "github.com/yseto/switch-traffic-to-mackerel/snmp" ) const ( @@ -110,10 +110,7 @@ func main() { log.Info("start") - gosnmp.Default.Target = collectParams.target - gosnmp.Default.Community = collectParams.community - gosnmp.Default.Timeout = time.Duration(10 * time.Second) - gosnmp.Default.Context = ctx + snmp.Init(ctx, collectParams.target, collectParams.community) if apikey == "" { log.SetLevel(logrus.DebugLevel) @@ -128,11 +125,11 @@ func main() { } func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { - err := gosnmp.Default.Connect() + err := snmp.Default.Connect() if err != nil { return nil, err } - defer gosnmp.Default.Conn.Close() + defer snmp.Default.Conn.Close() ifNumber, err := getInterfaceNumber() if err != nil { @@ -193,7 +190,7 @@ func captureIfIndex(oid, name string) (uint64, error) { } func getInterfaceNumber() (uint64, error) { - result, err := gosnmp.Default.Get([]string{MIBifNumber}) + result, err := snmp.Default.Get([]string{MIBifNumber}) if err != nil { return 0, err } @@ -208,7 +205,7 @@ func getInterfaceNumber() (uint64, error) { func bulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { kv := make(map[uint64]string, length) - err := gosnmp.Default.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { + err := snmp.Default.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { index, err := captureIfIndex(MIBifDescr, pdu.Name) if err != nil { return err @@ -229,7 +226,7 @@ func bulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { func bulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { kv := make(map[uint64]bool, length) - err := gosnmp.Default.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { + err := snmp.Default.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { index, err := captureIfIndex(MIBifOperStatus, pdu.Name) if err != nil { return err @@ -255,7 +252,7 @@ func bulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { func bulkWalk(oid string, length uint64) (map[uint64]uint64, error) { kv := make(map[uint64]uint64, length) - err := gosnmp.Default.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { + err := snmp.Default.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { index, err := captureIfIndex(oid, pdu.Name) if err != nil { return err diff --git a/snmp/snmp.go b/snmp/snmp.go new file mode 100644 index 0000000..3c7c3a7 --- /dev/null +++ b/snmp/snmp.go @@ -0,0 +1,25 @@ +package snmp + +import ( + "context" + "time" + + "github.com/gosnmp/gosnmp" +) + +var Default *gosnmp.GoSNMP + +func Init(ctx context.Context, target, community string) { + Default = &gosnmp.GoSNMP{ + Context: ctx, + Target: target, + Port: 161, + Transport: "udp", + Community: community, + Version: gosnmp.Version2c, + Timeout: time.Duration(2) * time.Second, + Retries: 3, + ExponentialTimeout: true, + MaxOids: gosnmp.MaxOids, + } +} From b23053ff3cd2c24ab515c1aaa847e476b680275a Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 15:52:34 +0000 Subject: [PATCH 04/77] move some function to snmp package --- main.go | 104 ++------------------------------------------------- snmp/snmp.go | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 100 deletions(-) diff --git a/main.go b/main.go index 3e8d77e..cacabf0 100644 --- a/main.go +++ b/main.go @@ -8,22 +8,13 @@ import ( "os" "os/signal" "regexp" - "strconv" - "strings" - "github.com/gosnmp/gosnmp" "github.com/sirupsen/logrus" "github.com/yseto/switch-traffic-to-mackerel/mib" "github.com/yseto/switch-traffic-to-mackerel/snmp" ) -const ( - MIBifNumber = "1.3.6.1.2.1.2.1.0" - MIBifDescr = "1.3.6.1.2.1.2.2.1.2" - MIBifOperStatus = "1.3.6.1.2.1.2.2.1.8" -) - type MetricsDutum struct { IfIndex uint64 `json:"ifIndex"` Mib string `json:"mib"` @@ -131,18 +122,18 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { } defer snmp.Default.Conn.Close() - ifNumber, err := getInterfaceNumber() + ifNumber, err := snmp.GetInterfaceNumber() if err != nil { return nil, err } - ifDescr, err := bulkWalkGetInterfaceName(ifNumber) + ifDescr, err := snmp.BulkWalkGetInterfaceName(ifNumber) if err != nil { return nil, err } var ifOperStatus map[uint64]bool if *c.skipDownLinkState { - ifOperStatus, err = bulkWalkGetInterfaceState(ifNumber) + ifOperStatus, err = snmp.BulkWalkGetInterfaceState(ifNumber) if err != nil { return nil, err } @@ -151,7 +142,7 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { metrics := make([]MetricsDutum, 0) for _, mibName := range c.mibs { - values, err := bulkWalk(mib.Oidmapping[mibName], ifNumber) + values, err := snmp.BulkWalk(mib.Oidmapping[mibName], ifNumber) if err != nil { return nil, err } @@ -183,90 +174,3 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { } return metrics, nil } - -func captureIfIndex(oid, name string) (uint64, error) { - indexStr := strings.Replace(name, "."+oid+".", "", 1) - return strconv.ParseUint(indexStr, 10, 64) -} - -func getInterfaceNumber() (uint64, error) { - result, err := snmp.Default.Get([]string{MIBifNumber}) - if err != nil { - return 0, err - } - variable := result.Variables[0] - switch variable.Type { - case gosnmp.OctetString: - return 0, errors.New("cant get interface number") - default: - return gosnmp.ToBigInt(variable.Value).Uint64(), nil - } -} - -func bulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { - kv := make(map[uint64]string, length) - err := snmp.Default.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { - index, err := captureIfIndex(MIBifDescr, pdu.Name) - if err != nil { - return err - } - switch pdu.Type { - case gosnmp.OctetString: - kv[index] = string(pdu.Value.([]byte)) - default: - return errors.New("cant parse interface name.") - } - return nil - }) - if err != nil { - return nil, err - } - return kv, nil -} - -func bulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { - kv := make(map[uint64]bool, length) - err := snmp.Default.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { - index, err := captureIfIndex(MIBifOperStatus, pdu.Name) - if err != nil { - return err - } - switch pdu.Type { - case gosnmp.OctetString: - return errors.New("cant parse value.") - default: - tmp := gosnmp.ToBigInt(pdu.Value).Uint64() - if tmp != 2 { - kv[index] = true - } else { - kv[index] = false - } - } - return nil - }) - if err != nil { - return nil, err - } - return kv, nil -} - -func bulkWalk(oid string, length uint64) (map[uint64]uint64, error) { - kv := make(map[uint64]uint64, length) - err := snmp.Default.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { - index, err := captureIfIndex(oid, pdu.Name) - if err != nil { - return err - } - switch pdu.Type { - case gosnmp.OctetString: - return errors.New("cant parse value.") - default: - kv[index] = gosnmp.ToBigInt(pdu.Value).Uint64() - } - return nil - }) - if err != nil { - return nil, err - } - return kv, nil -} diff --git a/snmp/snmp.go b/snmp/snmp.go index 3c7c3a7..87fa4a3 100644 --- a/snmp/snmp.go +++ b/snmp/snmp.go @@ -2,6 +2,9 @@ package snmp import ( "context" + "errors" + "strconv" + "strings" "time" "github.com/gosnmp/gosnmp" @@ -23,3 +26,96 @@ func Init(ctx context.Context, target, community string) { MaxOids: gosnmp.MaxOids, } } + +const ( + MIBifNumber = "1.3.6.1.2.1.2.1.0" + MIBifDescr = "1.3.6.1.2.1.2.2.1.2" + MIBifOperStatus = "1.3.6.1.2.1.2.2.1.8" +) + +func GetInterfaceNumber() (uint64, error) { + result, err := Default.Get([]string{MIBifNumber}) + if err != nil { + return 0, err + } + variable := result.Variables[0] + switch variable.Type { + case gosnmp.OctetString: + return 0, errors.New("cant get interface number") + default: + return gosnmp.ToBigInt(variable.Value).Uint64(), nil + } +} + +func BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { + kv := make(map[uint64]string, length) + err := Default.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { + index, err := captureIfIndex(MIBifDescr, pdu.Name) + if err != nil { + return err + } + switch pdu.Type { + case gosnmp.OctetString: + kv[index] = string(pdu.Value.([]byte)) + default: + return errors.New("cant parse interface name.") + } + return nil + }) + if err != nil { + return nil, err + } + return kv, nil +} + +func BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { + kv := make(map[uint64]bool, length) + err := Default.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { + index, err := captureIfIndex(MIBifOperStatus, pdu.Name) + if err != nil { + return err + } + switch pdu.Type { + case gosnmp.OctetString: + return errors.New("cant parse value.") + default: + tmp := gosnmp.ToBigInt(pdu.Value).Uint64() + if tmp != 2 { + kv[index] = true + } else { + kv[index] = false + } + } + return nil + }) + if err != nil { + return nil, err + } + return kv, nil +} + +func BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { + kv := make(map[uint64]uint64, length) + err := Default.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { + index, err := captureIfIndex(oid, pdu.Name) + if err != nil { + return err + } + switch pdu.Type { + case gosnmp.OctetString: + return errors.New("cant parse value.") + default: + kv[index] = gosnmp.ToBigInt(pdu.Value).Uint64() + } + return nil + }) + if err != nil { + return nil, err + } + return kv, nil +} + +func captureIfIndex(oid, name string) (uint64, error) { + indexStr := strings.Replace(name, "."+oid+".", "", 1) + return strconv.ParseUint(indexStr, 10, 64) +} From 47e42a65b68c673b2e5887b1754b47f771f48fa0 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 16:13:20 +0000 Subject: [PATCH 05/77] refine snmp package --- main.go | 14 ++++++-------- snmp/snmp.go | 43 +++++++++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index cacabf0..5e7a4fc 100644 --- a/main.go +++ b/main.go @@ -101,8 +101,6 @@ func main() { log.Info("start") - snmp.Init(ctx, collectParams.target, collectParams.community) - if apikey == "" { log.SetLevel(logrus.DebugLevel) @@ -116,24 +114,24 @@ func main() { } func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { - err := snmp.Default.Connect() + snmpClient, err := snmp.Init(ctx, c.target, c.community) if err != nil { return nil, err } - defer snmp.Default.Conn.Close() + defer snmpClient.Close() - ifNumber, err := snmp.GetInterfaceNumber() + ifNumber, err := snmpClient.GetInterfaceNumber() if err != nil { return nil, err } - ifDescr, err := snmp.BulkWalkGetInterfaceName(ifNumber) + ifDescr, err := snmpClient.BulkWalkGetInterfaceName(ifNumber) if err != nil { return nil, err } var ifOperStatus map[uint64]bool if *c.skipDownLinkState { - ifOperStatus, err = snmp.BulkWalkGetInterfaceState(ifNumber) + ifOperStatus, err = snmpClient.BulkWalkGetInterfaceState(ifNumber) if err != nil { return nil, err } @@ -142,7 +140,7 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { metrics := make([]MetricsDutum, 0) for _, mibName := range c.mibs { - values, err := snmp.BulkWalk(mib.Oidmapping[mibName], ifNumber) + values, err := snmpClient.BulkWalk(mib.Oidmapping[mibName], ifNumber) if err != nil { return nil, err } diff --git a/snmp/snmp.go b/snmp/snmp.go index 87fa4a3..bae6220 100644 --- a/snmp/snmp.go +++ b/snmp/snmp.go @@ -10,10 +10,18 @@ import ( "github.com/gosnmp/gosnmp" ) -var Default *gosnmp.GoSNMP +const ( + MIBifNumber = "1.3.6.1.2.1.2.1.0" + MIBifDescr = "1.3.6.1.2.1.2.2.1.2" + MIBifOperStatus = "1.3.6.1.2.1.2.2.1.8" +) + +type SNMP struct { + handler *gosnmp.GoSNMP +} -func Init(ctx context.Context, target, community string) { - Default = &gosnmp.GoSNMP{ +func Init(ctx context.Context, target, community string) (*SNMP, error) { + g := &gosnmp.GoSNMP{ Context: ctx, Target: target, Port: 161, @@ -25,16 +33,19 @@ func Init(ctx context.Context, target, community string) { ExponentialTimeout: true, MaxOids: gosnmp.MaxOids, } + err := g.Connect() + if err != nil { + return nil, err + } + return &SNMP{handler: g}, nil } -const ( - MIBifNumber = "1.3.6.1.2.1.2.1.0" - MIBifDescr = "1.3.6.1.2.1.2.2.1.2" - MIBifOperStatus = "1.3.6.1.2.1.2.2.1.8" -) +func (s *SNMP) Close() error { + return s.handler.Conn.Close() +} -func GetInterfaceNumber() (uint64, error) { - result, err := Default.Get([]string{MIBifNumber}) +func (s *SNMP) GetInterfaceNumber() (uint64, error) { + result, err := s.handler.Get([]string{MIBifNumber}) if err != nil { return 0, err } @@ -47,9 +58,9 @@ func GetInterfaceNumber() (uint64, error) { } } -func BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { +func (s *SNMP) BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { kv := make(map[uint64]string, length) - err := Default.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { + err := s.handler.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { index, err := captureIfIndex(MIBifDescr, pdu.Name) if err != nil { return err @@ -68,9 +79,9 @@ func BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { return kv, nil } -func BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { +func (s *SNMP) BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { kv := make(map[uint64]bool, length) - err := Default.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { + err := s.handler.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { index, err := captureIfIndex(MIBifOperStatus, pdu.Name) if err != nil { return err @@ -94,9 +105,9 @@ func BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { return kv, nil } -func BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { +func (s *SNMP) BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { kv := make(map[uint64]uint64, length) - err := Default.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { + err := s.handler.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { index, err := captureIfIndex(oid, pdu.Name) if err != nil { return err From 782043f48b8bd71bee8adc9fc32f1bb4efff22a6 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 16:48:47 +0000 Subject: [PATCH 06/77] create config package. and sample --- config.yaml.sample | 13 +++++++++++++ config/config.go | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 config.yaml.sample create mode 100644 config/config.go diff --git a/config.yaml.sample b/config.yaml.sample new file mode 100644 index 0000000..1e5aa7b --- /dev/null +++ b/config.yaml.sample @@ -0,0 +1,13 @@ +community: public # the community string for device +target: 192.2.0.1 # ip address +interface: + include: "" # include interface name + exclude: "" # exclude interface name +mibs: # capture mib name + - ifHCInOctets + - ifHCOutOctets +skip-linkdown: true +name: "" # display Name on Mackerel +mackerel: + x-api-key: xxxxx + host-id: xxxxx diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..dae74b1 --- /dev/null +++ b/config/config.go @@ -0,0 +1,21 @@ +package config + +type Config struct { + Community string `yaml:"community"` + Target string `yaml:"target"` + Interface *Interface `yaml:"interface"` + Mibs []string `yaml:"mibs"` + SkipLinkdown bool `yaml:"skip-linkdown"` + Name string `yaml:"name"` + Mackerel *Mackerel `yaml:"mackerel"` +} + +type Interface struct { + Include string `yaml:"include"` + Exclude string `yaml:"exclude"` +} + +type Mackerel struct { + HostID string `yaml:"host-id"` + ApiKey string `yaml:"x-api-key"` +} From 75210bad7acf30b2bdb9c7dffae42e23ee47a31c Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 16:49:52 +0000 Subject: [PATCH 07/77] add yaml --- go.mod | 1 + go.sum | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4ad4834..ff0ce3e 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/gosnmp/gosnmp v1.35.0 github.com/mackerelio/mackerel-client-go v0.23.0 github.com/sirupsen/logrus v1.9.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/go.sum b/go.sum index d4455e9..1e2e6e5 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From cfe56acb51614ae6d3bdd0ba99a3a77ba0365122 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 16:57:23 +0000 Subject: [PATCH 08/77] rewrite config --- config/config.go | 4 ++-- main.go | 57 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index dae74b1..435f7a8 100644 --- a/config/config.go +++ b/config/config.go @@ -11,8 +11,8 @@ type Config struct { } type Interface struct { - Include string `yaml:"include"` - Exclude string `yaml:"exclude"` + Include *string `yaml:"include"` + Exclude *string `yaml:"exclude"` } type Mackerel struct { diff --git a/main.go b/main.go index 5e7a4fc..ae75098 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,9 @@ import ( "regexp" "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "github.com/yseto/switch-traffic-to-mackerel/config" "github.com/yseto/switch-traffic-to-mackerel/mib" "github.com/yseto/switch-traffic-to-mackerel/snmp" ) @@ -35,16 +37,39 @@ var apikey = os.Getenv("MACKEREL_API_KEY") func parseFlags() (*CollectParams, error) { var community, target, name string - flag.StringVar(&community, "community", "public", "the community string for device") - flag.StringVar(&target, "target", "127.0.0.1", "ip address") - includeInterface := flag.String("include-interface", "", "include interface name") - excludeInterface := flag.String("exclude-interface", "", "exclude interface name") + var includeInterface, excludeInterface *string rawMibs := flag.String("mibs", "all", "mib name joind with ',' or 'all'") level := flag.Bool("verbose", false, "verbose") skipDownLinkState := flag.Bool("skip-down-link-state", false, "skip down link state") flag.StringVar(&name, "name", "", "name") + var configFilename string + flag.StringVar(&configFilename, "config", "config.yaml", "config `filename`") flag.Parse() + f, err := os.ReadFile(configFilename) + if err != nil { + log.Fatal(err) + } + var t config.Config + err = yaml.Unmarshal(f, &t) + if err != nil { + log.Fatalf("error: %v", err) + } + + if t.Community == "" { + log.Fatal("community is needed.") + } + community = t.Community + if t.Target == "" { + log.Fatal("target is needed.") + } + target = t.Target + + if t.Interface != nil { + includeInterface = t.Interface.Include + excludeInterface = t.Interface.Exclude + } + logLevel := logrus.WarnLevel if *level { logLevel = logrus.DebugLevel @@ -55,16 +80,22 @@ func parseFlags() (*CollectParams, error) { name = target } - if *includeInterface != "" && *excludeInterface != "" { + if includeInterface != nil && excludeInterface != nil { return nil, errors.New("excludeInterface, includeInterface is exclusive control.") } - includeReg, err := regexp.Compile(*includeInterface) - if err != nil { - return nil, err + var includeReg *regexp.Regexp + if includeInterface != nil { + includeReg, err = regexp.Compile(*includeInterface) + if err != nil { + return nil, err + } } - excludeReg, err := regexp.Compile(*excludeInterface) - if err != nil { - return nil, err + var excludeReg *regexp.Regexp + if excludeInterface != nil { + excludeReg, err = regexp.Compile(*excludeInterface) + if err != nil { + return nil, err + } } mibs, err := mib.Validate(rawMibs) @@ -147,11 +178,11 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { for ifIndex, value := range values { ifName := ifDescr[ifIndex] - if *c.includeInterface != "" && !c.includeRegexp.MatchString(ifName) { + if c.includeInterface != nil && !c.includeRegexp.MatchString(ifName) { continue } - if *c.excludeInterface != "" && c.excludeRegexp.MatchString(ifName) { + if c.excludeInterface != nil && c.excludeRegexp.MatchString(ifName) { continue } From eb143c99b28517d4af31b34d1a15a9424ca07b4e Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:01:35 +0000 Subject: [PATCH 09/77] fix mib.Validate, apply config --- main.go | 3 +-- mib/mib.go | 20 +++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index ae75098..204485b 100644 --- a/main.go +++ b/main.go @@ -38,7 +38,6 @@ var apikey = os.Getenv("MACKEREL_API_KEY") func parseFlags() (*CollectParams, error) { var community, target, name string var includeInterface, excludeInterface *string - rawMibs := flag.String("mibs", "all", "mib name joind with ',' or 'all'") level := flag.Bool("verbose", false, "verbose") skipDownLinkState := flag.Bool("skip-down-link-state", false, "skip down link state") flag.StringVar(&name, "name", "", "name") @@ -98,7 +97,7 @@ func parseFlags() (*CollectParams, error) { } } - mibs, err := mib.Validate(rawMibs) + mibs, err := mib.Validate(t.Mibs) if err != nil { return nil, err } diff --git a/mib/mib.go b/mib/mib.go index 552593f..82d63ec 100644 --- a/mib/mib.go +++ b/mib/mib.go @@ -2,7 +2,6 @@ package mib import ( "fmt" - "strings" ) var Oidmapping = map[string]string{ @@ -16,10 +15,9 @@ var Oidmapping = map[string]string{ "ifOutErrors": "1.3.6.1.2.1.2.2.1.20", } -func Validate(rawMibs *string) ([]string, error) { +func Validate(rawMibs []string) ([]string, error) { var parseMibs []string - switch *rawMibs { - case "all": + if len(rawMibs) == 0 { for key := range Oidmapping { // skipped 32 bit octets. if key == "ifInOctets" || key == "ifOutOctets" { @@ -27,14 +25,14 @@ func Validate(rawMibs *string) ([]string, error) { } parseMibs = append(parseMibs, key) } - case "": - default: - for _, name := range strings.Split(*rawMibs, ",") { - if _, exists := Oidmapping[name]; !exists { - return nil, fmt.Errorf("mib %s is not supported.", name) - } - parseMibs = append(parseMibs, name) + return parseMibs, nil + } + + for _, name := range rawMibs { + if _, exists := Oidmapping[name]; !exists { + return nil, fmt.Errorf("mib %s is not supported.", name) } + parseMibs = append(parseMibs, name) } return parseMibs, nil } From 4df6c717ad45ad2be7da160ee752b8f9e67908fc Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:04:04 +0000 Subject: [PATCH 10/77] apply skip link down state. --- main.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 204485b..587ac11 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ type CollectParams struct { mibs []string includeRegexp, excludeRegexp *regexp.Regexp includeInterface, excludeInterface *string - skipDownLinkState *bool + skipDownLinkState bool } var log = logrus.New() @@ -39,7 +39,6 @@ func parseFlags() (*CollectParams, error) { var community, target, name string var includeInterface, excludeInterface *string level := flag.Bool("verbose", false, "verbose") - skipDownLinkState := flag.Bool("skip-down-link-state", false, "skip down link state") flag.StringVar(&name, "name", "", "name") var configFilename string flag.StringVar(&configFilename, "config", "config.yaml", "config `filename`") @@ -111,7 +110,7 @@ func parseFlags() (*CollectParams, error) { excludeRegexp: excludeReg, includeInterface: includeInterface, excludeInterface: excludeInterface, - skipDownLinkState: skipDownLinkState, + skipDownLinkState: t.SkipLinkdown, }, nil } @@ -160,7 +159,7 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { } var ifOperStatus map[uint64]bool - if *c.skipDownLinkState { + if c.skipDownLinkState { ifOperStatus, err = snmpClient.BulkWalkGetInterfaceState(ifNumber) if err != nil { return nil, err @@ -186,7 +185,7 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { } // skip when down(2) - if *c.skipDownLinkState && !ifOperStatus[ifIndex] { + if c.skipDownLinkState && !ifOperStatus[ifIndex] { continue } From 1ea9529d950e4208c434498ee895bb34597b9913 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:09:12 +0000 Subject: [PATCH 11/77] apply Name --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 587ac11..c36df69 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,6 @@ func parseFlags() (*CollectParams, error) { var community, target, name string var includeInterface, excludeInterface *string level := flag.Bool("verbose", false, "verbose") - flag.StringVar(&name, "name", "", "name") var configFilename string flag.StringVar(&configFilename, "config", "config.yaml", "config `filename`") flag.Parse() @@ -74,6 +73,7 @@ func parseFlags() (*CollectParams, error) { } log.SetLevel(logLevel) + name = t.Name if name == "" { name = target } From 6e56edadea1bf3ea7fd4ee0b23c787dd240df2c4 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:20:17 +0000 Subject: [PATCH 12/77] added debug print --- go.mod | 1 + go.sum | 2 ++ main.go | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ff0ce3e..0ebd0ff 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/gosnmp/gosnmp v1.35.0 github.com/mackerelio/mackerel-client-go v0.23.0 + github.com/maruel/natural v1.1.0 github.com/sirupsen/logrus v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 1e2e6e5..852d214 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/gosnmp/gosnmp v1.35.0/go.mod h1:2AvKZ3n9aEl5TJEo/fFmf/FGO4Nj4cVeEc5yu github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/mackerelio/mackerel-client-go v0.23.0 h1:C6ENbhfAZWofpr2Vc/cRTNdCkQonQ/nRlA08K+9+m/M= github.com/mackerelio/mackerel-client-go v0.23.0/go.mod h1:VM9KAjzs7wkROQ9WdZRvkY8K/bIUllN82dxyK36ZU3s= +github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= +github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= diff --git a/main.go b/main.go index c36df69..d992a08 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,10 @@ import ( "os" "os/signal" "regexp" + "sort" + "strings" + "github.com/maruel/natural" "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" @@ -24,6 +27,10 @@ type MetricsDutum struct { Value uint64 `json:"value"` } +func (m *MetricsDutum) String() string { + return fmt.Sprintf("%d\t%s\t%s\t%d", m.IfIndex, m.IfName, m.Mib, m.Value) +} + type CollectParams struct { community, target, name string mibs []string @@ -133,10 +140,17 @@ func main() { if apikey == "" { log.SetLevel(logrus.DebugLevel) - _, err := collect(ctx, collectParams) + dutum, err := collect(ctx, collectParams) if err != nil { log.Fatal(err) } + + var dutumStr []string + for i := range dutum { + dutumStr = append(dutumStr, dutum[i].String()) + } + sort.Sort(natural.StringSlice(dutumStr)) + fmt.Println(strings.Join(dutumStr, "\n")) } else { runMackerel(ctx, collectParams) } From 382e3fcacdb19b1b7446867784a50d1d28889acb Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:21:19 +0000 Subject: [PATCH 13/77] remove loglevel --- main.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/main.go b/main.go index d992a08..79b8a9a 100644 --- a/main.go +++ b/main.go @@ -45,7 +45,6 @@ var apikey = os.Getenv("MACKEREL_API_KEY") func parseFlags() (*CollectParams, error) { var community, target, name string var includeInterface, excludeInterface *string - level := flag.Bool("verbose", false, "verbose") var configFilename string flag.StringVar(&configFilename, "config", "config.yaml", "config `filename`") flag.Parse() @@ -74,12 +73,6 @@ func parseFlags() (*CollectParams, error) { excludeInterface = t.Interface.Exclude } - logLevel := logrus.WarnLevel - if *level { - logLevel = logrus.DebugLevel - } - log.SetLevel(logLevel) - name = t.Name if name == "" { name = target @@ -138,8 +131,6 @@ func main() { log.Info("start") if apikey == "" { - log.SetLevel(logrus.DebugLevel) - dutum, err := collect(ctx, collectParams) if err != nil { log.Fatal(err) @@ -203,13 +194,6 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { continue } - log.WithFields(logrus.Fields{ - "IfIndex": ifIndex, - "Mib": mibName, - "IfName": ifName, - "Value": value, - }).Debug() - metrics = append(metrics, MetricsDutum{IfIndex: ifIndex, Mib: mibName, IfName: ifName, Value: value}) } } From c1f421c4d9bc60dfc96d284138804d40c4c0a2af Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:26:20 +0000 Subject: [PATCH 14/77] remove useless variable --- main.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 79b8a9a..16bacaf 100644 --- a/main.go +++ b/main.go @@ -32,11 +32,10 @@ func (m *MetricsDutum) String() string { } type CollectParams struct { - community, target, name string - mibs []string - includeRegexp, excludeRegexp *regexp.Regexp - includeInterface, excludeInterface *string - skipDownLinkState bool + community, target, name string + mibs []string + includeRegexp, excludeRegexp *regexp.Regexp + skipDownLinkState bool } var log = logrus.New() @@ -108,8 +107,6 @@ func parseFlags() (*CollectParams, error) { mibs: mibs, includeRegexp: includeReg, excludeRegexp: excludeReg, - includeInterface: includeInterface, - excludeInterface: excludeInterface, skipDownLinkState: t.SkipLinkdown, }, nil } @@ -181,11 +178,11 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { for ifIndex, value := range values { ifName := ifDescr[ifIndex] - if c.includeInterface != nil && !c.includeRegexp.MatchString(ifName) { + if c.includeRegexp != nil && !c.includeRegexp.MatchString(ifName) { continue } - if c.excludeInterface != nil && c.excludeRegexp.MatchString(ifName) { + if c.excludeRegexp != nil && c.excludeRegexp.MatchString(ifName) { continue } From 54465840c7b5bf2842833a23e28e78a16a0e8d32 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:28:14 +0000 Subject: [PATCH 15/77] cleanup --- main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 16bacaf..f899cc2 100644 --- a/main.go +++ b/main.go @@ -42,8 +42,6 @@ var log = logrus.New() var apikey = os.Getenv("MACKEREL_API_KEY") func parseFlags() (*CollectParams, error) { - var community, target, name string - var includeInterface, excludeInterface *string var configFilename string flag.StringVar(&configFilename, "config", "config.yaml", "config `filename`") flag.Parse() @@ -58,6 +56,7 @@ func parseFlags() (*CollectParams, error) { log.Fatalf("error: %v", err) } + var community, target, name string if t.Community == "" { log.Fatal("community is needed.") } @@ -67,16 +66,17 @@ func parseFlags() (*CollectParams, error) { } target = t.Target - if t.Interface != nil { - includeInterface = t.Interface.Include - excludeInterface = t.Interface.Exclude - } - name = t.Name if name == "" { name = target } + var includeInterface, excludeInterface *string + if t.Interface != nil { + includeInterface = t.Interface.Include + excludeInterface = t.Interface.Exclude + } + if includeInterface != nil && excludeInterface != nil { return nil, errors.New("excludeInterface, includeInterface is exclusive control.") } From 131c1a2e5a26c79f675ef864e6ed276eb9c381c6 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:30:44 +0000 Subject: [PATCH 16/77] cleanup --- main.go | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index f899cc2..821cee1 100644 --- a/main.go +++ b/main.go @@ -71,27 +71,22 @@ func parseFlags() (*CollectParams, error) { name = target } - var includeInterface, excludeInterface *string + var includeReg, excludeReg *regexp.Regexp if t.Interface != nil { - includeInterface = t.Interface.Include - excludeInterface = t.Interface.Exclude - } - - if includeInterface != nil && excludeInterface != nil { - return nil, errors.New("excludeInterface, includeInterface is exclusive control.") - } - var includeReg *regexp.Regexp - if includeInterface != nil { - includeReg, err = regexp.Compile(*includeInterface) - if err != nil { - return nil, err + if t.Interface.Include != nil && t.Interface.Exclude != nil { + return nil, errors.New("Interface.Exclude, Interface.Include is exclusive control.") } - } - var excludeReg *regexp.Regexp - if excludeInterface != nil { - excludeReg, err = regexp.Compile(*excludeInterface) - if err != nil { - return nil, err + if t.Interface.Include != nil { + includeReg, err = regexp.Compile(*t.Interface.Include) + if err != nil { + return nil, err + } + } + if t.Interface.Exclude != nil { + excludeReg, err = regexp.Compile(*t.Interface.Exclude) + if err != nil { + return nil, err + } } } From 859f358e4ebdf38fc4e6786289dc667f75bd3935 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:36:01 +0000 Subject: [PATCH 17/77] cleanup --- main.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/main.go b/main.go index 821cee1..53c6b7d 100644 --- a/main.go +++ b/main.go @@ -56,54 +56,49 @@ func parseFlags() (*CollectParams, error) { log.Fatalf("error: %v", err) } - var community, target, name string if t.Community == "" { log.Fatal("community is needed.") } - community = t.Community if t.Target == "" { log.Fatal("target is needed.") } - target = t.Target - name = t.Name + name := t.Name if name == "" { - name = target + name = t.Target + } + + c := &CollectParams{ + target: t.Target, + community: t.Community, + skipDownLinkState: t.SkipLinkdown, + name: name, } - var includeReg, excludeReg *regexp.Regexp if t.Interface != nil { if t.Interface.Include != nil && t.Interface.Exclude != nil { return nil, errors.New("Interface.Exclude, Interface.Include is exclusive control.") } if t.Interface.Include != nil { - includeReg, err = regexp.Compile(*t.Interface.Include) + c.includeRegexp, err = regexp.Compile(*t.Interface.Include) if err != nil { return nil, err } } if t.Interface.Exclude != nil { - excludeReg, err = regexp.Compile(*t.Interface.Exclude) + c.excludeRegexp, err = regexp.Compile(*t.Interface.Exclude) if err != nil { return nil, err } } } - mibs, err := mib.Validate(t.Mibs) + c.mibs, err = mib.Validate(t.Mibs) if err != nil { return nil, err } - return &CollectParams{ - target: target, - name: name, - community: community, - mibs: mibs, - includeRegexp: includeReg, - excludeRegexp: excludeReg, - skipDownLinkState: t.SkipLinkdown, - }, nil + return c, nil } func main() { From 3a736c11a55e188a03042ccaba4307024bd044ee Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:38:08 +0000 Subject: [PATCH 18/77] fix return error --- main.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 53c6b7d..50814f2 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "flag" "fmt" "os" @@ -48,19 +47,19 @@ func parseFlags() (*CollectParams, error) { f, err := os.ReadFile(configFilename) if err != nil { - log.Fatal(err) + return nil, err } var t config.Config err = yaml.Unmarshal(f, &t) if err != nil { - log.Fatalf("error: %v", err) + return nil, err } if t.Community == "" { - log.Fatal("community is needed.") + return nil, fmt.Errorf("community is needed.") } if t.Target == "" { - log.Fatal("target is needed.") + return nil, fmt.Errorf("target is needed.") } name := t.Name @@ -77,7 +76,7 @@ func parseFlags() (*CollectParams, error) { if t.Interface != nil { if t.Interface.Include != nil && t.Interface.Exclude != nil { - return nil, errors.New("Interface.Exclude, Interface.Include is exclusive control.") + return nil, fmt.Errorf("Interface.Exclude, Interface.Include is exclusive control.") } if t.Interface.Include != nil { c.includeRegexp, err = regexp.Compile(*t.Interface.Include) From 63dda7b3c2e32b81ccb1901d5289dc4e886d4bda Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:40:37 +0000 Subject: [PATCH 19/77] parseFlags -> parseConfig --- main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 50814f2..f5ba072 100644 --- a/main.go +++ b/main.go @@ -40,12 +40,8 @@ type CollectParams struct { var log = logrus.New() var apikey = os.Getenv("MACKEREL_API_KEY") -func parseFlags() (*CollectParams, error) { - var configFilename string - flag.StringVar(&configFilename, "config", "config.yaml", "config `filename`") - flag.Parse() - - f, err := os.ReadFile(configFilename) +func parseConfig(filename string) (*CollectParams, error) { + f, err := os.ReadFile(filename) if err != nil { return nil, err } @@ -109,7 +105,11 @@ func main() { FullTimestamp: true, }) - collectParams, err := parseFlags() + var filename string + flag.StringVar(&filename, "config", "config.yaml", "config `filename`") + flag.Parse() + + collectParams, err := parseConfig(filename) if err != nil { log.Fatal(err) } From 143be28af369f2ea24e7a410b640069b0a082c54 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:46:41 +0000 Subject: [PATCH 20/77] rename --- mackerel.go | 8 ++++---- main.go | 37 ++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/mackerel.go b/mackerel.go index d832666..6e90dc9 100644 --- a/mackerel.go +++ b/mackerel.go @@ -123,7 +123,7 @@ func initialForMackerel(c *CollectParams, client *mackerel.Client) (*string, err interfaces := []mackerel.Interface{ mackerel.Interface{ Name: "main", - IPv4Addresses: []string{c.target}, + IPv4Addresses: []string{c.Target}, }, } var hostId string @@ -134,7 +134,7 @@ func initialForMackerel(c *CollectParams, client *mackerel.Client) (*string, err } hostId = string(bytes) _, err = client.UpdateHost(hostId, &mackerel.UpdateHostParam{ - Name: c.name, + Name: c.Name, Interfaces: interfaces, }) if err != nil { @@ -142,7 +142,7 @@ func initialForMackerel(c *CollectParams, client *mackerel.Client) (*string, err } } else { hostId, err = client.CreateHost(&mackerel.CreateHostParam{ - Name: c.name, + Name: c.Name, Interfaces: interfaces, }) if err != nil { @@ -165,7 +165,7 @@ func (c *CollectParams) hostIdPath() (string, error) { if err != nil { return "", err } - return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.target)), nil + return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil } func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectParams *CollectParams) { diff --git a/main.go b/main.go index f5ba072..a8ce12a 100644 --- a/main.go +++ b/main.go @@ -31,10 +31,13 @@ func (m *MetricsDutum) String() string { } type CollectParams struct { - community, target, name string - mibs []string - includeRegexp, excludeRegexp *regexp.Regexp - skipDownLinkState bool + Community string + Target string + Name string + MIBs []string + IncludeRegexp *regexp.Regexp + ExcludeRegexp *regexp.Regexp + SkipDownLinkState bool } var log = logrus.New() @@ -64,10 +67,10 @@ func parseConfig(filename string) (*CollectParams, error) { } c := &CollectParams{ - target: t.Target, - community: t.Community, - skipDownLinkState: t.SkipLinkdown, - name: name, + Target: t.Target, + Community: t.Community, + SkipDownLinkState: t.SkipLinkdown, + Name: name, } if t.Interface != nil { @@ -75,20 +78,20 @@ func parseConfig(filename string) (*CollectParams, error) { return nil, fmt.Errorf("Interface.Exclude, Interface.Include is exclusive control.") } if t.Interface.Include != nil { - c.includeRegexp, err = regexp.Compile(*t.Interface.Include) + c.IncludeRegexp, err = regexp.Compile(*t.Interface.Include) if err != nil { return nil, err } } if t.Interface.Exclude != nil { - c.excludeRegexp, err = regexp.Compile(*t.Interface.Exclude) + c.ExcludeRegexp, err = regexp.Compile(*t.Interface.Exclude) if err != nil { return nil, err } } } - c.mibs, err = mib.Validate(t.Mibs) + c.MIBs, err = mib.Validate(t.Mibs) if err != nil { return nil, err } @@ -134,7 +137,7 @@ func main() { } func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { - snmpClient, err := snmp.Init(ctx, c.target, c.community) + snmpClient, err := snmp.Init(ctx, c.Target, c.Community) if err != nil { return nil, err } @@ -150,7 +153,7 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { } var ifOperStatus map[uint64]bool - if c.skipDownLinkState { + if c.SkipDownLinkState { ifOperStatus, err = snmpClient.BulkWalkGetInterfaceState(ifNumber) if err != nil { return nil, err @@ -159,7 +162,7 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { metrics := make([]MetricsDutum, 0) - for _, mibName := range c.mibs { + for _, mibName := range c.MIBs { values, err := snmpClient.BulkWalk(mib.Oidmapping[mibName], ifNumber) if err != nil { return nil, err @@ -167,16 +170,16 @@ func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { for ifIndex, value := range values { ifName := ifDescr[ifIndex] - if c.includeRegexp != nil && !c.includeRegexp.MatchString(ifName) { + if c.IncludeRegexp != nil && !c.IncludeRegexp.MatchString(ifName) { continue } - if c.excludeRegexp != nil && c.excludeRegexp.MatchString(ifName) { + if c.ExcludeRegexp != nil && c.ExcludeRegexp.MatchString(ifName) { continue } // skip when down(2) - if c.skipDownLinkState && !ifOperStatus[ifIndex] { + if c.SkipDownLinkState && !ifOperStatus[ifIndex] { continue } From f3644a4c6366b3ac910dc0cdd578b75187e86b24 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:48:15 +0000 Subject: [PATCH 21/77] move collectParam -> config.Collector --- config/config.go | 25 +++++++++++++++++++++++++ mackerel.go | 20 ++++++-------------- main.go | 16 +++------------- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/config/config.go b/config/config.go index 435f7a8..cb36cfe 100644 --- a/config/config.go +++ b/config/config.go @@ -1,5 +1,12 @@ package config +import ( + "fmt" + "os" + "path/filepath" + "regexp" +) + type Config struct { Community string `yaml:"community"` Target string `yaml:"target"` @@ -19,3 +26,21 @@ type Mackerel struct { HostID string `yaml:"host-id"` ApiKey string `yaml:"x-api-key"` } + +type Collector struct { + Community string + Target string + Name string + MIBs []string + IncludeRegexp *regexp.Regexp + ExcludeRegexp *regexp.Regexp + SkipDownLinkState bool +} + +func (c *Collector) HostIdPath() (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil +} diff --git a/mackerel.go b/mackerel.go index 6e90dc9..4531852 100644 --- a/mackerel.go +++ b/mackerel.go @@ -6,12 +6,12 @@ import ( "fmt" "math" "os" - "path/filepath" "strings" "sync" "time" mackerel "github.com/mackerelio/mackerel-client-go" + "github.com/yseto/switch-traffic-to-mackerel/config" ) var buffers = list.New() @@ -90,7 +90,7 @@ var deltaValues = map[string]bool{ "ifHCOutOctets": true, } -func runMackerel(ctx context.Context, collectParams *CollectParams) { +func runMackerel(ctx context.Context, collectParams *config.Collector) { client := mackerel.NewClient(apikey) hostId, err := initialForMackerel(collectParams, client) @@ -113,10 +113,10 @@ func runMackerel(ctx context.Context, collectParams *CollectParams) { wg.Wait() } -func initialForMackerel(c *CollectParams, client *mackerel.Client) (*string, error) { +func initialForMackerel(c *config.Collector, client *mackerel.Client) (*string, error) { log.Info("init for mackerel") - idPath, err := c.hostIdPath() + idPath, err := c.HostIdPath() if err != nil { return nil, err } @@ -160,15 +160,7 @@ func initialForMackerel(c *CollectParams, client *mackerel.Client) (*string, err return &hostId, nil } -func (c *CollectParams) hostIdPath() (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil -} - -func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectParams *CollectParams) { +func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectParams *config.Collector) { t := time.NewTicker(1 * time.Minute) defer func() { t.Stop() @@ -201,7 +193,7 @@ func calcurateDiff(a, b, overflow uint64) uint64 { } } -func innerTicker(ctx context.Context, hostId *string, collectParams *CollectParams) error { +func innerTicker(ctx context.Context, hostId *string, collectParams *config.Collector) error { rawMetrics, err := collect(ctx, collectParams) if err != nil { return err diff --git a/main.go b/main.go index a8ce12a..ec102b3 100644 --- a/main.go +++ b/main.go @@ -30,20 +30,10 @@ func (m *MetricsDutum) String() string { return fmt.Sprintf("%d\t%s\t%s\t%d", m.IfIndex, m.IfName, m.Mib, m.Value) } -type CollectParams struct { - Community string - Target string - Name string - MIBs []string - IncludeRegexp *regexp.Regexp - ExcludeRegexp *regexp.Regexp - SkipDownLinkState bool -} - var log = logrus.New() var apikey = os.Getenv("MACKEREL_API_KEY") -func parseConfig(filename string) (*CollectParams, error) { +func parseConfig(filename string) (*config.Collector, error) { f, err := os.ReadFile(filename) if err != nil { return nil, err @@ -66,7 +56,7 @@ func parseConfig(filename string) (*CollectParams, error) { name = t.Target } - c := &CollectParams{ + c := &config.Collector{ Target: t.Target, Community: t.Community, SkipDownLinkState: t.SkipLinkdown, @@ -136,7 +126,7 @@ func main() { } } -func collect(ctx context.Context, c *CollectParams) ([]MetricsDutum, error) { +func collect(ctx context.Context, c *config.Collector) ([]MetricsDutum, error) { snmpClient, err := snmp.Init(ctx, c.Target, c.Community) if err != nil { return nil, err From 0d3eddae58bd30ff40bf0838f71d235fbe867210 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:50:18 +0000 Subject: [PATCH 22/77] config.Parse --- config/config.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 60 +----------------------------------------------- 2 files changed, 61 insertions(+), 59 deletions(-) diff --git a/config/config.go b/config/config.go index cb36cfe..f6ecd62 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,10 @@ import ( "os" "path/filepath" "regexp" + + "gopkg.in/yaml.v3" + + "github.com/yseto/switch-traffic-to-mackerel/mib" ) type Config struct { @@ -44,3 +48,59 @@ func (c *Collector) HostIdPath() (string, error) { } return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil } + +func Parse(filename string) (*Collector, error) { + f, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + var t Config + err = yaml.Unmarshal(f, &t) + if err != nil { + return nil, err + } + + if t.Community == "" { + return nil, fmt.Errorf("community is needed.") + } + if t.Target == "" { + return nil, fmt.Errorf("target is needed.") + } + + name := t.Name + if name == "" { + name = t.Target + } + + c := &Collector{ + Target: t.Target, + Community: t.Community, + SkipDownLinkState: t.SkipLinkdown, + Name: name, + } + + if t.Interface != nil { + if t.Interface.Include != nil && t.Interface.Exclude != nil { + return nil, fmt.Errorf("Interface.Exclude, Interface.Include is exclusive control.") + } + if t.Interface.Include != nil { + c.IncludeRegexp, err = regexp.Compile(*t.Interface.Include) + if err != nil { + return nil, err + } + } + if t.Interface.Exclude != nil { + c.ExcludeRegexp, err = regexp.Compile(*t.Interface.Exclude) + if err != nil { + return nil, err + } + } + } + + c.MIBs, err = mib.Validate(t.Mibs) + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/main.go b/main.go index ec102b3..a7ea1b4 100644 --- a/main.go +++ b/main.go @@ -6,13 +6,11 @@ import ( "fmt" "os" "os/signal" - "regexp" "sort" "strings" "github.com/maruel/natural" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" "github.com/yseto/switch-traffic-to-mackerel/config" "github.com/yseto/switch-traffic-to-mackerel/mib" @@ -33,62 +31,6 @@ func (m *MetricsDutum) String() string { var log = logrus.New() var apikey = os.Getenv("MACKEREL_API_KEY") -func parseConfig(filename string) (*config.Collector, error) { - f, err := os.ReadFile(filename) - if err != nil { - return nil, err - } - var t config.Config - err = yaml.Unmarshal(f, &t) - if err != nil { - return nil, err - } - - if t.Community == "" { - return nil, fmt.Errorf("community is needed.") - } - if t.Target == "" { - return nil, fmt.Errorf("target is needed.") - } - - name := t.Name - if name == "" { - name = t.Target - } - - c := &config.Collector{ - Target: t.Target, - Community: t.Community, - SkipDownLinkState: t.SkipLinkdown, - Name: name, - } - - if t.Interface != nil { - if t.Interface.Include != nil && t.Interface.Exclude != nil { - return nil, fmt.Errorf("Interface.Exclude, Interface.Include is exclusive control.") - } - if t.Interface.Include != nil { - c.IncludeRegexp, err = regexp.Compile(*t.Interface.Include) - if err != nil { - return nil, err - } - } - if t.Interface.Exclude != nil { - c.ExcludeRegexp, err = regexp.Compile(*t.Interface.Exclude) - if err != nil { - return nil, err - } - } - } - - c.MIBs, err = mib.Validate(t.Mibs) - if err != nil { - return nil, err - } - - return c, nil -} - func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() @@ -102,7 +44,7 @@ func main() { flag.StringVar(&filename, "config", "config.yaml", "config `filename`") flag.Parse() - collectParams, err := parseConfig(filename) + collectParams, err := config.Parse(filename) if err != nil { log.Fatal(err) } From 693327747c6235e9db831e5bce6f84ac0425001f Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 17:56:38 +0000 Subject: [PATCH 23/77] create collector package --- collector/collector.go | 62 +++++++++++++++++++++++++++++++++++++ collector/types.go | 14 +++++++++ mackerel.go | 8 +++-- main.go | 70 ++---------------------------------------- 4 files changed, 83 insertions(+), 71 deletions(-) create mode 100644 collector/collector.go create mode 100644 collector/types.go diff --git a/collector/collector.go b/collector/collector.go new file mode 100644 index 0000000..f3259aa --- /dev/null +++ b/collector/collector.go @@ -0,0 +1,62 @@ +package collector + +import ( + "context" + + "github.com/yseto/switch-traffic-to-mackerel/config" + "github.com/yseto/switch-traffic-to-mackerel/mib" + "github.com/yseto/switch-traffic-to-mackerel/snmp" +) + +func Do(ctx context.Context, c *config.Collector) ([]MetricsDutum, error) { + snmpClient, err := snmp.Init(ctx, c.Target, c.Community) + if err != nil { + return nil, err + } + defer snmpClient.Close() + + ifNumber, err := snmpClient.GetInterfaceNumber() + if err != nil { + return nil, err + } + ifDescr, err := snmpClient.BulkWalkGetInterfaceName(ifNumber) + if err != nil { + return nil, err + } + + var ifOperStatus map[uint64]bool + if c.SkipDownLinkState { + ifOperStatus, err = snmpClient.BulkWalkGetInterfaceState(ifNumber) + if err != nil { + return nil, err + } + } + + metrics := make([]MetricsDutum, 0) + + for _, mibName := range c.MIBs { + values, err := snmpClient.BulkWalk(mib.Oidmapping[mibName], ifNumber) + if err != nil { + return nil, err + } + + for ifIndex, value := range values { + ifName := ifDescr[ifIndex] + if c.IncludeRegexp != nil && !c.IncludeRegexp.MatchString(ifName) { + continue + } + + if c.ExcludeRegexp != nil && c.ExcludeRegexp.MatchString(ifName) { + continue + } + + // skip when down(2) + if c.SkipDownLinkState && !ifOperStatus[ifIndex] { + continue + } + + metrics = append(metrics, MetricsDutum{IfIndex: ifIndex, Mib: mibName, IfName: ifName, Value: value}) + } + } + return metrics, nil +} diff --git a/collector/types.go b/collector/types.go new file mode 100644 index 0000000..0f3bb8f --- /dev/null +++ b/collector/types.go @@ -0,0 +1,14 @@ +package collector + +import "fmt" + +type MetricsDutum struct { + IfIndex uint64 `json:"ifIndex"` + Mib string `json:"mib"` + IfName string `json:"ifName"` + Value uint64 `json:"value"` +} + +func (m *MetricsDutum) String() string { + return fmt.Sprintf("%d\t%s\t%s\t%d", m.IfIndex, m.IfName, m.Mib, m.Value) +} diff --git a/mackerel.go b/mackerel.go index 4531852..c190349 100644 --- a/mackerel.go +++ b/mackerel.go @@ -11,12 +11,14 @@ import ( "time" mackerel "github.com/mackerelio/mackerel-client-go" + + "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" ) var buffers = list.New() var mutex = &sync.Mutex{} -var snapshot []MetricsDutum +var snapshot []collector.MetricsDutum var graphDefs = []*mackerel.GraphDefsParam{ &mackerel.GraphDefsParam{ @@ -98,7 +100,7 @@ func runMackerel(ctx context.Context, collectParams *config.Collector) { log.Fatal(err) } - snapshot, err = collect(ctx, collectParams) + snapshot, err = collector.Do(ctx, collectParams) if err != nil { log.Fatal(err) } @@ -194,7 +196,7 @@ func calcurateDiff(a, b, overflow uint64) uint64 { } func innerTicker(ctx context.Context, hostId *string, collectParams *config.Collector) error { - rawMetrics, err := collect(ctx, collectParams) + rawMetrics, err := collector.Do(ctx, collectParams) if err != nil { return err } diff --git a/main.go b/main.go index a7ea1b4..b1d989e 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "context" "flag" - "fmt" "os" "os/signal" "sort" @@ -12,22 +11,10 @@ import ( "github.com/maruel/natural" "github.com/sirupsen/logrus" + "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" - "github.com/yseto/switch-traffic-to-mackerel/mib" - "github.com/yseto/switch-traffic-to-mackerel/snmp" ) -type MetricsDutum struct { - IfIndex uint64 `json:"ifIndex"` - Mib string `json:"mib"` - IfName string `json:"ifName"` - Value uint64 `json:"value"` -} - -func (m *MetricsDutum) String() string { - return fmt.Sprintf("%d\t%s\t%s\t%d", m.IfIndex, m.IfName, m.Mib, m.Value) -} - var log = logrus.New() var apikey = os.Getenv("MACKEREL_API_KEY") @@ -52,7 +39,7 @@ func main() { log.Info("start") if apikey == "" { - dutum, err := collect(ctx, collectParams) + dutum, err := collector.Do(ctx, collectParams) if err != nil { log.Fatal(err) } @@ -67,56 +54,3 @@ func main() { runMackerel(ctx, collectParams) } } - -func collect(ctx context.Context, c *config.Collector) ([]MetricsDutum, error) { - snmpClient, err := snmp.Init(ctx, c.Target, c.Community) - if err != nil { - return nil, err - } - defer snmpClient.Close() - - ifNumber, err := snmpClient.GetInterfaceNumber() - if err != nil { - return nil, err - } - ifDescr, err := snmpClient.BulkWalkGetInterfaceName(ifNumber) - if err != nil { - return nil, err - } - - var ifOperStatus map[uint64]bool - if c.SkipDownLinkState { - ifOperStatus, err = snmpClient.BulkWalkGetInterfaceState(ifNumber) - if err != nil { - return nil, err - } - } - - metrics := make([]MetricsDutum, 0) - - for _, mibName := range c.MIBs { - values, err := snmpClient.BulkWalk(mib.Oidmapping[mibName], ifNumber) - if err != nil { - return nil, err - } - - for ifIndex, value := range values { - ifName := ifDescr[ifIndex] - if c.IncludeRegexp != nil && !c.IncludeRegexp.MatchString(ifName) { - continue - } - - if c.ExcludeRegexp != nil && c.ExcludeRegexp.MatchString(ifName) { - continue - } - - // skip when down(2) - if c.SkipDownLinkState && !ifOperStatus[ifIndex] { - continue - } - - metrics = append(metrics, MetricsDutum{IfIndex: ifIndex, Mib: mibName, IfName: ifName, Value: value}) - } - } - return metrics, nil -} From 0ebf44fb14b06a43195b39160f48c6bc462287df Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 18:07:44 +0000 Subject: [PATCH 24/77] move debugPrint to collector --- collector/collector.go | 14 ++++++++++++++ main.go | 12 +----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index f3259aa..c2d1511 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -2,6 +2,11 @@ package collector import ( "context" + "fmt" + "sort" + "strings" + + "github.com/maruel/natural" "github.com/yseto/switch-traffic-to-mackerel/config" "github.com/yseto/switch-traffic-to-mackerel/mib" @@ -60,3 +65,12 @@ func Do(ctx context.Context, c *config.Collector) ([]MetricsDutum, error) { } return metrics, nil } + +func debugPrint(dutum []MetricsDutum) { + var dutumStr []string + for i := range dutum { + dutumStr = append(dutumStr, dutum[i].String()) + } + sort.Sort(natural.StringSlice(dutumStr)) + fmt.Println(strings.Join(dutumStr, "\n")) +} diff --git a/main.go b/main.go index b1d989e..732fd23 100644 --- a/main.go +++ b/main.go @@ -5,10 +5,7 @@ import ( "flag" "os" "os/signal" - "sort" - "strings" - "github.com/maruel/natural" "github.com/sirupsen/logrus" "github.com/yseto/switch-traffic-to-mackerel/collector" @@ -39,17 +36,10 @@ func main() { log.Info("start") if apikey == "" { - dutum, err := collector.Do(ctx, collectParams) + _, err := collector.Do(ctx, collectParams) if err != nil { log.Fatal(err) } - - var dutumStr []string - for i := range dutum { - dutumStr = append(dutumStr, dutum[i].String()) - } - sort.Sort(natural.StringSlice(dutumStr)) - fmt.Println(strings.Join(dutumStr, "\n")) } else { runMackerel(ctx, collectParams) } From 053ff09a82f7ac3a4759ce6fb9cf12e0c525679e Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 18:08:25 +0000 Subject: [PATCH 25/77] support debug option --- collector/collector.go | 3 +++ config/config.go | 3 +++ main.go | 3 +++ 3 files changed, 9 insertions(+) diff --git a/collector/collector.go b/collector/collector.go index c2d1511..b5e2f35 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -63,6 +63,9 @@ func Do(ctx context.Context, c *config.Collector) ([]MetricsDutum, error) { metrics = append(metrics, MetricsDutum{IfIndex: ifIndex, Mib: mibName, IfName: ifName, Value: value}) } } + if c.Debug { + debugPrint(metrics) + } return metrics, nil } diff --git a/config/config.go b/config/config.go index f6ecd62..591685f 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ type Config struct { SkipLinkdown bool `yaml:"skip-linkdown"` Name string `yaml:"name"` Mackerel *Mackerel `yaml:"mackerel"` + Debug bool `yaml:"debug"` } type Interface struct { @@ -39,6 +40,7 @@ type Collector struct { IncludeRegexp *regexp.Regexp ExcludeRegexp *regexp.Regexp SkipDownLinkState bool + Debug bool } func (c *Collector) HostIdPath() (string, error) { @@ -77,6 +79,7 @@ func Parse(filename string) (*Collector, error) { Community: t.Community, SkipDownLinkState: t.SkipLinkdown, Name: name, + Debug: t.Debug, } if t.Interface != nil { diff --git a/main.go b/main.go index 732fd23..b78f026 100644 --- a/main.go +++ b/main.go @@ -25,13 +25,16 @@ func main() { }) var filename string + var debug bool flag.StringVar(&filename, "config", "config.yaml", "config `filename`") + flag.BoolVar(&debug, "debug", false, "debug") flag.Parse() collectParams, err := config.Parse(filename) if err != nil { log.Fatal(err) } + collectParams.Debug = (collectParams.Debug || debug) log.Info("start") From ba883bb21d00be71932830bcfcdaf68897351165 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 18:12:11 +0000 Subject: [PATCH 26/77] replace logger --- go.mod | 2 +- go.sum | 8 -------- mackerel.go | 13 +++++++------ main.go | 11 ++--------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 0ebd0ff..f77883c 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( github.com/gosnmp/gosnmp v1.35.0 github.com/mackerelio/mackerel-client-go v0.23.0 github.com/maruel/natural v1.1.0 - github.com/sirupsen/logrus v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.2.0 // indirect golang.org/x/tools v0.3.0 // indirect diff --git a/go.sum b/go.sum index 852d214..c31e9c3 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gosnmp/gosnmp v1.35.0 h1:EuWWNPxTCdAUx2/NbQcSa3WdNxjzpy4Phv57b4MWpJM= @@ -9,11 +8,6 @@ github.com/mackerelio/mackerel-client-go v0.23.0/go.mod h1:VM9KAjzs7wkROQ9WdZRvk github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -27,7 +21,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -37,6 +30,5 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mackerel.go b/mackerel.go index c190349..583c238 100644 --- a/mackerel.go +++ b/mackerel.go @@ -4,6 +4,7 @@ import ( "container/list" "context" "fmt" + "log" "math" "os" "strings" @@ -116,7 +117,7 @@ func runMackerel(ctx context.Context, collectParams *config.Collector) { } func initialForMackerel(c *config.Collector, client *mackerel.Client) (*string, error) { - log.Info("init for mackerel") + log.Println("init for mackerel") idPath, err := c.HostIdPath() if err != nil { @@ -174,10 +175,10 @@ func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectPara case <-t.C: err := innerTicker(ctx, hostId, collectParams) if err != nil { - log.Warn(err) + log.Println(err.Error()) } case <-ctx.Done(): - log.Warn("cancellation from context:", ctx.Err()) + log.Println("cancellation from context:", ctx.Err()) return } } @@ -259,7 +260,7 @@ func sendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client sendToMackerel(ctx, client, hostId) case <-ctx.Done(): - log.Warn("cancellation from context:", ctx.Err()) + log.Println("cancellation from context:", ctx.Err()) return } } @@ -276,10 +277,10 @@ func sendToMackerel(ctx context.Context, client *mackerel.Client, hostId *string err := client.PostHostMetricValuesByHostID(*hostId, e.Value.([](*mackerel.MetricValue))) if err != nil { - log.Warn(err) + log.Println(err) return } else { - log.Info("success") + log.Println("success") } mutex.Lock() buffers.Remove(e) diff --git a/main.go b/main.go index b78f026..b5112dc 100644 --- a/main.go +++ b/main.go @@ -3,27 +3,20 @@ package main import ( "context" "flag" + "log" "os" "os/signal" - "github.com/sirupsen/logrus" - "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" ) -var log = logrus.New() var apikey = os.Getenv("MACKEREL_API_KEY") func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() - log.SetFormatter(&logrus.TextFormatter{ - DisableColors: true, - FullTimestamp: true, - }) - var filename string var debug bool flag.StringVar(&filename, "config", "config.yaml", "config `filename`") @@ -36,7 +29,7 @@ func main() { } collectParams.Debug = (collectParams.Debug || debug) - log.Info("start") + log.Println("start") if apikey == "" { _, err := collector.Do(ctx, collectParams) From e92ab36a7af6d7254585fee4478f484439c9a69f Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 18:20:08 +0000 Subject: [PATCH 27/77] for mackerel --- config/config.go | 5 +++++ mackerel.go | 2 +- main.go | 4 +--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 591685f..06dba83 100644 --- a/config/config.go +++ b/config/config.go @@ -41,6 +41,7 @@ type Collector struct { ExcludeRegexp *regexp.Regexp SkipDownLinkState bool Debug bool + Mackerel *Mackerel } func (c *Collector) HostIdPath() (string, error) { @@ -105,5 +106,9 @@ func Parse(filename string) (*Collector, error) { return nil, err } + if t.Mackerel != nil { + c.Mackerel = t.Mackerel + } + return c, nil } diff --git a/mackerel.go b/mackerel.go index 583c238..35f04ab 100644 --- a/mackerel.go +++ b/mackerel.go @@ -94,7 +94,7 @@ var deltaValues = map[string]bool{ } func runMackerel(ctx context.Context, collectParams *config.Collector) { - client := mackerel.NewClient(apikey) + client := mackerel.NewClient(collectParams.Mackerel.ApiKey) hostId, err := initialForMackerel(collectParams, client) if err != nil { diff --git a/main.go b/main.go index b5112dc..47583dd 100644 --- a/main.go +++ b/main.go @@ -11,8 +11,6 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/config" ) -var apikey = os.Getenv("MACKEREL_API_KEY") - func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() @@ -31,7 +29,7 @@ func main() { log.Println("start") - if apikey == "" { + if collectParams.Mackerel == nil { _, err := collector.Do(ctx, collectParams) if err != nil { log.Fatal(err) From f499c190284057b6f5f9500ee6c61a7d4190b499 Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 18:26:35 +0000 Subject: [PATCH 28/77] create mackerel package, move graphdef --- mackerel.go | 52 +++----------------------------------------- mackerel/graphdef.go | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 49 deletions(-) create mode 100644 mackerel/graphdef.go diff --git a/mackerel.go b/mackerel.go index 35f04ab..acad36d 100644 --- a/mackerel.go +++ b/mackerel.go @@ -15,59 +15,13 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" + mckr "github.com/yseto/switch-traffic-to-mackerel/mackerel" ) var buffers = list.New() var mutex = &sync.Mutex{} var snapshot []collector.MetricsDutum -var graphDefs = []*mackerel.GraphDefsParam{ - &mackerel.GraphDefsParam{ - Name: "custom.interface.ifInDiscards", - Unit: "integer", - DisplayName: "In Discards", - Metrics: []*mackerel.GraphDefsMetric{ - &mackerel.GraphDefsMetric{ - Name: "custom.interface.ifInDiscards.*", - DisplayName: "%1", - }, - }, - }, - &mackerel.GraphDefsParam{ - Name: "custom.interface.ifOutDiscards", - Unit: "integer", - DisplayName: "Out Discards", - Metrics: []*mackerel.GraphDefsMetric{ - &mackerel.GraphDefsMetric{ - Name: "custom.interface.ifOutDiscards.*", - DisplayName: "%1", - }, - }, - }, - &mackerel.GraphDefsParam{ - Name: "custom.interface.ifInErrors", - Unit: "integer", - DisplayName: "In Errors", - Metrics: []*mackerel.GraphDefsMetric{ - &mackerel.GraphDefsMetric{ - Name: "custom.interface.ifInErrors.*", - DisplayName: "%1", - }, - }, - }, - &mackerel.GraphDefsParam{ - Name: "custom.interface.ifOutErrors", - Unit: "integer", - DisplayName: "Out Errors", - Metrics: []*mackerel.GraphDefsMetric{ - &mackerel.GraphDefsMetric{ - Name: "custom.interface.ifOutErrors.*", - DisplayName: "%1", - }, - }, - }, -} - var overflowValue = map[string]uint64{ "ifInOctets": math.MaxUint32, "ifOutOctets": math.MaxUint32, @@ -124,7 +78,7 @@ func initialForMackerel(c *config.Collector, client *mackerel.Client) (*string, return nil, err } interfaces := []mackerel.Interface{ - mackerel.Interface{ + { Name: "main", IPv4Addresses: []string{c.Target}, }, @@ -156,7 +110,7 @@ func initialForMackerel(c *config.Collector, client *mackerel.Client) (*string, return nil, err } } - err = client.CreateGraphDefs(graphDefs) + err = client.CreateGraphDefs(mckr.GraphDefs) if err != nil { return nil, err } diff --git a/mackerel/graphdef.go b/mackerel/graphdef.go new file mode 100644 index 0000000..85aaf93 --- /dev/null +++ b/mackerel/graphdef.go @@ -0,0 +1,50 @@ +package mackerel + +import "github.com/mackerelio/mackerel-client-go" + +var GraphDefs = []*mackerel.GraphDefsParam{ + { + Name: "custom.interface.ifInDiscards", + Unit: "integer", + DisplayName: "In Discards", + Metrics: []*mackerel.GraphDefsMetric{ + { + Name: "custom.interface.ifInDiscards.*", + DisplayName: "%1", + }, + }, + }, + { + Name: "custom.interface.ifOutDiscards", + Unit: "integer", + DisplayName: "Out Discards", + Metrics: []*mackerel.GraphDefsMetric{ + { + Name: "custom.interface.ifOutDiscards.*", + DisplayName: "%1", + }, + }, + }, + { + Name: "custom.interface.ifInErrors", + Unit: "integer", + DisplayName: "In Errors", + Metrics: []*mackerel.GraphDefsMetric{ + { + Name: "custom.interface.ifInErrors.*", + DisplayName: "%1", + }, + }, + }, + { + Name: "custom.interface.ifOutErrors", + Unit: "integer", + DisplayName: "Out Errors", + Metrics: []*mackerel.GraphDefsMetric{ + { + Name: "custom.interface.ifOutErrors.*", + DisplayName: "%1", + }, + }, + }, +} From 945c486bc02ef251b606b8b692a30e6c412a3c6b Mon Sep 17 00:00:00 2001 From: yseto Date: Tue, 14 Mar 2023 18:29:16 +0000 Subject: [PATCH 29/77] config.Parse rename Init --- config/config.go | 5 ++++- main.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 06dba83..5eb3a19 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,8 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/mib" ) +var loadedFilename string + type Config struct { Community string `yaml:"community"` Target string `yaml:"target"` @@ -52,7 +54,8 @@ func (c *Collector) HostIdPath() (string, error) { return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil } -func Parse(filename string) (*Collector, error) { +func Init(filename string) (*Collector, error) { + loadedFilename = filename f, err := os.ReadFile(filename) if err != nil { return nil, err diff --git a/main.go b/main.go index 47583dd..8dc0d82 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ func main() { flag.BoolVar(&debug, "debug", false, "debug") flag.Parse() - collectParams, err := config.Parse(filename) + collectParams, err := config.Init(filename) if err != nil { log.Fatal(err) } From e6f1277067318d195eb2b7a3c4b87b79d110fefa Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 11:26:39 +0000 Subject: [PATCH 30/77] rename --- collector/collector.go | 2 +- config/config.go | 12 ++++++------ mackerel.go | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index b5e2f35..884ab52 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -13,7 +13,7 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/snmp" ) -func Do(ctx context.Context, c *config.Collector) ([]MetricsDutum, error) { +func Do(ctx context.Context, c *config.Config) ([]MetricsDutum, error) { snmpClient, err := snmp.Init(ctx, c.Target, c.Community) if err != nil { return nil, err diff --git a/config/config.go b/config/config.go index 5eb3a19..c90711e 100644 --- a/config/config.go +++ b/config/config.go @@ -13,7 +13,7 @@ import ( var loadedFilename string -type Config struct { +type YAMLConfig struct { Community string `yaml:"community"` Target string `yaml:"target"` Interface *Interface `yaml:"interface"` @@ -34,7 +34,7 @@ type Mackerel struct { ApiKey string `yaml:"x-api-key"` } -type Collector struct { +type Config struct { Community string Target string Name string @@ -46,7 +46,7 @@ type Collector struct { Mackerel *Mackerel } -func (c *Collector) HostIdPath() (string, error) { +func (c *Config) HostIdPath() (string, error) { wd, err := os.Getwd() if err != nil { return "", err @@ -54,13 +54,13 @@ func (c *Collector) HostIdPath() (string, error) { return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil } -func Init(filename string) (*Collector, error) { +func Init(filename string) (*Config, error) { loadedFilename = filename f, err := os.ReadFile(filename) if err != nil { return nil, err } - var t Config + var t YAMLConfig err = yaml.Unmarshal(f, &t) if err != nil { return nil, err @@ -78,7 +78,7 @@ func Init(filename string) (*Collector, error) { name = t.Target } - c := &Collector{ + c := &Config{ Target: t.Target, Community: t.Community, SkipDownLinkState: t.SkipLinkdown, diff --git a/mackerel.go b/mackerel.go index acad36d..7c9530e 100644 --- a/mackerel.go +++ b/mackerel.go @@ -47,7 +47,7 @@ var deltaValues = map[string]bool{ "ifHCOutOctets": true, } -func runMackerel(ctx context.Context, collectParams *config.Collector) { +func runMackerel(ctx context.Context, collectParams *config.Config) { client := mackerel.NewClient(collectParams.Mackerel.ApiKey) hostId, err := initialForMackerel(collectParams, client) @@ -70,7 +70,7 @@ func runMackerel(ctx context.Context, collectParams *config.Collector) { wg.Wait() } -func initialForMackerel(c *config.Collector, client *mackerel.Client) (*string, error) { +func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { log.Println("init for mackerel") idPath, err := c.HostIdPath() @@ -117,7 +117,7 @@ func initialForMackerel(c *config.Collector, client *mackerel.Client) (*string, return &hostId, nil } -func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectParams *config.Collector) { +func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectParams *config.Config) { t := time.NewTicker(1 * time.Minute) defer func() { t.Stop() @@ -150,7 +150,7 @@ func calcurateDiff(a, b, overflow uint64) uint64 { } } -func innerTicker(ctx context.Context, hostId *string, collectParams *config.Collector) error { +func innerTicker(ctx context.Context, hostId *string, collectParams *config.Config) error { rawMetrics, err := collector.Do(ctx, collectParams) if err != nil { return err From 20af4833f18cceca3f01c36b0b902ea113d6797d Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 11:30:27 +0000 Subject: [PATCH 31/77] remove unused argument. --- mackerel.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mackerel.go b/mackerel.go index 7c9530e..f0cdfdb 100644 --- a/mackerel.go +++ b/mackerel.go @@ -63,7 +63,7 @@ func runMackerel(ctx context.Context, collectParams *config.Config) { wg := sync.WaitGroup{} wg.Add(1) - go ticker(ctx, &wg, hostId, collectParams) + go ticker(ctx, &wg, collectParams) wg.Add(1) go sendTicker(ctx, &wg, client, hostId) @@ -117,7 +117,7 @@ func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return &hostId, nil } -func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectParams *config.Config) { +func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config) { t := time.NewTicker(1 * time.Minute) defer func() { t.Stop() @@ -127,7 +127,7 @@ func ticker(ctx context.Context, wg *sync.WaitGroup, hostId *string, collectPara for { select { case <-t.C: - err := innerTicker(ctx, hostId, collectParams) + err := innerTicker(ctx, collectParams) if err != nil { log.Println(err.Error()) } @@ -150,7 +150,7 @@ func calcurateDiff(a, b, overflow uint64) uint64 { } } -func innerTicker(ctx context.Context, hostId *string, collectParams *config.Config) error { +func innerTicker(ctx context.Context, collectParams *config.Config) error { rawMetrics, err := collector.Do(ctx, collectParams) if err != nil { return err From 823f5b9a6bb5843c11154062250cb5d722d9e61c Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 11:34:27 +0000 Subject: [PATCH 32/77] dry-run --- config/config.go | 3 +++ main.go | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index c90711e..bd5f372 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,7 @@ type YAMLConfig struct { Name string `yaml:"name"` Mackerel *Mackerel `yaml:"mackerel"` Debug bool `yaml:"debug"` + DryRun bool `yaml:"dry-run"` } type Interface struct { @@ -43,6 +44,7 @@ type Config struct { ExcludeRegexp *regexp.Regexp SkipDownLinkState bool Debug bool + DryRun bool Mackerel *Mackerel } @@ -84,6 +86,7 @@ func Init(filename string) (*Config, error) { SkipDownLinkState: t.SkipLinkdown, Name: name, Debug: t.Debug, + DryRun: t.DryRun, } if t.Interface != nil { diff --git a/main.go b/main.go index 8dc0d82..091e288 100644 --- a/main.go +++ b/main.go @@ -16,9 +16,10 @@ func main() { defer stop() var filename string - var debug bool + var debug, dryrun bool flag.StringVar(&filename, "config", "config.yaml", "config `filename`") flag.BoolVar(&debug, "debug", false, "debug") + flag.BoolVar(&dryrun, "dry-run", false, "dry run") flag.Parse() collectParams, err := config.Init(filename) @@ -26,6 +27,7 @@ func main() { log.Fatal(err) } collectParams.Debug = (collectParams.Debug || debug) + collectParams.DryRun = (collectParams.DryRun || dryrun) log.Println("start") From 8593f87787f88cf31f3299ef822848c6cbb23bd9 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 11:34:52 +0000 Subject: [PATCH 33/77] return error --- mackerel.go | 21 ++++++++++++--------- main.go | 5 ++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mackerel.go b/mackerel.go index f0cdfdb..c116144 100644 --- a/mackerel.go +++ b/mackerel.go @@ -47,17 +47,11 @@ var deltaValues = map[string]bool{ "ifHCOutOctets": true, } -func runMackerel(ctx context.Context, collectParams *config.Config) { - client := mackerel.NewClient(collectParams.Mackerel.ApiKey) - - hostId, err := initialForMackerel(collectParams, client) - if err != nil { - log.Fatal(err) - } - +func runMackerel(ctx context.Context, collectParams *config.Config) error { + var err error snapshot, err = collector.Do(ctx, collectParams) if err != nil { - log.Fatal(err) + return err } wg := sync.WaitGroup{} @@ -65,9 +59,18 @@ func runMackerel(ctx context.Context, collectParams *config.Config) { wg.Add(1) go ticker(ctx, &wg, collectParams) + client := mackerel.NewClient(collectParams.Mackerel.ApiKey) + + hostId, err := initialForMackerel(collectParams, client) + if err != nil { + return err + } + wg.Add(1) go sendTicker(ctx, &wg, client, hostId) wg.Wait() + + return nil } func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { diff --git a/main.go b/main.go index 091e288..efac8ef 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,9 @@ func main() { log.Fatal(err) } } else { - runMackerel(ctx, collectParams) + err = runMackerel(ctx, collectParams) + if err != nil { + log.Fatal(err) + } } } From 77f9ff6b5850bb4ed6f4c15f84c12a02923c7e42 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 11:47:59 +0000 Subject: [PATCH 34/77] added date time, clear terminal --- collector/collector.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/collector/collector.go b/collector/collector.go index 884ab52..c522ddc 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/maruel/natural" @@ -75,5 +76,8 @@ func debugPrint(dutum []MetricsDutum) { dutumStr = append(dutumStr, dutum[i].String()) } sort.Sort(natural.StringSlice(dutumStr)) + // debug print. + fmt.Print("\033[H\033[2J") + fmt.Println(time.Now().Format(time.ANSIC)) fmt.Println(strings.Join(dutumStr, "\n")) } From 47d25acda184ac82d209901b52af88f0bcfbb954 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 11:48:19 +0000 Subject: [PATCH 35/77] no send metrics when dryrun --- mackerel.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mackerel.go b/mackerel.go index c116144..b511652 100644 --- a/mackerel.go +++ b/mackerel.go @@ -59,6 +59,11 @@ func runMackerel(ctx context.Context, collectParams *config.Config) error { wg.Add(1) go ticker(ctx, &wg, collectParams) + if collectParams.DryRun { + wg.Wait() + return nil + } + client := mackerel.NewClient(collectParams.Mackerel.ApiKey) hostId, err := initialForMackerel(collectParams, client) From 579fa0da81bd12286cb4f6fcca4e41cfbee7d6c7 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:09:20 +0000 Subject: [PATCH 36/77] move func. --- mackerel.go | 31 ------------------------------- main.go | 48 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/mackerel.go b/mackerel.go index b511652..f5f8eea 100644 --- a/mackerel.go +++ b/mackerel.go @@ -47,37 +47,6 @@ var deltaValues = map[string]bool{ "ifHCOutOctets": true, } -func runMackerel(ctx context.Context, collectParams *config.Config) error { - var err error - snapshot, err = collector.Do(ctx, collectParams) - if err != nil { - return err - } - - wg := sync.WaitGroup{} - - wg.Add(1) - go ticker(ctx, &wg, collectParams) - - if collectParams.DryRun { - wg.Wait() - return nil - } - - client := mackerel.NewClient(collectParams.Mackerel.ApiKey) - - hostId, err := initialForMackerel(collectParams, client) - if err != nil { - return err - } - - wg.Add(1) - go sendTicker(ctx, &wg, client, hostId) - wg.Wait() - - return nil -} - func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { log.Println("init for mackerel") diff --git a/main.go b/main.go index efac8ef..4cd31a3 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,9 @@ import ( "log" "os" "os/signal" + "sync" + mackerel "github.com/mackerelio/mackerel-client-go" "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" ) @@ -29,17 +31,39 @@ func main() { collectParams.Debug = (collectParams.Debug || debug) collectParams.DryRun = (collectParams.DryRun || dryrun) - log.Println("start") - - if collectParams.Mackerel == nil { - _, err := collector.Do(ctx, collectParams) - if err != nil { - log.Fatal(err) - } - } else { - err = runMackerel(ctx, collectParams) - if err != nil { - log.Fatal(err) - } + err = run(ctx, collectParams) + if err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, collectParams *config.Config) error { + var err error + snapshot, err = collector.Do(ctx, collectParams) + if err != nil { + return err + } + + wg := sync.WaitGroup{} + + wg.Add(1) + go ticker(ctx, &wg, collectParams) + + if collectParams.DryRun { + wg.Wait() + return nil + } + + client := mackerel.NewClient(collectParams.Mackerel.ApiKey) + + hostId, err := initialForMackerel(collectParams, client) + if err != nil { + return err } + + wg.Add(1) + go sendTicker(ctx, &wg, client, hostId) + wg.Wait() + + return nil } From 23156c09892a4b7fc37642e57b9be1255b0a1aa5 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:11:07 +0000 Subject: [PATCH 37/77] added guard for dryrun --- mackerel.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mackerel.go b/mackerel.go index f5f8eea..69688ed 100644 --- a/mackerel.go +++ b/mackerel.go @@ -133,6 +133,10 @@ func innerTicker(ctx context.Context, collectParams *config.Config) error { return err } + if collectParams.DryRun { + return nil + } + prevSnapshot := snapshot snapshot = rawMetrics From 3b4f422973e5d65cf1df7ba2cf95ea6173f1c71e Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:39:03 +0000 Subject: [PATCH 38/77] cleanup --- mackerel.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/mackerel.go b/mackerel.go index 69688ed..4a27e57 100644 --- a/mackerel.go +++ b/mackerel.go @@ -104,10 +104,13 @@ func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Confi for { select { case <-t.C: - err := innerTicker(ctx, collectParams) + rawMetrics, err := collector.Do(ctx, collectParams) if err != nil { log.Println(err.Error()) } + if !collectParams.DryRun { + Enqueue(rawMetrics) + } case <-ctx.Done(): log.Println("cancellation from context:", ctx.Err()) return @@ -127,16 +130,7 @@ func calcurateDiff(a, b, overflow uint64) uint64 { } } -func innerTicker(ctx context.Context, collectParams *config.Config) error { - rawMetrics, err := collector.Do(ctx, collectParams) - if err != nil { - return err - } - - if collectParams.DryRun { - return nil - } - +func Enqueue(rawMetrics []collector.MetricsDutum) { prevSnapshot := snapshot snapshot = rawMetrics @@ -171,14 +165,11 @@ func innerTicker(ctx context.Context, collectParams *config.Config) error { Time: now, Value: value, }) - } mutex.Lock() buffers.PushBack(metrics) mutex.Unlock() - - return nil } func sendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client, hostId *string) { From 2a992aa6236dc5f5f8e26876e29dd85cc26ce941 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:40:12 +0000 Subject: [PATCH 39/77] cleanup --- mackerel.go | 24 ------------------------ main.go | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/mackerel.go b/mackerel.go index 4a27e57..0c0f486 100644 --- a/mackerel.go +++ b/mackerel.go @@ -94,30 +94,6 @@ func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return &hostId, nil } -func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config) { - t := time.NewTicker(1 * time.Minute) - defer func() { - t.Stop() - wg.Done() - }() - - for { - select { - case <-t.C: - rawMetrics, err := collector.Do(ctx, collectParams) - if err != nil { - log.Println(err.Error()) - } - if !collectParams.DryRun { - Enqueue(rawMetrics) - } - case <-ctx.Done(): - log.Println("cancellation from context:", ctx.Err()) - return - } - } -} - func escapeInterfaceName(ifName string) string { return strings.Replace(strings.Replace(strings.Replace(ifName, "/", "-", -1), ".", "_", -1), " ", "", -1) } diff --git a/main.go b/main.go index 4cd31a3..cfdbdf9 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "sync" + "time" mackerel "github.com/mackerelio/mackerel-client-go" "github.com/yseto/switch-traffic-to-mackerel/collector" @@ -67,3 +68,27 @@ func run(ctx context.Context, collectParams *config.Config) error { return nil } + +func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config) { + t := time.NewTicker(1 * time.Minute) + defer func() { + t.Stop() + wg.Done() + }() + + for { + select { + case <-t.C: + rawMetrics, err := collector.Do(ctx, collectParams) + if err != nil { + log.Println(err.Error()) + } + if !collectParams.DryRun { + Enqueue(rawMetrics) + } + case <-ctx.Done(): + log.Println("cancellation from context:", ctx.Err()) + return + } + } +} From 08de6744e252a8a8b5f5a3e8ccd7626248bfe7b6 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:42:29 +0000 Subject: [PATCH 40/77] move --- mackerel.go => mackerel/mackerel.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) rename mackerel.go => mackerel/mackerel.go (97%) diff --git a/mackerel.go b/mackerel/mackerel.go similarity index 97% rename from mackerel.go rename to mackerel/mackerel.go index 0c0f486..c998fe0 100644 --- a/mackerel.go +++ b/mackerel/mackerel.go @@ -1,4 +1,4 @@ -package main +package mackerel import ( "container/list" @@ -15,7 +15,6 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" - mckr "github.com/yseto/switch-traffic-to-mackerel/mackerel" ) var buffers = list.New() @@ -87,7 +86,7 @@ func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return nil, err } } - err = client.CreateGraphDefs(mckr.GraphDefs) + err = client.CreateGraphDefs(GraphDefs) if err != nil { return nil, err } From 02f958a007b7b9cf8c13b890ccbfdf873c18869d Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:44:19 +0000 Subject: [PATCH 41/77] cleanup --- mackerel/enqueue.go | 89 ++++++++++++++++++++++++++++++++++++++++++++ mackerel/mackerel.go | 82 ---------------------------------------- 2 files changed, 89 insertions(+), 82 deletions(-) create mode 100644 mackerel/enqueue.go diff --git a/mackerel/enqueue.go b/mackerel/enqueue.go new file mode 100644 index 0000000..90b4d76 --- /dev/null +++ b/mackerel/enqueue.go @@ -0,0 +1,89 @@ +package mackerel + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/mackerelio/mackerel-client-go" + "github.com/yseto/switch-traffic-to-mackerel/collector" +) + +var overflowValue = map[string]uint64{ + "ifInOctets": math.MaxUint32, + "ifOutOctets": math.MaxUint32, + "ifHCInOctets": math.MaxUint64, + "ifHCOutOctets": math.MaxUint64, + "ifInDiscards": math.MaxUint64, + "ifOutDiscards": math.MaxUint64, + "ifInErrors": math.MaxUint64, + "ifOutErrors": math.MaxUint64, +} + +var receiveDirection = map[string]bool{ + "ifInOctets": true, + "ifHCInOctets": true, + "ifInDiscards": true, + "ifInErrors": true, +} + +var deltaValues = map[string]bool{ + "ifInOctets": true, + "ifOutOctets": true, + "ifHCInOctets": true, + "ifHCOutOctets": true, +} + +func escapeInterfaceName(ifName string) string { + return strings.Replace(strings.Replace(strings.Replace(ifName, "/", "-", -1), ".", "_", -1), " ", "", -1) +} + +func calcurateDiff(a, b, overflow uint64) uint64 { + if b < a { + return overflow - a + b + } else { + return b - a + } +} +func Enqueue(rawMetrics []collector.MetricsDutum) { + prevSnapshot := snapshot + snapshot = rawMetrics + + now := time.Now().Unix() + + metrics := make([]*mackerel.MetricValue, 0) + for _, metric := range rawMetrics { + prevValue := metric.Value + for _, v := range prevSnapshot { + if v.IfIndex == metric.IfIndex && v.Mib == metric.Mib { + prevValue = v.Value + break + } + } + + value := calcurateDiff(prevValue, metric.Value, overflowValue[metric.Mib]) + + var name string + ifName := escapeInterfaceName(metric.IfName) + if deltaValues[metric.Mib] { + direction := "txBytes" + if receiveDirection[metric.Mib] { + direction = "rxBytes" + } + name = fmt.Sprintf("interface.%s.%s.delta", ifName, direction) + value /= 60 + } else { + name = fmt.Sprintf("custom.interface.%s.%s", metric.Mib, ifName) + } + metrics = append(metrics, &mackerel.MetricValue{ + Name: name, + Time: now, + Value: value, + }) + } + + mutex.Lock() + buffers.PushBack(metrics) + mutex.Unlock() +} diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index c998fe0..22a9440 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -3,11 +3,8 @@ package mackerel import ( "container/list" "context" - "fmt" "log" - "math" "os" - "strings" "sync" "time" @@ -21,31 +18,6 @@ var buffers = list.New() var mutex = &sync.Mutex{} var snapshot []collector.MetricsDutum -var overflowValue = map[string]uint64{ - "ifInOctets": math.MaxUint32, - "ifOutOctets": math.MaxUint32, - "ifHCInOctets": math.MaxUint64, - "ifHCOutOctets": math.MaxUint64, - "ifInDiscards": math.MaxUint64, - "ifOutDiscards": math.MaxUint64, - "ifInErrors": math.MaxUint64, - "ifOutErrors": math.MaxUint64, -} - -var receiveDirection = map[string]bool{ - "ifInOctets": true, - "ifHCInOctets": true, - "ifInDiscards": true, - "ifInErrors": true, -} - -var deltaValues = map[string]bool{ - "ifInOctets": true, - "ifOutOctets": true, - "ifHCInOctets": true, - "ifHCOutOctets": true, -} - func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { log.Println("init for mackerel") @@ -93,60 +65,6 @@ func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return &hostId, nil } -func escapeInterfaceName(ifName string) string { - return strings.Replace(strings.Replace(strings.Replace(ifName, "/", "-", -1), ".", "_", -1), " ", "", -1) -} - -func calcurateDiff(a, b, overflow uint64) uint64 { - if b < a { - return overflow - a + b - } else { - return b - a - } -} - -func Enqueue(rawMetrics []collector.MetricsDutum) { - prevSnapshot := snapshot - snapshot = rawMetrics - - now := time.Now().Unix() - - metrics := make([]*mackerel.MetricValue, 0) - for _, metric := range rawMetrics { - prevValue := metric.Value - for _, v := range prevSnapshot { - if v.IfIndex == metric.IfIndex && v.Mib == metric.Mib { - prevValue = v.Value - break - } - } - - value := calcurateDiff(prevValue, metric.Value, overflowValue[metric.Mib]) - - var name string - ifName := escapeInterfaceName(metric.IfName) - if deltaValues[metric.Mib] { - direction := "txBytes" - if receiveDirection[metric.Mib] { - direction = "rxBytes" - } - name = fmt.Sprintf("interface.%s.%s.delta", ifName, direction) - value /= 60 - } else { - name = fmt.Sprintf("custom.interface.%s.%s", metric.Mib, ifName) - } - metrics = append(metrics, &mackerel.MetricValue{ - Name: name, - Time: now, - Value: value, - }) - } - - mutex.Lock() - buffers.PushBack(metrics) - mutex.Unlock() -} - func sendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client, hostId *string) { t := time.NewTicker(500 * time.Millisecond) From e5af4a6a1c33f8c42f0be08e8c90d1cae8994967 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:46:37 +0000 Subject: [PATCH 42/77] cleanup --- mackerel/enqueue.go | 4 ++-- mackerel/mackerel.go | 6 +++--- main.go | 9 +++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mackerel/enqueue.go b/mackerel/enqueue.go index 90b4d76..4fd6f20 100644 --- a/mackerel/enqueue.go +++ b/mackerel/enqueue.go @@ -47,8 +47,8 @@ func calcurateDiff(a, b, overflow uint64) uint64 { } } func Enqueue(rawMetrics []collector.MetricsDutum) { - prevSnapshot := snapshot - snapshot = rawMetrics + prevSnapshot := Snapshot + Snapshot = rawMetrics now := time.Now().Unix() diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index 22a9440..ca96e1d 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -16,9 +16,9 @@ import ( var buffers = list.New() var mutex = &sync.Mutex{} -var snapshot []collector.MetricsDutum +var Snapshot []collector.MetricsDutum -func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { +func InitialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { log.Println("init for mackerel") idPath, err := c.HostIdPath() @@ -65,7 +65,7 @@ func initialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return &hostId, nil } -func sendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client, hostId *string) { +func SendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client, hostId *string) { t := time.NewTicker(500 * time.Millisecond) defer func() { diff --git a/main.go b/main.go index cfdbdf9..67e87e6 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( mackerel "github.com/mackerelio/mackerel-client-go" "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" + mckr "github.com/yseto/switch-traffic-to-mackerel/mackerel" ) func main() { @@ -40,7 +41,7 @@ func main() { func run(ctx context.Context, collectParams *config.Config) error { var err error - snapshot, err = collector.Do(ctx, collectParams) + mckr.Snapshot, err = collector.Do(ctx, collectParams) if err != nil { return err } @@ -57,13 +58,13 @@ func run(ctx context.Context, collectParams *config.Config) error { client := mackerel.NewClient(collectParams.Mackerel.ApiKey) - hostId, err := initialForMackerel(collectParams, client) + hostId, err := mckr.InitialForMackerel(collectParams, client) if err != nil { return err } wg.Add(1) - go sendTicker(ctx, &wg, client, hostId) + go mckr.SendTicker(ctx, &wg, client, hostId) wg.Wait() return nil @@ -84,7 +85,7 @@ func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Confi log.Println(err.Error()) } if !collectParams.DryRun { - Enqueue(rawMetrics) + mckr.Enqueue(rawMetrics) } case <-ctx.Done(): log.Println("cancellation from context:", ctx.Err()) From e8020729b7a8ee6bedf453b50b674eddae326734 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 12:56:17 +0000 Subject: [PATCH 43/77] cleanup --- mackerel/mackerel.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index ca96e1d..f389d7a 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -14,6 +14,30 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/config" ) +type MackerelClient interface { + CreateHost(param *mackerel.CreateHostParam) (string, error) + UpdateHost(hostID string, param *mackerel.UpdateHostParam) (string, error) + CreateGraphDefs(payloads []*mackerel.GraphDefsParam) error + PostHostMetricValuesByHostID(hostID string, metricValues []*mackerel.MetricValue) error +} + +type Queue struct { + sync.Mutex + + buffers *list.List + Snapshot []collector.MetricsDutum + client MackerelClient +} + +func NewQueue(apikey string) *Queue { + client := mackerel.NewClient(apikey) + + return &Queue{ + buffers: list.New(), + client: client, + } +} + var buffers = list.New() var mutex = &sync.Mutex{} var Snapshot []collector.MetricsDutum From 783c6a49e5384188e39b9c947e0b6a54a1ae1acf Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:03:20 +0000 Subject: [PATCH 44/77] cleanup --- mackerel/enqueue.go | 12 ++++++------ mackerel/mackerel.go | 37 +++++++++++++++++-------------------- main.go | 18 ++++++++---------- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/mackerel/enqueue.go b/mackerel/enqueue.go index 4fd6f20..afc9c92 100644 --- a/mackerel/enqueue.go +++ b/mackerel/enqueue.go @@ -46,9 +46,9 @@ func calcurateDiff(a, b, overflow uint64) uint64 { return b - a } } -func Enqueue(rawMetrics []collector.MetricsDutum) { - prevSnapshot := Snapshot - Snapshot = rawMetrics +func (q *Queue) Enqueue(rawMetrics []collector.MetricsDutum) { + prevSnapshot := q.Snapshot + q.Snapshot = rawMetrics now := time.Now().Unix() @@ -83,7 +83,7 @@ func Enqueue(rawMetrics []collector.MetricsDutum) { }) } - mutex.Lock() - buffers.PushBack(metrics) - mutex.Unlock() + q.Lock() + q.buffers.PushBack(metrics) + q.Unlock() } diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index f389d7a..4e794f3 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -29,20 +29,17 @@ type Queue struct { client MackerelClient } -func NewQueue(apikey string) *Queue { +func NewQueue(apikey string, snapshot []collector.MetricsDutum) *Queue { client := mackerel.NewClient(apikey) return &Queue{ - buffers: list.New(), - client: client, + buffers: list.New(), + client: client, + Snapshot: snapshot, } } -var buffers = list.New() -var mutex = &sync.Mutex{} -var Snapshot []collector.MetricsDutum - -func InitialForMackerel(c *config.Config, client *mackerel.Client) (*string, error) { +func (q *Queue) InitialForMackerel(c *config.Config) (*string, error) { log.Println("init for mackerel") idPath, err := c.HostIdPath() @@ -62,7 +59,7 @@ func InitialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return nil, err } hostId = string(bytes) - _, err = client.UpdateHost(hostId, &mackerel.UpdateHostParam{ + _, err = q.client.UpdateHost(hostId, &mackerel.UpdateHostParam{ Name: c.Name, Interfaces: interfaces, }) @@ -70,7 +67,7 @@ func InitialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return nil, err } } else { - hostId, err = client.CreateHost(&mackerel.CreateHostParam{ + hostId, err = q.client.CreateHost(&mackerel.CreateHostParam{ Name: c.Name, Interfaces: interfaces, }) @@ -82,14 +79,14 @@ func InitialForMackerel(c *config.Config, client *mackerel.Client) (*string, err return nil, err } } - err = client.CreateGraphDefs(GraphDefs) + err = q.client.CreateGraphDefs(GraphDefs) if err != nil { return nil, err } return &hostId, nil } -func SendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client, hostId *string) { +func (q *Queue) SendTicker(ctx context.Context, wg *sync.WaitGroup, hostId *string) { t := time.NewTicker(500 * time.Millisecond) defer func() { @@ -100,7 +97,7 @@ func SendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client for { select { case <-t.C: - sendToMackerel(ctx, client, hostId) + q.sendToMackerel(ctx, hostId) case <-ctx.Done(): log.Println("cancellation from context:", ctx.Err()) @@ -109,23 +106,23 @@ func SendTicker(ctx context.Context, wg *sync.WaitGroup, client *mackerel.Client } } -func sendToMackerel(ctx context.Context, client *mackerel.Client, hostId *string) { - if buffers.Len() == 0 { +func (q *Queue) sendToMackerel(ctx context.Context, hostId *string) { + if q.buffers.Len() == 0 { return } - e := buffers.Front() + e := q.buffers.Front() // log.Infof("send current value: %#v", e.Value) // log.Infof("buffers len: %d", buffers.Len()) - err := client.PostHostMetricValuesByHostID(*hostId, e.Value.([](*mackerel.MetricValue))) + err := q.client.PostHostMetricValuesByHostID(*hostId, e.Value.([](*mackerel.MetricValue))) if err != nil { log.Println(err) return } else { log.Println("success") } - mutex.Lock() - buffers.Remove(e) - mutex.Unlock() + q.Lock() + q.buffers.Remove(e) + q.Unlock() } diff --git a/main.go b/main.go index 67e87e6..ada3731 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( "sync" "time" - mackerel "github.com/mackerelio/mackerel-client-go" "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" mckr "github.com/yseto/switch-traffic-to-mackerel/mackerel" @@ -40,37 +39,36 @@ func main() { } func run(ctx context.Context, collectParams *config.Config) error { - var err error - mckr.Snapshot, err = collector.Do(ctx, collectParams) + snapshot, err := collector.Do(ctx, collectParams) if err != nil { return err } + queue := mckr.NewQueue(collectParams.Mackerel.ApiKey, snapshot) + wg := sync.WaitGroup{} wg.Add(1) - go ticker(ctx, &wg, collectParams) + go ticker(ctx, &wg, collectParams, queue) if collectParams.DryRun { wg.Wait() return nil } - client := mackerel.NewClient(collectParams.Mackerel.ApiKey) - - hostId, err := mckr.InitialForMackerel(collectParams, client) + hostId, err := queue.InitialForMackerel(collectParams) if err != nil { return err } wg.Add(1) - go mckr.SendTicker(ctx, &wg, client, hostId) + go queue.SendTicker(ctx, &wg, hostId) wg.Wait() return nil } -func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config) { +func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config, queue *mckr.Queue) { t := time.NewTicker(1 * time.Minute) defer func() { t.Stop() @@ -85,7 +83,7 @@ func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Confi log.Println(err.Error()) } if !collectParams.DryRun { - mckr.Enqueue(rawMetrics) + queue.Enqueue(rawMetrics) } case <-ctx.Done(): log.Println("cancellation from context:", ctx.Err()) From 1945465be09e91ca2342ae2ec3dc8c9be80d3763 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:35:28 +0000 Subject: [PATCH 45/77] cleanup --- mackerel/mackerel.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index 4e794f3..cc7fd66 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -27,15 +27,22 @@ type Queue struct { buffers *list.List Snapshot []collector.MetricsDutum client MackerelClient + + hostID string + targetAddr string + name string } -func NewQueue(apikey string, snapshot []collector.MetricsDutum) *Queue { +func NewQueue(apikey, hostID, targetAddr, name string, snapshot []collector.MetricsDutum) *Queue { client := mackerel.NewClient(apikey) return &Queue{ - buffers: list.New(), - client: client, - Snapshot: snapshot, + buffers: list.New(), + client: client, + Snapshot: snapshot, + hostID: hostID, + targetAddr: targetAddr, + name: name, } } From 06b630ad3c8c8f27eb42204da92d4ddb83c48d47 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:36:24 +0000 Subject: [PATCH 46/77] cleanup --- mackerel/mackerel.go | 46 +++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index cc7fd66..0858e94 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -4,14 +4,12 @@ import ( "container/list" "context" "log" - "os" "sync" "time" mackerel "github.com/mackerelio/mackerel-client-go" "github.com/yseto/switch-traffic-to-mackerel/collector" - "github.com/yseto/switch-traffic-to-mackerel/config" ) type MackerelClient interface { @@ -46,51 +44,39 @@ func NewQueue(apikey, hostID, targetAddr, name string, snapshot []collector.Metr } } -func (q *Queue) InitialForMackerel(c *config.Config) (*string, error) { +func (q *Queue) InitialForMackerel() (*string, error) { log.Println("init for mackerel") - idPath, err := c.HostIdPath() - if err != nil { - return nil, err - } interfaces := []mackerel.Interface{ { Name: "main", - IPv4Addresses: []string{c.Target}, + IPv4Addresses: []string{q.targetAddr}, }, } - var hostId string - if _, err := os.Stat(idPath); err == nil { - bytes, err := os.ReadFile(idPath) - if err != nil { - return nil, err - } - hostId = string(bytes) - _, err = q.client.UpdateHost(hostId, &mackerel.UpdateHostParam{ - Name: c.Name, + + var newHostID *string + var err error + if q.hostID != "" { + _, err = q.client.UpdateHost(q.hostID, &mackerel.UpdateHostParam{ + Name: q.name, Interfaces: interfaces, }) - if err != nil { - return nil, err - } } else { - hostId, err = q.client.CreateHost(&mackerel.CreateHostParam{ - Name: c.Name, + q.hostID, err = q.client.CreateHost(&mackerel.CreateHostParam{ + Name: q.name, Interfaces: interfaces, }) - if err != nil { - return nil, err - } - err = os.WriteFile(idPath, []byte(hostId), 0666) - if err != nil { - return nil, err - } + newHostID = &q.hostID + } + if err != nil { + return nil, err } + err = q.client.CreateGraphDefs(GraphDefs) if err != nil { return nil, err } - return &hostId, nil + return newHostID, nil } func (q *Queue) SendTicker(ctx context.Context, wg *sync.WaitGroup, hostId *string) { From d16fee8c7977d999735afc98fad8eaa524bc6a12 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:37:44 +0000 Subject: [PATCH 47/77] cleanup --- mackerel/mackerel.go | 8 ++++---- main.go | 12 +++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index 0858e94..d790c5a 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -79,7 +79,7 @@ func (q *Queue) InitialForMackerel() (*string, error) { return newHostID, nil } -func (q *Queue) SendTicker(ctx context.Context, wg *sync.WaitGroup, hostId *string) { +func (q *Queue) SendTicker(ctx context.Context, wg *sync.WaitGroup) { t := time.NewTicker(500 * time.Millisecond) defer func() { @@ -90,7 +90,7 @@ func (q *Queue) SendTicker(ctx context.Context, wg *sync.WaitGroup, hostId *stri for { select { case <-t.C: - q.sendToMackerel(ctx, hostId) + q.sendToMackerel(ctx) case <-ctx.Done(): log.Println("cancellation from context:", ctx.Err()) @@ -99,7 +99,7 @@ func (q *Queue) SendTicker(ctx context.Context, wg *sync.WaitGroup, hostId *stri } } -func (q *Queue) sendToMackerel(ctx context.Context, hostId *string) { +func (q *Queue) sendToMackerel(ctx context.Context) { if q.buffers.Len() == 0 { return } @@ -108,7 +108,7 @@ func (q *Queue) sendToMackerel(ctx context.Context, hostId *string) { // log.Infof("send current value: %#v", e.Value) // log.Infof("buffers len: %d", buffers.Len()) - err := q.client.PostHostMetricValuesByHostID(*hostId, e.Value.([](*mackerel.MetricValue))) + err := q.client.PostHostMetricValuesByHostID(q.hostID, e.Value.([](*mackerel.MetricValue))) if err != nil { log.Println(err) return diff --git a/main.go b/main.go index ada3731..95bd922 100644 --- a/main.go +++ b/main.go @@ -44,7 +44,13 @@ func run(ctx context.Context, collectParams *config.Config) error { return err } - queue := mckr.NewQueue(collectParams.Mackerel.ApiKey, snapshot) + // TODO collectParams.Mackerel == nil + queue := mckr.NewQueue( + collectParams.Mackerel.ApiKey, + collectParams.Mackerel.HostID, + collectParams.Target, + collectParams.Name, + snapshot) wg := sync.WaitGroup{} @@ -56,13 +62,13 @@ func run(ctx context.Context, collectParams *config.Config) error { return nil } - hostId, err := queue.InitialForMackerel(collectParams) + newHostID, err := queue.InitialForMackerel() if err != nil { return err } wg.Add(1) - go queue.SendTicker(ctx, &wg, hostId) + go queue.SendTicker(ctx, &wg) wg.Wait() return nil From a8863278dbfcdd4576978a5207c42f1ae163134e Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:37:50 +0000 Subject: [PATCH 48/77] cleanup --- config/config.go | 28 ++++++++++++++++++++++++++++ mackerel/mackerel.go | 1 + main.go | 3 +++ 3 files changed, 32 insertions(+) diff --git a/config/config.go b/config/config.go index bd5f372..eb25aa2 100644 --- a/config/config.go +++ b/config/config.go @@ -118,3 +118,31 @@ func Init(filename string) (*Config, error) { return c, nil } + +func (c *Config) Save(hostID string) error { + stat, err := os.Stat(loadedFilename) + if err != nil { + return err + } + perm := stat.Mode() + + // read original config + f, err := os.ReadFile(loadedFilename) + if err != nil { + return err + } + var t YAMLConfig + err = yaml.Unmarshal(f, &t) + if err != nil { + return err + } + + // added Mackerel.HostID + t.Mackerel.HostID = hostID + + b, err := yaml.Marshal(t) + if err != nil { + return err + } + return os.WriteFile(loadedFilename, b, perm) +} diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index d790c5a..9aecc7d 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -44,6 +44,7 @@ func NewQueue(apikey, hostID, targetAddr, name string, snapshot []collector.Metr } } +// return host ID when create. func (q *Queue) InitialForMackerel() (*string, error) { log.Println("init for mackerel") diff --git a/main.go b/main.go index 95bd922..4c1d733 100644 --- a/main.go +++ b/main.go @@ -66,6 +66,9 @@ func run(ctx context.Context, collectParams *config.Config) error { if err != nil { return err } + if newHostID != nil { + collectParams.Save(*newHostID) + } wg.Add(1) go queue.SendTicker(ctx, &wg) From d2c290279bdfb6254e4e20d118e70c2ba24293a6 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:50:43 +0000 Subject: [PATCH 49/77] omitempty --- config/config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index eb25aa2..c7c4bbd 100644 --- a/config/config.go +++ b/config/config.go @@ -17,17 +17,17 @@ type YAMLConfig struct { Community string `yaml:"community"` Target string `yaml:"target"` Interface *Interface `yaml:"interface"` - Mibs []string `yaml:"mibs"` - SkipLinkdown bool `yaml:"skip-linkdown"` + Mibs []string `yaml:"mibs,omitempty"` + SkipLinkdown bool `yaml:"skip-linkdown,omitempty"` Name string `yaml:"name"` - Mackerel *Mackerel `yaml:"mackerel"` - Debug bool `yaml:"debug"` - DryRun bool `yaml:"dry-run"` + Mackerel *Mackerel `yaml:"mackerel,omitempty"` + Debug bool `yaml:"debug,omitempty"` + DryRun bool `yaml:"dry-run,omitempty"` } type Interface struct { - Include *string `yaml:"include"` - Exclude *string `yaml:"exclude"` + Include *string `yaml:"include,omitempty"` + Exclude *string `yaml:"exclude,omitempty"` } type Mackerel struct { From 2f15eafc72fb70cadd81ff2677f4b7f8b0155ffc Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:50:56 +0000 Subject: [PATCH 50/77] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b6b072 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +config.yaml From 4ed0b5df385c49fc3a0cfe45be659865a2e3acf3 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 13:52:12 +0000 Subject: [PATCH 51/77] cleanup --- main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 4c1d733..6921b54 100644 --- a/main.go +++ b/main.go @@ -52,10 +52,10 @@ func run(ctx context.Context, collectParams *config.Config) error { collectParams.Name, snapshot) - wg := sync.WaitGroup{} + wg := &sync.WaitGroup{} wg.Add(1) - go ticker(ctx, &wg, collectParams, queue) + go ticker(ctx, wg, collectParams, queue) if collectParams.DryRun { wg.Wait() @@ -71,7 +71,7 @@ func run(ctx context.Context, collectParams *config.Config) error { } wg.Add(1) - go queue.SendTicker(ctx, &wg) + go queue.SendTicker(ctx, wg) wg.Wait() return nil From dffaf8828090f22ac651e2499794eaa6add899e2 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 14:12:19 +0000 Subject: [PATCH 52/77] force dry-run --- mackerel/mackerel.go | 20 ++++++++++++++------ main.go | 23 ++++++++++++++++------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index 9aecc7d..a50c4a1 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -31,16 +31,24 @@ type Queue struct { name string } -func NewQueue(apikey, hostID, targetAddr, name string, snapshot []collector.MetricsDutum) *Queue { - client := mackerel.NewClient(apikey) +type QueueArg struct { + Apikey string + HostID string + TargetAddr string + Name string + Snapshot []collector.MetricsDutum +} + +func NewQueue(qa *QueueArg) *Queue { + client := mackerel.NewClient(qa.Apikey) return &Queue{ buffers: list.New(), client: client, - Snapshot: snapshot, - hostID: hostID, - targetAddr: targetAddr, - name: name, + Snapshot: qa.Snapshot, + hostID: qa.HostID, + targetAddr: qa.TargetAddr, + name: qa.Name, } } diff --git a/main.go b/main.go index 6921b54..353cb15 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,11 @@ func main() { collectParams.Debug = (collectParams.Debug || debug) collectParams.DryRun = (collectParams.DryRun || dryrun) + if collectParams.Mackerel == nil { + log.Println("force dry-run.") + collectParams.DryRun = true + } + err = run(ctx, collectParams) if err != nil { log.Fatal(err) @@ -44,13 +49,17 @@ func run(ctx context.Context, collectParams *config.Config) error { return err } - // TODO collectParams.Mackerel == nil - queue := mckr.NewQueue( - collectParams.Mackerel.ApiKey, - collectParams.Mackerel.HostID, - collectParams.Target, - collectParams.Name, - snapshot) + qa := &mckr.QueueArg{ + TargetAddr: collectParams.Target, + Name: collectParams.Name, + Snapshot: snapshot, + } + if collectParams.Mackerel != nil { + qa.Apikey = collectParams.Mackerel.ApiKey + qa.HostID = collectParams.Mackerel.HostID + } + + queue := mckr.NewQueue(qa) wg := &sync.WaitGroup{} From 2f2f16b8740510d8bae67d38bfce9321d4779d23 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 14:16:51 +0000 Subject: [PATCH 53/77] cleanup --- mackerel/mackerel.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index a50c4a1..139f2e1 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -114,9 +114,6 @@ func (q *Queue) sendToMackerel(ctx context.Context) { } e := q.buffers.Front() - // log.Infof("send current value: %#v", e.Value) - // log.Infof("buffers len: %d", buffers.Len()) - err := q.client.PostHostMetricValuesByHostID(q.hostID, e.Value.([](*mackerel.MetricValue))) if err != nil { log.Println(err) From adf912e6320673977ddb985c46afbb2a19e36332 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 14:17:20 +0000 Subject: [PATCH 54/77] cleanup --- main.go | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/main.go b/main.go index 353cb15..5520caa 100644 --- a/main.go +++ b/main.go @@ -25,65 +25,56 @@ func main() { flag.BoolVar(&dryrun, "dry-run", false, "dry run") flag.Parse() - collectParams, err := config.Init(filename) + c, err := config.Init(filename) if err != nil { log.Fatal(err) } - collectParams.Debug = (collectParams.Debug || debug) - collectParams.DryRun = (collectParams.DryRun || dryrun) + c.Debug = (c.Debug || debug) + c.DryRun = (c.DryRun || dryrun) - if collectParams.Mackerel == nil { + if c.Mackerel == nil { log.Println("force dry-run.") - collectParams.DryRun = true + c.DryRun = true } - err = run(ctx, collectParams) + snapshot, err := collector.Do(ctx, c) if err != nil { log.Fatal(err) } -} - -func run(ctx context.Context, collectParams *config.Config) error { - snapshot, err := collector.Do(ctx, collectParams) - if err != nil { - return err - } qa := &mckr.QueueArg{ - TargetAddr: collectParams.Target, - Name: collectParams.Name, + TargetAddr: c.Target, + Name: c.Name, Snapshot: snapshot, } - if collectParams.Mackerel != nil { - qa.Apikey = collectParams.Mackerel.ApiKey - qa.HostID = collectParams.Mackerel.HostID + if c.Mackerel != nil { + qa.Apikey = c.Mackerel.ApiKey + qa.HostID = c.Mackerel.HostID } - queue := mckr.NewQueue(qa) wg := &sync.WaitGroup{} - wg.Add(1) - go ticker(ctx, wg, collectParams, queue) + go ticker(ctx, wg, c, queue) - if collectParams.DryRun { + if c.DryRun { wg.Wait() - return nil + return } newHostID, err := queue.InitialForMackerel() if err != nil { - return err + log.Fatal(err) } if newHostID != nil { - collectParams.Save(*newHostID) + if err = c.Save(*newHostID); err != nil { + log.Fatal(err) + } } wg.Add(1) go queue.SendTicker(ctx, wg) wg.Wait() - - return nil } func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config, queue *mckr.Queue) { From c6e2c8cf4cfb17ddc3bbd3573b6b62e9302fd1b9 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 14:18:51 +0000 Subject: [PATCH 55/77] cleanup --- mackerel/enqueue.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mackerel/enqueue.go b/mackerel/enqueue.go index afc9c92..afb11fd 100644 --- a/mackerel/enqueue.go +++ b/mackerel/enqueue.go @@ -46,6 +46,7 @@ func calcurateDiff(a, b, overflow uint64) uint64 { return b - a } } + func (q *Queue) Enqueue(rawMetrics []collector.MetricsDutum) { prevSnapshot := q.Snapshot q.Snapshot = rawMetrics From 6a40e5a92dbbc7dbbb248163e609a189bd6a5299 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 15:10:52 +0000 Subject: [PATCH 56/77] remove unused function --- config/config.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/config/config.go b/config/config.go index c7c4bbd..ede5290 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import ( "fmt" "os" - "path/filepath" "regexp" "gopkg.in/yaml.v3" @@ -48,14 +47,6 @@ type Config struct { Mackerel *Mackerel } -func (c *Config) HostIdPath() (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - return filepath.Join(wd, fmt.Sprintf("%s.id.txt", c.Target)), nil -} - func Init(filename string) (*Config, error) { loadedFilename = filename f, err := os.ReadFile(filename) From c03c2ff604730e73bf544c74e2e038e18168cce7 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 15:11:12 +0000 Subject: [PATCH 57/77] cleanup --- config/config.go | 28 ---------------------------- config/save.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 28 deletions(-) create mode 100644 config/save.go diff --git a/config/config.go b/config/config.go index ede5290..02c001c 100644 --- a/config/config.go +++ b/config/config.go @@ -109,31 +109,3 @@ func Init(filename string) (*Config, error) { return c, nil } - -func (c *Config) Save(hostID string) error { - stat, err := os.Stat(loadedFilename) - if err != nil { - return err - } - perm := stat.Mode() - - // read original config - f, err := os.ReadFile(loadedFilename) - if err != nil { - return err - } - var t YAMLConfig - err = yaml.Unmarshal(f, &t) - if err != nil { - return err - } - - // added Mackerel.HostID - t.Mackerel.HostID = hostID - - b, err := yaml.Marshal(t) - if err != nil { - return err - } - return os.WriteFile(loadedFilename, b, perm) -} diff --git a/config/save.go b/config/save.go new file mode 100644 index 0000000..e27a9b8 --- /dev/null +++ b/config/save.go @@ -0,0 +1,35 @@ +package config + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +func (c *Config) Save(hostID string) error { + stat, err := os.Stat(loadedFilename) + if err != nil { + return err + } + perm := stat.Mode() + + // read original config + f, err := os.ReadFile(loadedFilename) + if err != nil { + return err + } + var t YAMLConfig + err = yaml.Unmarshal(f, &t) + if err != nil { + return err + } + + // added Mackerel.HostID + t.Mackerel.HostID = hostID + + b, err := yaml.Marshal(t) + if err != nil { + return err + } + return os.WriteFile(loadedFilename, b, perm) +} From 2e2f8f6610dd1c3f028e3db99372e40038ba95b6 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 15:12:52 +0000 Subject: [PATCH 58/77] cleanup --- mackerel/graphdef.go | 2 +- mackerel/mackerel.go | 10 +++++----- main.go | 15 ++++++++------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mackerel/graphdef.go b/mackerel/graphdef.go index 85aaf93..d64003f 100644 --- a/mackerel/graphdef.go +++ b/mackerel/graphdef.go @@ -2,7 +2,7 @@ package mackerel import "github.com/mackerelio/mackerel-client-go" -var GraphDefs = []*mackerel.GraphDefsParam{ +var graphDefs = []*mackerel.GraphDefsParam{ { Name: "custom.interface.ifInDiscards", Unit: "integer", diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index 139f2e1..61b98a1 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -12,7 +12,7 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/collector" ) -type MackerelClient interface { +type mackerelClient interface { CreateHost(param *mackerel.CreateHostParam) (string, error) UpdateHost(hostID string, param *mackerel.UpdateHostParam) (string, error) CreateGraphDefs(payloads []*mackerel.GraphDefsParam) error @@ -24,7 +24,7 @@ type Queue struct { buffers *list.List Snapshot []collector.MetricsDutum - client MackerelClient + client mackerelClient hostID string targetAddr string @@ -53,8 +53,8 @@ func NewQueue(qa *QueueArg) *Queue { } // return host ID when create. -func (q *Queue) InitialForMackerel() (*string, error) { - log.Println("init for mackerel") +func (q *Queue) Init() (*string, error) { + log.Println("init queue") interfaces := []mackerel.Interface{ { @@ -81,7 +81,7 @@ func (q *Queue) InitialForMackerel() (*string, error) { return nil, err } - err = q.client.CreateGraphDefs(GraphDefs) + err = q.client.CreateGraphDefs(graphDefs) if err != nil { return nil, err } diff --git a/main.go b/main.go index 5520caa..0724498 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/collector" "github.com/yseto/switch-traffic-to-mackerel/config" - mckr "github.com/yseto/switch-traffic-to-mackerel/mackerel" + "github.com/yseto/switch-traffic-to-mackerel/mackerel" ) func main() { @@ -42,7 +42,7 @@ func main() { log.Fatal(err) } - qa := &mckr.QueueArg{ + qa := &mackerel.QueueArg{ TargetAddr: c.Target, Name: c.Name, Snapshot: snapshot, @@ -51,7 +51,7 @@ func main() { qa.Apikey = c.Mackerel.ApiKey qa.HostID = c.Mackerel.HostID } - queue := mckr.NewQueue(qa) + queue := mackerel.NewQueue(qa) wg := &sync.WaitGroup{} wg.Add(1) @@ -62,11 +62,12 @@ func main() { return } - newHostID, err := queue.InitialForMackerel() + newHostID, err := queue.Init() if err != nil { log.Fatal(err) } if newHostID != nil { + log.Println("save HostID") if err = c.Save(*newHostID); err != nil { log.Fatal(err) } @@ -77,7 +78,7 @@ func main() { wg.Wait() } -func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Config, queue *mckr.Queue) { +func ticker(ctx context.Context, wg *sync.WaitGroup, c *config.Config, queue *mackerel.Queue) { t := time.NewTicker(1 * time.Minute) defer func() { t.Stop() @@ -87,11 +88,11 @@ func ticker(ctx context.Context, wg *sync.WaitGroup, collectParams *config.Confi for { select { case <-t.C: - rawMetrics, err := collector.Do(ctx, collectParams) + rawMetrics, err := collector.Do(ctx, c) if err != nil { log.Println(err.Error()) } - if !collectParams.DryRun { + if !c.DryRun { queue.Enqueue(rawMetrics) } case <-ctx.Done(): From 905dc3d98cd635cc1e69992ca2dd6d98f0c3c3ad Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 15:18:26 +0000 Subject: [PATCH 59/77] go 1.20 --- .github/workflows/release.yml | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5079f8b..70790c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.20 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v3 diff --git a/go.mod b/go.mod index f77883c..6aa0870 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/yseto/switch-traffic-to-mackerel -go 1.18 +go 1.20 require ( github.com/gosnmp/gosnmp v1.35.0 From 2baba993065241d51d8ebe9e17bf5a27452d35bf Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 15:22:32 +0000 Subject: [PATCH 60/77] update github.com/mackerelio/mackerel-client-go v0.24.0 --- go.mod | 9 ++------- go.sum | 23 ++--------------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 6aa0870..c76dbe3 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,9 @@ go 1.20 require ( github.com/gosnmp/gosnmp v1.35.0 - github.com/mackerelio/mackerel-client-go v0.23.0 + github.com/mackerelio/mackerel-client-go v0.24.0 github.com/maruel/natural v1.1.0 gopkg.in/yaml.v3 v3.0.1 ) -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/tools v0.3.0 // indirect -) +require github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index c31e9c3..8d87ce9 100644 --- a/go.sum +++ b/go.sum @@ -3,31 +3,12 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/gosnmp/gosnmp v1.35.0 h1:EuWWNPxTCdAUx2/NbQcSa3WdNxjzpy4Phv57b4MWpJM= github.com/gosnmp/gosnmp v1.35.0/go.mod h1:2AvKZ3n9aEl5TJEo/fFmf/FGO4Nj4cVeEc5yuk88CYc= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/mackerelio/mackerel-client-go v0.23.0 h1:C6ENbhfAZWofpr2Vc/cRTNdCkQonQ/nRlA08K+9+m/M= -github.com/mackerelio/mackerel-client-go v0.23.0/go.mod h1:VM9KAjzs7wkROQ9WdZRvkY8K/bIUllN82dxyK36ZU3s= +github.com/mackerelio/mackerel-client-go v0.24.0 h1:Y9FeTgrQlDdtLU7FtMM6hd5mL5T4TvGCVUY0LH+bceQ= +github.com/mackerelio/mackerel-client-go v0.24.0/go.mod h1:b4qVMQi+w4rxtKQIFycLWXNBtIi9d0r571RzYmg/aXo= github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= -golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 1066b22133ea260b3eca0adf60cc520d695759db Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 16:12:51 +0000 Subject: [PATCH 61/77] cleanup --- config.yaml.sample | 2 +- config/config.go | 11 ++--------- main.go | 5 ++++- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/config.yaml.sample b/config.yaml.sample index 1e5aa7b..37603ee 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -7,7 +7,7 @@ mibs: # capture mib name - ifHCInOctets - ifHCOutOctets skip-linkdown: true -name: "" # display Name on Mackerel mackerel: x-api-key: xxxxx host-id: xxxxx + name: "" # display Name on Mackerel diff --git a/config/config.go b/config/config.go index 02c001c..da0711c 100644 --- a/config/config.go +++ b/config/config.go @@ -15,10 +15,9 @@ var loadedFilename string type YAMLConfig struct { Community string `yaml:"community"` Target string `yaml:"target"` - Interface *Interface `yaml:"interface"` + Interface *Interface `yaml:"interface,omitempty"` Mibs []string `yaml:"mibs,omitempty"` SkipLinkdown bool `yaml:"skip-linkdown,omitempty"` - Name string `yaml:"name"` Mackerel *Mackerel `yaml:"mackerel,omitempty"` Debug bool `yaml:"debug,omitempty"` DryRun bool `yaml:"dry-run,omitempty"` @@ -32,12 +31,12 @@ type Interface struct { type Mackerel struct { HostID string `yaml:"host-id"` ApiKey string `yaml:"x-api-key"` + Name string `yaml:"name,omitempty"` } type Config struct { Community string Target string - Name string MIBs []string IncludeRegexp *regexp.Regexp ExcludeRegexp *regexp.Regexp @@ -66,16 +65,10 @@ func Init(filename string) (*Config, error) { return nil, fmt.Errorf("target is needed.") } - name := t.Name - if name == "" { - name = t.Target - } - c := &Config{ Target: t.Target, Community: t.Community, SkipDownLinkState: t.SkipLinkdown, - Name: name, Debug: t.Debug, DryRun: t.DryRun, } diff --git a/main.go b/main.go index 0724498..c6d3bee 100644 --- a/main.go +++ b/main.go @@ -44,12 +44,15 @@ func main() { qa := &mackerel.QueueArg{ TargetAddr: c.Target, - Name: c.Name, Snapshot: snapshot, } if c.Mackerel != nil { qa.Apikey = c.Mackerel.ApiKey qa.HostID = c.Mackerel.HostID + qa.Name = c.Mackerel.Name + if qa.Name == "" { + qa.Name = c.Target + } } queue := mackerel.NewQueue(qa) From 25646ba9d8234d9bc9b995f30b5d64b6bedeba21 Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 16:13:09 +0000 Subject: [PATCH 62/77] update README --- README.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 429e4b0..71350f9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,75 @@ # switch-traffic-to-mackerel -## usage +**これは趣味プロダクトです** + +ネットワークスイッチなどに対してSNMPで問い合わせを行い、インターフェイスの通信量などの統計情報を取得するプログラムです。 + +レポジトリ名にある通り、[mackerel.io](https://ja.mackerel.io/)に情報を送ることを主な目的としています。 + +## 特徴 + +- GETBULKを用いて値を取得するので、比較的速く動作します。 +- 取り込むインターフェイス名を正規表現で指定することができるので、取り込みたくないインターフェイスを除外できます。 +- mackerelに対して、通信量をシステムメトリックとして投稿するため、このプログラムが異常終了した場合など送信が失敗している状態に、死活監視で気づくことができます。 +- mackerelとの通信が途絶えた場合でもプログラム内部でキャッシュし、通信が再開できたときに一斉に送信します。 + +## 使い方 + +1. config.yaml.sample を config.yaml という名前でコピーします +2. config.yaml を開き加工します +3. `switch-traffic-to-mackerel -config config.yaml` で起動する + +## 設定ファイルの内容 + +```yaml +community: public # (必須)取得する対象のスイッチなどの SNMP コミュニティ名を設定する +target: 192.2.0.1 # (必須)取得する対象のスイッチなどの IPアドレスを設定する +interface: # (オプション)取り込むインターフェイスをインターフェイス名を使って絞り込むことができます。includeとexcludeはそれぞれ排他です。 + include: "" # 取得時に取り込みたいインターフェイス名を正規表現で指定します + exclude: "" # 取得時に取り込みたくないインターフェイス名を正規表現で指定します +mibs: # (オプション)取り込みたい情報を設定できます。無指定時は、以下に示されるMIBについての情報が取り込まれます + - ifHCInOctets + - ifHCOutOctets + - ifInDiscards + - ifOutDiscards + - ifInErrors + - ifOutErrors +# 機器によっては ifHCInOctets、ifHCOutOctets への対応ができない場合があります。その場合は、以下を明示的に指定する必要があります +# - ifInOctets +# - ifOutOctets +debug: false # (オプション) true時、デバッグ表示を有効にします。取り込むインターフェイス名およびその値を表示します +dry-run: false # (オプション) true時、mackerel への送信を抑制します。mackerel についての情報が設定ファイルに含まれてない場合は、強制的に true となります。 +skip-linkdown: false # (オプション) downしているインターフェイスについては取り込みをスキップするオプションです +mackerel: # (オプション)Mackerel に送信する時のパラメータ + name: "" # (オプション)Mackerel に登録するホスト名 + x-api-key: xxxxx # (必須) Mackerel の APIキー + host-id: xxxxx # (オプション) Mackerel でのホストID、無指定時の場合、プログラム内で自動的に取得し、設定ファイルを更新します。 +``` + +## v0.0.1 からの移行 + +v0.0.1 までは設定ファイルを基本的に使用していませんでした。そのため設定ファイルを作成する必要があります。 + +以下のようなコマンドラインで起動させていた場合は、後述するようなYAMLになります ``` export MACKEREL_API_KEY=xxxx ./switch-traffic-to-mackerel -target 192.0.2.1 -mibs ifHCInOctets,ifHCOutOctets -include-interface 'ge-0/0/\d+$|ae0$' -skip-down-link-state -name sw1 ``` +```yaml +community: public +target: 192.0.2.1 +mibs: + - ifHCInOctets + - ifHCOutOctets +interface: + include: ge-0/0/\d+$|ae0$ +skip-linkdown: true +mackerel: + host-id: <192.0.2.1.id.txt というような ${target}.id.txt というファイルにホストIDが記録されているので転記してください> + x-api-key: xxxx # 環境変数を読まなくなりましたので、直接記述してください + name: sw1 +``` + + From 2795d77983a615ac0cc4021cef06f73dfeb4408b Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 16:30:31 +0000 Subject: [PATCH 63/77] trim tab --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 71350f9..8d609f9 100644 --- a/README.md +++ b/README.md @@ -28,15 +28,15 @@ interface: # (オプション)取り込むインターフェイスをインタ include: "" # 取得時に取り込みたいインターフェイス名を正規表現で指定します exclude: "" # 取得時に取り込みたくないインターフェイス名を正規表現で指定します mibs: # (オプション)取り込みたい情報を設定できます。無指定時は、以下に示されるMIBについての情報が取り込まれます - - ifHCInOctets - - ifHCOutOctets - - ifInDiscards - - ifOutDiscards - - ifInErrors - - ifOutErrors + - ifHCInOctets + - ifHCOutOctets + - ifInDiscards + - ifOutDiscards + - ifInErrors + - ifOutErrors # 機器によっては ifHCInOctets、ifHCOutOctets への対応ができない場合があります。その場合は、以下を明示的に指定する必要があります -# - ifInOctets -# - ifOutOctets +# - ifInOctets +# - ifOutOctets debug: false # (オプション) true時、デバッグ表示を有効にします。取り込むインターフェイス名およびその値を表示します dry-run: false # (オプション) true時、mackerel への送信を抑制します。mackerel についての情報が設定ファイルに含まれてない場合は、強制的に true となります。 skip-linkdown: false # (オプション) downしているインターフェイスについては取り込みをスキップするオプションです From c78c01c786ba8e04f2bde5e23839a3e486b5e7ec Mon Sep 17 00:00:00 2001 From: yseto Date: Wed, 15 Mar 2023 16:33:06 +0000 Subject: [PATCH 64/77] add file on archive. --- .goreleaser.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.goreleaser.yml b/.goreleaser.yml index 0eebf02..baccd79 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -8,6 +8,8 @@ builds: archives: - format: zip name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" + files: + - config.yaml.sample release: github: owner: yseto From ed0b43642b639658adb59a7d501de1639b4447cd Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 11:57:33 +0000 Subject: [PATCH 65/77] added tests. --- go.mod | 1 + go.sum | 2 + mackerel/enqueue_test.go | 135 +++++++++++++++++++++ mackerel/mackerel_test.go | 244 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 382 insertions(+) create mode 100644 mackerel/enqueue_test.go create mode 100644 mackerel/mackerel_test.go diff --git a/go.mod b/go.mod index c76dbe3..2e88614 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/yseto/switch-traffic-to-mackerel go 1.20 require ( + github.com/google/go-cmp v0.5.9 github.com/gosnmp/gosnmp v1.35.0 github.com/mackerelio/mackerel-client-go v0.24.0 github.com/maruel/natural v1.1.0 diff --git a/go.sum b/go.sum index 8d87ce9..2b65850 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gosnmp/gosnmp v1.35.0 h1:EuWWNPxTCdAUx2/NbQcSa3WdNxjzpy4Phv57b4MWpJM= github.com/gosnmp/gosnmp v1.35.0/go.mod h1:2AvKZ3n9aEl5TJEo/fFmf/FGO4Nj4cVeEc5yuk88CYc= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= diff --git a/mackerel/enqueue_test.go b/mackerel/enqueue_test.go new file mode 100644 index 0000000..0769e02 --- /dev/null +++ b/mackerel/enqueue_test.go @@ -0,0 +1,135 @@ +package mackerel + +import ( + "container/list" + "math" + "reflect" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/mackerelio/mackerel-client-go" + + "github.com/yseto/switch-traffic-to-mackerel/collector" +) + +func compare[T any](t *testing.T, a, b T) { + t.Helper() + if !reflect.DeepEqual(a, b) { + t.Errorf("invalid %v %v", a, b) + } +} + +func TestEscapeInterfaceName(t *testing.T) { + compare(t, escapeInterfaceName("a/1.hello hello"), "a-1_hellohello") +} + +func TestCalcurateDiff(t *testing.T) { + compare(t, calcurateDiff(1, 2, 4), 1) + compare(t, calcurateDiff(2, 2, 4), 0) + compare(t, calcurateDiff(3, 2, 4), 3) + compare(t, calcurateDiff(4, 2, 4), 2) + compare(t, calcurateDiff(5, 2, 4), 1) +} + +func TestEnqueue(t *testing.T) { + t.Run("replace Snapshot", func(t *testing.T) { + queue := &Queue{ + buffers: list.New(), + Snapshot: []collector.MetricsDutum{ + { + IfIndex: 1, + Mib: "", + IfName: "eth0", + Value: 1, + }, + }, + } + newSnapshot := []collector.MetricsDutum{ + { + IfIndex: 1, + Mib: "", + IfName: "eth0", + Value: 1, + }, + } + queue.Enqueue(newSnapshot) + + if !reflect.DeepEqual(queue.Snapshot, newSnapshot) { + t.Error("replace Snapshot is invalid") + } + }) + + t.Run("calcurate", func(t *testing.T) { + queue := &Queue{ + buffers: list.New(), + Snapshot: []collector.MetricsDutum{ + { + IfIndex: 1, + Mib: "ifHCInOctets", + IfName: "eth0", + Value: 1, + }, + { + IfIndex: 1, + Mib: "ifHCOutOctets", + IfName: "eth0", + Value: math.MaxUint64, + }, + { + IfIndex: 1, + Mib: "ifInDiscards", + IfName: "eth0", + Value: 0, + }, + }, + } + newSnapshot := []collector.MetricsDutum{ + { + IfIndex: 1, + Mib: "ifHCInOctets", + IfName: "eth0", + Value: 1, + }, + { + IfIndex: 1, + Mib: "ifHCOutOctets", + IfName: "eth0", + Value: 60, + }, + { + IfIndex: 1, + Mib: "ifInDiscards", + IfName: "eth0", + Value: 1, + }, + } + queue.Enqueue(newSnapshot) + + e := queue.buffers.Front() + + actual := e.Value.([](*mackerel.MetricValue)) + expected := []*mackerel.MetricValue{ + { + Name: "interface.eth0.rxBytes.delta", + Time: time.Now().Unix(), + Value: uint64(0), + }, + { + Name: "interface.eth0.txBytes.delta", + Time: time.Now().Unix(), + Value: uint64(1), + }, + { + Name: "custom.interface.ifInDiscards.eth0", + Time: time.Now().Unix(), + Value: uint64(1), + }, + } + + if diff := cmp.Diff(actual, expected); diff != "" { + t.Errorf("failed transform %s", diff) + } + }) + +} diff --git a/mackerel/mackerel_test.go b/mackerel/mackerel_test.go new file mode 100644 index 0000000..466de55 --- /dev/null +++ b/mackerel/mackerel_test.go @@ -0,0 +1,244 @@ +package mackerel + +import ( + "container/list" + "context" + "errors" + "reflect" + "testing" + + "github.com/mackerelio/mackerel-client-go" +) + +type mackerelClientMock struct { + createParam mackerel.CreateHostParam + updateParam mackerel.UpdateHostParam + graphDef []*mackerel.GraphDefsParam + hostID string + metricValues []*mackerel.MetricValue + + returnHostID string + returnError error + returnErrorGraphDef error +} + +func (m *mackerelClientMock) CreateHost(param *mackerel.CreateHostParam) (string, error) { + m.createParam = *param + return m.returnHostID, m.returnError +} +func (m *mackerelClientMock) UpdateHost(hostID string, param *mackerel.UpdateHostParam) (string, error) { + m.updateParam = *param + return m.returnHostID, m.returnError +} +func (m *mackerelClientMock) CreateGraphDefs(payloads []*mackerel.GraphDefsParam) error { + m.graphDef = payloads + return m.returnErrorGraphDef +} +func (m *mackerelClientMock) PostHostMetricValuesByHostID(hostID string, metricValues []*mackerel.MetricValue) error { + m.hostID = hostID + m.metricValues = metricValues + return m.returnError +} + +func TestInit(t *testing.T) { + id := "1234567890" + createHost := mackerel.CreateHostParam{ + Name: "hostname", + Interfaces: []mackerel.Interface{ + { + Name: "main", + IPv4Addresses: []string{"192.0.2.1"}, + }, + }, + } + updateHost := mackerel.UpdateHostParam{ + Name: "hostname", + Interfaces: []mackerel.Interface{ + { + Name: "main", + IPv4Addresses: []string{"192.0.2.2"}, + }, + }, + } + e := errors.New("error") + tests := []struct { + name string + expectedCreateParam mackerel.CreateHostParam + expectedUpdateParam mackerel.UpdateHostParam + expectedError error + expectedGraphDef []*mackerel.GraphDefsParam + hostID string + returnHostID *string + queue *Queue + mock *mackerelClientMock + }{ + { + name: "create host when hostID is empty", + expectedCreateParam: createHost, + queue: &Queue{ + buffers: list.New(), + name: "hostname", + targetAddr: "192.0.2.1", + }, + returnHostID: &id, + mock: &mackerelClientMock{ + returnHostID: "1234567890", + }, + expectedGraphDef: graphDefs, + }, + { + name: "update host when hostID is exist", + expectedUpdateParam: updateHost, + queue: &Queue{ + buffers: list.New(), + name: "hostname", + targetAddr: "192.0.2.2", + hostID: "0987654321", + }, + mock: &mackerelClientMock{}, + expectedGraphDef: graphDefs, + }, + { + name: "create host is error", + expectedCreateParam: createHost, + expectedError: e, + queue: &Queue{ + buffers: list.New(), + name: "hostname", + targetAddr: "192.0.2.1", + }, + mock: &mackerelClientMock{ + returnError: e, + }, + expectedGraphDef: nil, + }, + { + name: "update host is error", + expectedUpdateParam: updateHost, + expectedError: e, + queue: &Queue{ + buffers: list.New(), + name: "hostname", + targetAddr: "192.0.2.2", + hostID: "0987654321", + }, + mock: &mackerelClientMock{ + returnError: e, + }, + expectedGraphDef: nil, + }, + { + name: "createGraphDef is error", + expectedUpdateParam: updateHost, + expectedError: e, + queue: &Queue{ + buffers: list.New(), + name: "hostname", + targetAddr: "192.0.2.2", + hostID: "0987654321", + }, + mock: &mackerelClientMock{ + returnErrorGraphDef: e, + }, + expectedGraphDef: graphDefs, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.queue.client = tc.mock + newHostID, err := tc.queue.Init() + if !errors.Is(err, tc.expectedError) { + t.Error("invalid error") + } + if !reflect.DeepEqual(newHostID, tc.returnHostID) { + t.Error("newHostID is invalid") + } + if !reflect.DeepEqual(tc.mock.createParam, tc.expectedCreateParam) { + t.Error("createParam is invalid") + } + if !reflect.DeepEqual(tc.mock.updateParam, tc.expectedUpdateParam) { + t.Error("updateParam is invalid") + } + if !reflect.DeepEqual(tc.mock.graphDef, tc.expectedGraphDef) { + t.Error("CreateGraphDefs is invalid") + } + }) + } + +} + +func TestSendToMackerel(t *testing.T) { + ctx := context.Background() + + t.Run("when empty queue", func(t *testing.T) { + mock := &mackerelClientMock{} + queue := &Queue{ + buffers: list.New(), + hostID: "0987654321", + client: mock, + } + + queue.sendToMackerel(ctx) + + if mock.hostID != "" { + t.Error("invalid get hostID") + } + }) + + t.Run("when queue length = 1", func(t *testing.T) { + mock := &mackerelClientMock{} + queue := &Queue{ + buffers: list.New(), + hostID: "0987654321", + client: mock, + } + + queue.buffers.PushBack([]*mackerel.MetricValue{ + { + Name: "foo", + }, + }) + + queue.sendToMackerel(ctx) + + if mock.hostID == "" { + t.Error("invalid need hostID") + } + + if queue.buffers.Len() != 0 { + t.Error("invalid queue length") + } + }) + + t.Run("when queue length = 2", func(t *testing.T) { + mock := &mackerelClientMock{} + queue := &Queue{ + buffers: list.New(), + hostID: "0987654321", + client: mock, + } + + queue.buffers.PushBack([]*mackerel.MetricValue{ + { + Name: "foo", + }, + }) + queue.buffers.PushBack([]*mackerel.MetricValue{ + { + Name: "foo", + }, + }) + + queue.sendToMackerel(ctx) + + if mock.hostID == "" { + t.Error("invalid need hostID") + } + + if queue.buffers.Len() != 1 { + t.Error("invalid queue length") + } + }) + +} From 1061a0e5b8874ec43ab1258ad4725e2fd27c3785 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 12:35:14 +0000 Subject: [PATCH 66/77] fix func --- snmp/snmp.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/snmp/snmp.go b/snmp/snmp.go index bae6220..b76404e 100644 --- a/snmp/snmp.go +++ b/snmp/snmp.go @@ -61,7 +61,7 @@ func (s *SNMP) GetInterfaceNumber() (uint64, error) { func (s *SNMP) BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { kv := make(map[uint64]string, length) err := s.handler.BulkWalk(MIBifDescr, func(pdu gosnmp.SnmpPDU) error { - index, err := captureIfIndex(MIBifDescr, pdu.Name) + index, err := captureIfIndex(pdu.Name) if err != nil { return err } @@ -82,7 +82,7 @@ func (s *SNMP) BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error func (s *SNMP) BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { kv := make(map[uint64]bool, length) err := s.handler.BulkWalk(MIBifOperStatus, func(pdu gosnmp.SnmpPDU) error { - index, err := captureIfIndex(MIBifOperStatus, pdu.Name) + index, err := captureIfIndex(pdu.Name) if err != nil { return err } @@ -108,7 +108,7 @@ func (s *SNMP) BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) func (s *SNMP) BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { kv := make(map[uint64]uint64, length) err := s.handler.BulkWalk(oid, func(pdu gosnmp.SnmpPDU) error { - index, err := captureIfIndex(oid, pdu.Name) + index, err := captureIfIndex(pdu.Name) if err != nil { return err } @@ -126,7 +126,7 @@ func (s *SNMP) BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { return kv, nil } -func captureIfIndex(oid, name string) (uint64, error) { - indexStr := strings.Replace(name, "."+oid+".", "", 1) - return strconv.ParseUint(indexStr, 10, 64) +func captureIfIndex(name string) (uint64, error) { + sl := strings.Split(name, ".") + return strconv.ParseUint(sl[len(sl)-1], 10, 64) } From aadae8a5735cf88bc1eca54b32a4fab00e15f224 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 12:39:13 +0000 Subject: [PATCH 67/77] organize error --- snmp/snmp.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/snmp/snmp.go b/snmp/snmp.go index b76404e..fd8e242 100644 --- a/snmp/snmp.go +++ b/snmp/snmp.go @@ -44,6 +44,12 @@ func (s *SNMP) Close() error { return s.handler.Conn.Close() } +var ( + errGetInterfaceNumber = errors.New("cant get interface number") + errParseInterfaceName = errors.New("cant parse interface name") + errParseError = errors.New("cant parse value.") +) + func (s *SNMP) GetInterfaceNumber() (uint64, error) { result, err := s.handler.Get([]string{MIBifNumber}) if err != nil { @@ -52,7 +58,7 @@ func (s *SNMP) GetInterfaceNumber() (uint64, error) { variable := result.Variables[0] switch variable.Type { case gosnmp.OctetString: - return 0, errors.New("cant get interface number") + return 0, errGetInterfaceNumber default: return gosnmp.ToBigInt(variable.Value).Uint64(), nil } @@ -69,7 +75,7 @@ func (s *SNMP) BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error case gosnmp.OctetString: kv[index] = string(pdu.Value.([]byte)) default: - return errors.New("cant parse interface name.") + return errParseInterfaceName } return nil }) @@ -88,7 +94,7 @@ func (s *SNMP) BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) } switch pdu.Type { case gosnmp.OctetString: - return errors.New("cant parse value.") + return errParseError default: tmp := gosnmp.ToBigInt(pdu.Value).Uint64() if tmp != 2 { @@ -114,7 +120,7 @@ func (s *SNMP) BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { } switch pdu.Type { case gosnmp.OctetString: - return errors.New("cant parse value.") + return errParseError default: kv[index] = gosnmp.ToBigInt(pdu.Value).Uint64() } From 0e955fa455651c4f7a898919f892998b8c23aa05 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 12:57:15 +0000 Subject: [PATCH 68/77] snmp wrap handler --- snmp/handler.go | 41 +++++++++++++++++++++++++++++++++++++++++ snmp/snmp.go | 18 +++--------------- 2 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 snmp/handler.go diff --git a/snmp/handler.go b/snmp/handler.go new file mode 100644 index 0000000..3ed04d3 --- /dev/null +++ b/snmp/handler.go @@ -0,0 +1,41 @@ +package snmp + +import ( + "context" + "time" + + "github.com/gosnmp/gosnmp" +) + +type Handler interface { + Get(oids []string) (result *gosnmp.SnmpPacket, err error) + BulkWalk(rootOid string, walkFn gosnmp.WalkFunc) error + + Connect() error + Close() error +} + +type snmpHandler struct { + gosnmp.GoSNMP +} + +func NewHandler(ctx context.Context, target, community string) Handler { + return &snmpHandler{ + gosnmp.GoSNMP{ + Context: ctx, + Target: target, + Port: 161, + Transport: "udp", + Community: community, + Version: gosnmp.Version2c, + Timeout: time.Duration(2) * time.Second, + Retries: 3, + ExponentialTimeout: true, + MaxOids: gosnmp.MaxOids, + }, + } +} + +func (x *snmpHandler) Close() error { + return x.GoSNMP.Conn.Close() +} diff --git a/snmp/snmp.go b/snmp/snmp.go index fd8e242..dbd9de3 100644 --- a/snmp/snmp.go +++ b/snmp/snmp.go @@ -5,7 +5,6 @@ import ( "errors" "strconv" "strings" - "time" "github.com/gosnmp/gosnmp" ) @@ -17,22 +16,11 @@ const ( ) type SNMP struct { - handler *gosnmp.GoSNMP + handler Handler } func Init(ctx context.Context, target, community string) (*SNMP, error) { - g := &gosnmp.GoSNMP{ - Context: ctx, - Target: target, - Port: 161, - Transport: "udp", - Community: community, - Version: gosnmp.Version2c, - Timeout: time.Duration(2) * time.Second, - Retries: 3, - ExponentialTimeout: true, - MaxOids: gosnmp.MaxOids, - } + g := NewHandler(ctx, target, community) err := g.Connect() if err != nil { return nil, err @@ -41,7 +29,7 @@ func Init(ctx context.Context, target, community string) (*SNMP, error) { } func (s *SNMP) Close() error { - return s.handler.Conn.Close() + return s.handler.Close() } var ( From 23d0067cda60d38a0e3722b38c9d7010390ce199 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 14:09:05 +0000 Subject: [PATCH 69/77] snmp package tests --- snmp/snmp.go | 5 ++ snmp/snmp_test.go | 158 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 snmp/snmp_test.go diff --git a/snmp/snmp.go b/snmp/snmp.go index dbd9de3..9d2fcaa 100644 --- a/snmp/snmp.go +++ b/snmp/snmp.go @@ -80,6 +80,11 @@ func (s *SNMP) BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) if err != nil { return err } + /* + up(1) + down(2) + testing(3) + */ switch pdu.Type { case gosnmp.OctetString: return errParseError diff --git a/snmp/snmp_test.go b/snmp/snmp_test.go new file mode 100644 index 0000000..008ad86 --- /dev/null +++ b/snmp/snmp_test.go @@ -0,0 +1,158 @@ +package snmp + +import ( + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/gosnmp/gosnmp" +) + +type mockHandler struct { + oids []string + rootOid string + result *gosnmp.SnmpPacket + pdus []gosnmp.SnmpPDU +} + +func (m *mockHandler) Get(oids []string) (result *gosnmp.SnmpPacket, err error) { + m.oids = oids + return m.result, nil +} + +func (m *mockHandler) BulkWalk(rootOid string, walkFn gosnmp.WalkFunc) error { + m.rootOid = rootOid + for i := range m.pdus { + if err := walkFn(m.pdus[i]); err != nil { + return err + } + } + return nil +} + +func (m *mockHandler) Connect() error { + return nil +} + +func (m *mockHandler) Close() error { + return nil +} + +func TestGetInterfaceNumber(t *testing.T) { + m := mockHandler{ + result: &gosnmp.SnmpPacket{ + Variables: []gosnmp.SnmpPDU{ + { + Value: uint64(3), + }, + }, + }, + } + s := &SNMP{handler: &m} + + actual, err := s.GetInterfaceNumber() + if err != nil { + t.Error("failed raised error") + } + if actual != 3 { + t.Error("invalid result") + } + if !reflect.DeepEqual(m.oids, []string{MIBifNumber}) { + t.Error("invalid argument") + } +} + +func TestBulkWalkGetInterfaceName(t *testing.T) { + m := mockHandler{ + pdus: []gosnmp.SnmpPDU{ + { + Name: "1.3.6.1.2.1.2.2.1.2.1", + Value: []byte("lo0"), + Type: gosnmp.OctetString, + }, + { + Name: "1.3.6.1.2.1.2.2.1.2.2", + Value: []byte("eth0"), + Type: gosnmp.OctetString, + }, + }, + } + s := &SNMP{handler: &m} + + actual, err := s.BulkWalkGetInterfaceName(2) + expected := map[uint64]string{ + 1: "lo0", + 2: "eth0", + } + if err != nil { + t.Error("failed raised error") + } + if d := cmp.Diff(actual, expected); d != "" { + t.Error("invalid result") + } + if !reflect.DeepEqual(m.rootOid, MIBifDescr) { + t.Error("invalid argument") + } +} + +func TestBulkWalkGetInterfaceState(t *testing.T) { + m := mockHandler{ + pdus: []gosnmp.SnmpPDU{ + { + Name: "1.3.6.1.2.1.2.2.1.8.1", + Value: 1, + }, + { + Name: "1.3.6.1.2.1.2.2.1.8.2", + Value: 2, + }, + }, + } + s := &SNMP{handler: &m} + + actual, err := s.BulkWalkGetInterfaceState(2) + expected := map[uint64]bool{ + 1: true, + 2: false, + } + if err != nil { + t.Error("failed raised error") + } + if d := cmp.Diff(actual, expected); d != "" { + t.Error("invalid result") + } + if !reflect.DeepEqual(m.rootOid, MIBifOperStatus) { + t.Error("invalid argument") + } +} + +func TestBulkWalk(t *testing.T) { + m := mockHandler{ + pdus: []gosnmp.SnmpPDU{ + { + Name: "1.3.6.1.2.1.2.2.1.10.1", + Value: 1, + }, + { + Name: "1.3.6.1.2.1.2.2.1.10.2", + Value: 2, + }, + }, + } + s := &SNMP{handler: &m} + + actual, err := s.BulkWalk("1.3.6.1.2.1.2.2.1.10", 2) + expected := map[uint64]uint64{ + 1: 1, + 2: 2, + } + if err != nil { + t.Error("failed raised error") + } + if d := cmp.Diff(actual, expected); d != "" { + t.Error("invalid result") + } + if !reflect.DeepEqual(m.rootOid, "1.3.6.1.2.1.2.2.1.10") { + t.Error("invalid argument") + } +} From a03160a589a435eba58923e53918be15082c85fa Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 14:26:31 +0000 Subject: [PATCH 70/77] mib package tests --- mib/mib_test.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 mib/mib_test.go diff --git a/mib/mib_test.go b/mib/mib_test.go new file mode 100644 index 0000000..6dc235a --- /dev/null +++ b/mib/mib_test.go @@ -0,0 +1,64 @@ +package mib + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestValidate(t *testing.T) { + t.Run("all", func(t *testing.T) { + var all []string + actual, err := Validate(all) + if err != nil { + t.Error("invalid raised error") + } + expected := []string{ + "ifOutDiscards", + "ifInErrors", + "ifOutErrors", + "ifHCInOctets", + "ifHCOutOctets", + "ifInDiscards", + } + if d := cmp.Diff( + actual, + expected, + cmpopts.SortSlices(func(i, j string) bool { return i < j }), + ); d != "" { + t.Errorf("invalid results %s", d) + } + }) + + t.Run("some values", func(t *testing.T) { + v := []string{"ifInErrors", "ifHCInOctets"} + actual, err := Validate(v) + if err != nil { + t.Error("invalid raised error") + } + expected := []string{ + "ifInErrors", + "ifHCInOctets", + } + if d := cmp.Diff( + actual, + expected, + cmpopts.SortSlices(func(i, j string) bool { return i < j }), + ); d != "" { + t.Errorf("invalid results %s", d) + } + }) + + t.Run("error", func(t *testing.T) { + v := []string{"aaaaaaaaaaaa"} + result, err := Validate(v) + if err == nil { + t.Error("failed raised error") + } + if result != nil { + t.Errorf("invalid result") + } + }) + +} From 52c3cb4fbbf902b3ad0990d6704e6554cc91aaa9 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 14:38:44 +0000 Subject: [PATCH 71/77] split function --- collector/collector.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/collector/collector.go b/collector/collector.go index c522ddc..1a9278a 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -14,13 +14,24 @@ import ( "github.com/yseto/switch-traffic-to-mackerel/snmp" ) +type snmpClientImpl interface { + BulkWalk(oid string, length uint64) (map[uint64]uint64, error) + BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) + BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) + Close() error + GetInterfaceNumber() (uint64, error) +} + func Do(ctx context.Context, c *config.Config) ([]MetricsDutum, error) { snmpClient, err := snmp.Init(ctx, c.Target, c.Community) if err != nil { return nil, err } defer snmpClient.Close() + return do(ctx, snmpClient, c) +} +func do(ctx context.Context, snmpClient snmpClientImpl, c *config.Config) ([]MetricsDutum, error) { ifNumber, err := snmpClient.GetInterfaceNumber() if err != nil { return nil, err From 9b516cad7d4a63c8bfd582e8d80ea1b4f1e0bee3 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 15:31:02 +0000 Subject: [PATCH 72/77] divide file --- collector/collector.go | 18 ------------------ collector/debugprint.go | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 collector/debugprint.go diff --git a/collector/collector.go b/collector/collector.go index 1a9278a..06e4dab 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -2,12 +2,6 @@ package collector import ( "context" - "fmt" - "sort" - "strings" - "time" - - "github.com/maruel/natural" "github.com/yseto/switch-traffic-to-mackerel/config" "github.com/yseto/switch-traffic-to-mackerel/mib" @@ -80,15 +74,3 @@ func do(ctx context.Context, snmpClient snmpClientImpl, c *config.Config) ([]Met } return metrics, nil } - -func debugPrint(dutum []MetricsDutum) { - var dutumStr []string - for i := range dutum { - dutumStr = append(dutumStr, dutum[i].String()) - } - sort.Sort(natural.StringSlice(dutumStr)) - // debug print. - fmt.Print("\033[H\033[2J") - fmt.Println(time.Now().Format(time.ANSIC)) - fmt.Println(strings.Join(dutumStr, "\n")) -} diff --git a/collector/debugprint.go b/collector/debugprint.go new file mode 100644 index 0000000..0401b43 --- /dev/null +++ b/collector/debugprint.go @@ -0,0 +1,22 @@ +package collector + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/maruel/natural" +) + +func debugPrint(dutum []MetricsDutum) { + var dutumStr []string + for i := range dutum { + dutumStr = append(dutumStr, dutum[i].String()) + } + sort.Sort(natural.StringSlice(dutumStr)) + // debug print. + fmt.Print("\033[H\033[2J") + fmt.Println(time.Now().Format(time.ANSIC)) + fmt.Println(strings.Join(dutumStr, "\n")) +} From 5d7aefad8ec1969d73768669ae86ee2b8c6b6b38 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 15:31:34 +0000 Subject: [PATCH 73/77] collector tests --- collector/collector_test.go | 164 ++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 collector/collector_test.go diff --git a/collector/collector_test.go b/collector/collector_test.go new file mode 100644 index 0000000..34bd1a7 --- /dev/null +++ b/collector/collector_test.go @@ -0,0 +1,164 @@ +package collector + +import ( + "context" + "errors" + "regexp" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/yseto/switch-traffic-to-mackerel/config" +) + +type mockSnmpClient struct { +} + +var errInvalid = errors.New("invalid error") + +func (m *mockSnmpClient) BulkWalk(oid string, length uint64) (map[uint64]uint64, error) { + switch oid { + case "1.3.6.1.2.1.31.1.1.1.6": + return map[uint64]uint64{ + 1: 60, + 2: 60, + 3: 60, + 4: 60, + }, nil + case "1.3.6.1.2.1.31.1.1.1.10": + return map[uint64]uint64{ + 1: 120, + 2: 120, + 3: 120, + 4: 120, + }, nil + default: + return nil, errInvalid + } +} +func (m *mockSnmpClient) BulkWalkGetInterfaceName(length uint64) (map[uint64]string, error) { + return map[uint64]string{ + 1: "lo0", + 2: "eth0", + 3: "eth1", + 4: "eth2", + }, nil +} +func (m *mockSnmpClient) BulkWalkGetInterfaceState(length uint64) (map[uint64]bool, error) { + return map[uint64]bool{ + 1: true, + 2: false, + 3: true, + 4: true, + }, nil +} +func (m *mockSnmpClient) Close() error { + return nil +} +func (m *mockSnmpClient) GetInterfaceNumber() (uint64, error) { + return 4, nil +} + +func TestDo(t *testing.T) { + ctx := context.Background() + + t.Run("non skip", func(t *testing.T) { + c := &config.Config{ + MIBs: []string{"ifHCInOctets", "ifHCOutOctets"}, + } + actual, err := do(ctx, &mockSnmpClient{}, c) + if err != nil { + t.Error("invalid raised error") + } + expected := []MetricsDutum{ + {IfIndex: 1, Mib: "ifHCInOctets", IfName: "lo0", Value: 60}, + {IfIndex: 2, Mib: "ifHCInOctets", IfName: "eth0", Value: 60}, + {IfIndex: 3, Mib: "ifHCInOctets", IfName: "eth1", Value: 60}, + {IfIndex: 4, Mib: "ifHCInOctets", IfName: "eth2", Value: 60}, + {IfIndex: 1, Mib: "ifHCOutOctets", IfName: "lo0", Value: 120}, + {IfIndex: 2, Mib: "ifHCOutOctets", IfName: "eth0", Value: 120}, + {IfIndex: 3, Mib: "ifHCOutOctets", IfName: "eth1", Value: 120}, + {IfIndex: 4, Mib: "ifHCOutOctets", IfName: "eth2", Value: 120}, + } + if d := cmp.Diff( + actual, + expected, + cmpopts.SortSlices(func(i, j MetricsDutum) bool { return i.String() < j.String() }), + ); d != "" { + t.Errorf("invalid result %s", d) + } + }) + + t.Run("skip include", func(t *testing.T) { + c := &config.Config{ + MIBs: []string{"ifHCInOctets", "ifHCOutOctets"}, + IncludeRegexp: regexp.MustCompile("lo?"), + } + actual, err := do(ctx, &mockSnmpClient{}, c) + if err != nil { + t.Error("invalid raised error") + } + expected := []MetricsDutum{ + {IfIndex: 1, Mib: "ifHCInOctets", IfName: "lo0", Value: 60}, + {IfIndex: 1, Mib: "ifHCOutOctets", IfName: "lo0", Value: 120}, + } + if d := cmp.Diff( + actual, + expected, + cmpopts.SortSlices(func(i, j MetricsDutum) bool { return i.String() < j.String() }), + ); d != "" { + t.Errorf("invalid result %s", d) + } + }) + t.Run("skip exclude", func(t *testing.T) { + c := &config.Config{ + MIBs: []string{"ifHCInOctets", "ifHCOutOctets"}, + ExcludeRegexp: regexp.MustCompile("0$"), + } + actual, err := do(ctx, &mockSnmpClient{}, c) + if err != nil { + t.Error("invalid raised error") + } + expected := []MetricsDutum{ + {IfIndex: 3, Mib: "ifHCInOctets", IfName: "eth1", Value: 60}, + {IfIndex: 4, Mib: "ifHCInOctets", IfName: "eth2", Value: 60}, + {IfIndex: 3, Mib: "ifHCOutOctets", IfName: "eth1", Value: 120}, + {IfIndex: 4, Mib: "ifHCOutOctets", IfName: "eth2", Value: 120}, + } + if d := cmp.Diff( + actual, + expected, + cmpopts.SortSlices(func(i, j MetricsDutum) bool { return i.String() < j.String() }), + ); d != "" { + t.Errorf("invalid result %s", d) + } + }) + + t.Run("non skip", func(t *testing.T) { + c := &config.Config{ + MIBs: []string{"ifHCInOctets", "ifHCOutOctets"}, + SkipDownLinkState: true, + } + actual, err := do(ctx, &mockSnmpClient{}, c) + if err != nil { + t.Error("invalid raised error") + } + expected := []MetricsDutum{ + {IfIndex: 1, Mib: "ifHCInOctets", IfName: "lo0", Value: 60}, + {IfIndex: 3, Mib: "ifHCInOctets", IfName: "eth1", Value: 60}, + {IfIndex: 4, Mib: "ifHCInOctets", IfName: "eth2", Value: 60}, + {IfIndex: 1, Mib: "ifHCOutOctets", IfName: "lo0", Value: 120}, + {IfIndex: 3, Mib: "ifHCOutOctets", IfName: "eth1", Value: 120}, + {IfIndex: 4, Mib: "ifHCOutOctets", IfName: "eth2", Value: 120}, + } + if d := cmp.Diff( + actual, + expected, + cmpopts.SortSlices(func(i, j MetricsDutum) bool { return i.String() < j.String() }), + ); d != "" { + t.Errorf("invalid result %s", d) + } + }) + +} From 0b7ad7d0b5ac04244b32750f1600ee4ea22efc68 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 15:32:13 +0000 Subject: [PATCH 74/77] debugprint tests --- collector/debugprint_test.go | 60 ++++++++++++++++++++++++ collector/testdata/TestDebugPrint.golden | 9 ++++ 2 files changed, 69 insertions(+) create mode 100644 collector/debugprint_test.go create mode 100644 collector/testdata/TestDebugPrint.golden diff --git a/collector/debugprint_test.go b/collector/debugprint_test.go new file mode 100644 index 0000000..6cc0ea6 --- /dev/null +++ b/collector/debugprint_test.go @@ -0,0 +1,60 @@ +package collector + +import ( + "context" + "flag" + "io" + "os" + "testing" + "time" + + "github.com/tenntenn/golden" + "github.com/tenntenn/testtime" + "github.com/yseto/switch-traffic-to-mackerel/config" +) + +var ( + flagUpdate bool + goldenDir string = "./testdata/" +) + +func init() { + flag.BoolVar(&flagUpdate, "update", true, "update golden files") +} + +func TestDebugPrint(t *testing.T) { + ctx := context.Background() + + got := capture(func() { + testtime.SetTime(t, time.Unix(1, 0)) + + c := &config.Config{ + MIBs: []string{"ifHCInOctets", "ifHCOutOctets"}, + Debug: true, + } + _, err := do(ctx, &mockSnmpClient{}, c) + if err != nil { + t.Error("invalid raised error") + } + }) + + if diff := golden.Check(t, flagUpdate, goldenDir, t.Name(), got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } +} + +func capture(f func()) string { + writer := os.Stdout + defer func() { + os.Stdout = writer + }() + + r, w, _ := os.Pipe() + os.Stdout = w + + f() + + w.Close() + out, _ := io.ReadAll(r) + return string(out) +} diff --git a/collector/testdata/TestDebugPrint.golden b/collector/testdata/TestDebugPrint.golden new file mode 100644 index 0000000..4ca50b3 --- /dev/null +++ b/collector/testdata/TestDebugPrint.golden @@ -0,0 +1,9 @@ +Thu Jan 1 00:00:01 1970 +1 lo0 ifHCInOctets 60 +1 lo0 ifHCOutOctets 120 +2 eth0 ifHCInOctets 60 +2 eth0 ifHCOutOctets 120 +3 eth1 ifHCInOctets 60 +3 eth1 ifHCOutOctets 120 +4 eth2 ifHCInOctets 60 +4 eth2 ifHCOutOctets 120 From 1485de78f62a8e7bf279d99697ae99c1d52f7284 Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 15:32:59 +0000 Subject: [PATCH 75/77] update Makefile --- Makefile | 4 ++++ go.mod | 9 ++++++++- go.sum | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3180574..f771fd4 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ all: build deps: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 + go install github.com/tenntenn/testtime/cmd/testtime@latest fmt: go fmt ./... @@ -13,3 +14,6 @@ lint: build: go build +test: + go test -overlay=`testtime` -v ./... + diff --git a/go.mod b/go.mod index 2e88614..e883c7a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,14 @@ require ( github.com/gosnmp/gosnmp v1.35.0 github.com/mackerelio/mackerel-client-go v0.24.0 github.com/maruel/natural v1.1.0 + github.com/tenntenn/golden v0.4.0 + github.com/tenntenn/testtime v0.2.2 gopkg.in/yaml.v3 v3.0.1 ) -require github.com/davecgh/go-spew v1.1.1 // indirect +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/josharian/mapfs v0.0.0-20210615234106-095c008854e6 // indirect + github.com/josharian/txtarfs v0.0.0-20210615234325-77aca6df5bca // indirect + golang.org/x/tools v0.1.5 // indirect +) diff --git a/go.sum b/go.sum index 2b65850..3bb52f2 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gosnmp/gosnmp v1.35.0 h1:EuWWNPxTCdAUx2/NbQcSa3WdNxjzpy4Phv57b4MWpJM= github.com/gosnmp/gosnmp v1.35.0/go.mod h1:2AvKZ3n9aEl5TJEo/fFmf/FGO4Nj4cVeEc5yuk88CYc= +github.com/josharian/mapfs v0.0.0-20210615234106-095c008854e6 h1:c+ctPFdISggaSNCfU1IueNBAsqetJSvMcpQlT+0OVdY= +github.com/josharian/mapfs v0.0.0-20210615234106-095c008854e6/go.mod h1:Rv/momJI8DgrWnBZip+SgagpcgORIZQE5SERlxNb8LY= +github.com/josharian/txtarfs v0.0.0-20210615234325-77aca6df5bca h1:a8xeK4GsWLE4LYo5VI4u1Cn7ZvT1NtXouXR3DdKLB8Q= +github.com/josharian/txtarfs v0.0.0-20210615234325-77aca6df5bca/go.mod h1:UbC32ft9G/jG+sZI8wLbIBNIrYr7vp/yqMDa9SxVBNA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/mackerelio/mackerel-client-go v0.24.0 h1:Y9FeTgrQlDdtLU7FtMM6hd5mL5T4TvGCVUY0LH+bceQ= github.com/mackerelio/mackerel-client-go v0.24.0/go.mod h1:b4qVMQi+w4rxtKQIFycLWXNBtIi9d0r571RzYmg/aXo= @@ -11,6 +15,34 @@ github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/tenntenn/golden v0.4.0 h1:ghZvhG1A3zp3JKBBywTt4GRJ4ieCtZi6Xw20kt8fpy0= +github.com/tenntenn/golden v0.4.0/go.mod h1:0xI/4lpoHR65AUTmd1RKR9S1Uv0JR3yR2Q1Ob2bKqQA= +github.com/tenntenn/testtime v0.2.2 h1:y6K00BUNg7cRE9WpkBX/Bn+WgmV5/a3hsw7xGNyF2p0= +github.com/tenntenn/testtime v0.2.2/go.mod h1:gXZpxnMoBEV+JZwooprQ65lIbR2Kzk5PpP/deHMn+Is= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 88d7aa5275d9807ac42c260bb94ece9562fb6f8c Mon Sep 17 00:00:00 2001 From: yseto Date: Fri, 17 Mar 2023 15:38:44 +0000 Subject: [PATCH 76/77] added ci tests --- .github/workflows/test.yml | 32 ++++++++++++++++++++++++++++++++ Makefile | 7 +++++-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e6c9afa --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,32 @@ +--- +name: test + +on: + push: + branches: + - main + tags: + - v* + pull_request: +env: + DEBIAN_FRONTEND: noninteractive +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: golangci/golangci-lint-action@v3 + test: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: "1.19.x" + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - run: make deps-test test diff --git a/Makefile b/Makefile index f771fd4..2238ea8 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,10 @@ all: build -deps: +deps-lint: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 + +deps-test: go install github.com/tenntenn/testtime/cmd/testtime@latest fmt: @@ -14,6 +16,7 @@ lint: build: go build -test: + +test: go test -overlay=`testtime` -v ./... From 9f2dc92d788f0d1c52a522943b20fdf7505a9125 Mon Sep 17 00:00:00 2001 From: yseto Date: Sat, 18 Mar 2023 08:07:50 +0000 Subject: [PATCH 77/77] reduce log --- mackerel/mackerel.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/mackerel/mackerel.go b/mackerel/mackerel.go index 61b98a1..9904349 100644 --- a/mackerel/mackerel.go +++ b/mackerel/mackerel.go @@ -118,8 +118,6 @@ func (q *Queue) sendToMackerel(ctx context.Context) { if err != nil { log.Println(err) return - } else { - log.Println("success") } q.Lock() q.buffers.Remove(e)