Skip to content

Commit 6ab32de

Browse files
author
Jeffrey Koehler
committed
update to output errors as metrics
1 parent 0ac59f1 commit 6ab32de

File tree

7 files changed

+96
-21
lines changed

7 files changed

+96
-21
lines changed

cmd/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func main() {
127127

128128
time.Sleep(time.Millisecond * 100)
129129

130-
d1.StopProcessing(context.Background(), "0", "1", "8")
130+
_ = d1.StopProcessing(context.Background(), "0", "1", "8")
131131
time.Sleep(time.Millisecond * 100)
132132

133133
i, _ = d1.GetWork(context.Background())

go.mod

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module github.com/streemtech/divider
22

3-
go 1.23
3+
go 1.24.0
4+
5+
toolchain go1.24.2
46

57
require (
68
github.com/google/uuid v1.6.0
@@ -10,17 +12,24 @@ require (
1012
)
1113

1214
require (
15+
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
1316
github.com/beorn7/perks v1.0.1 // indirect
1417
github.com/cespare/xxhash/v2 v2.3.0 // indirect
1518
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
16-
github.com/k0kubun/pp/v3 v3.2.0 // indirect
17-
github.com/mattn/go-colorable v0.1.13 // indirect
18-
github.com/mattn/go-isatty v0.0.16 // indirect
1919
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
2020
github.com/prometheus/client_model v0.6.1 // indirect
2121
github.com/prometheus/common v0.60.0 // indirect
2222
github.com/prometheus/procfs v0.15.1 // indirect
23-
golang.org/x/sys v0.26.0 // indirect
24-
golang.org/x/text v0.18.0 // indirect
23+
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
24+
golang.org/x/mod v0.24.0 // indirect
25+
golang.org/x/sync v0.13.0 // indirect
26+
golang.org/x/sys v0.32.0 // indirect
27+
golang.org/x/tools v0.32.0 // indirect
2528
google.golang.org/protobuf v1.35.1 // indirect
29+
honnef.co/go/tools v0.6.1 // indirect
30+
)
31+
32+
tool (
33+
golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
34+
honnef.co/go/tools/cmd/staticcheck
2635
)

go.sum

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
2+
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
13
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
24
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
35
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
@@ -14,14 +16,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
1416
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
1517
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
1618
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
17-
github.com/k0kubun/pp/v3 v3.2.0 h1:h33hNTZ9nVFNP3u2Fsgz8JXiF5JINoZfFq4SvKJwNcs=
18-
github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapdz1EwA=
1919
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
2020
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
21-
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
22-
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
23-
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
24-
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
2521
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
2622
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
2723
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -40,12 +36,19 @@ github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0
4036
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
4137
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
4238
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
43-
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
44-
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
45-
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
46-
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
47-
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
39+
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
40+
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
41+
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
42+
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
43+
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
44+
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
45+
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
46+
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
47+
golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
48+
golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
4849
google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
4950
google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
5051
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
5152
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
53+
honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
54+
honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=

makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ SHELL:=/bin/bash
55

66
go-test:
77
go test ./...
8-
shadow -strict $$(go list ./... | grep -v "api$$")
9-
staticcheck $$(go list ./... | grep -v "api$$")
8+
go run golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow -strict $$(go list ./... | grep -v "api$$")
9+
go run honnef.co/go/tools/cmd/staticcheck@latest $$(go list ./... | grep -v "api$$")
1010
golangci-lint run
1111

1212
newman-tests:

redisconsistent/metrics.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ var StopProcessingTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
7373
Help: "Returns the length of time that stop_processing request took",
7474
Buckets: HistogramBuckets,
7575
}, []string{"divider"})
76+
7677
var StartProcessingKeyCount = promauto.NewCounterVec(prometheus.CounterOpts{
7778
Namespace: "streemtech",
7879
Subsystem: "redis_work_divider",
@@ -93,6 +94,56 @@ var DividerAssignedItemsGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
9394
Help: "Returns the number of items assigned to this divider",
9495
}, []string{"divider"})
9596

97+
// error tracking counters
98+
var AddWorkToDividerError = prometheus.NewCounterVec(prometheus.CounterOpts{
99+
Namespace: "streemtech",
100+
Subsystem: "redis_work_divider",
101+
Name: "error_add_work_to_divider",
102+
Help: "An error was encountered attempting to add work to the stored work that needs to be done.",
103+
}, []string{"divider"})
104+
var CheckWorkInKeyRangeError = prometheus.NewCounterVec(prometheus.CounterOpts{
105+
Namespace: "streemtech",
106+
Subsystem: "redis_work_divider",
107+
Name: "error_check_work_in_key_range",
108+
Help: "an error was encountered attempting to check against storage if a given work key is in the worker's work coverage ranges.",
109+
}, []string{"divider"})
110+
var PublishAddWorkToDividerError = prometheus.NewCounterVec(prometheus.CounterOpts{
111+
Namespace: "streemtech",
112+
Subsystem: "redis_work_divider",
113+
Name: "error_publish_add_work_to_divider",
114+
Help: "An error was encountered attempting to publish the work to the new work stream",
115+
}, []string{"divider"})
116+
var PublishRemoveWorkFromDividerError = prometheus.NewCounterVec(prometheus.CounterOpts{
117+
Namespace: "streemtech",
118+
Subsystem: "redis_work_divider",
119+
Name: "error_remove_work_from_divider",
120+
Help: "An error was encountered attempting to publish the work to the remove work stream",
121+
}, []string{"divider"})
122+
var RectifyWorkError = prometheus.NewCounterVec(prometheus.CounterOpts{
123+
Namespace: "streemtech",
124+
Subsystem: "redis_work_divider",
125+
Name: "error_rectify_work",
126+
Help: "An error was encountered attempting to rectify the worker's work that needs to be processed/completed.",
127+
}, []string{"divider"})
128+
var RemoveWorkFromDividerError = prometheus.NewCounterVec(prometheus.CounterOpts{
129+
Namespace: "streemtech",
130+
Subsystem: "redis_work_divider",
131+
Name: "error_remove_work_from_divider",
132+
Help: "An error was encountered attempting to remove work from the stored work that needs to be done in redis",
133+
}, []string{"divider"})
134+
var StartWorkExternalError = prometheus.NewCounterVec(prometheus.CounterOpts{
135+
Namespace: "streemtech",
136+
Subsystem: "redis_work_divider",
137+
Name: "error_start_work_external",
138+
Help: "an error in the starter callback was encountered attempting to start wokring on one of the desired work",
139+
}, []string{"divider"})
140+
var StopWorkExternalError = prometheus.NewCounterVec(prometheus.CounterOpts{
141+
Namespace: "streemtech",
142+
Subsystem: "redis_work_divider",
143+
Name: "error_stop_work_external",
144+
Help: "an error in the stopper callback was encountered attempting to stop wokring on one of the desired work",
145+
}, []string{"divider"})
146+
96147
func ObserveDuration(metric *prometheus.HistogramVec, name string, taken time.Duration) {
97148
met, err := metric.GetMetricWithLabelValues(name)
98149
if err != nil {

redisconsistent/redis.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,13 @@ func (d *dividerWorker) StopProcessing(ctx context.Context, works ...string) err
142142
start := time.Now()
143143
err := d.storage.RemoveWorkFromDividedWork(ctx, works)
144144
if err != nil {
145+
ObserveInc(RemoveWorkFromDividerError, d.conf.metricsName, 1)
145146
return errors.Wrap(err, "failed to Remove Work From Divided Work")
146147
}
147148
for _, work := range works {
148149
err = d.removeWork.Publish(ctx, work)
149150
if err != nil {
151+
ObserveInc(PublishRemoveWorkFromDividerError, d.conf.metricsName, 1)
150152
return errors.Wrap(err, "failed to publish work removal")
151153
}
152154
}
@@ -164,13 +166,15 @@ func (d *dividerWorker) StartProcessing(ctx context.Context, works ...string) er
164166

165167
err := d.storage.AddWorkToDividedWork(ctx, works)
166168
if err != nil {
169+
ObserveInc(AddWorkToDividerError, d.conf.metricsName, 1)
167170
return errors.Wrap(err, "failed to Add Work To Divided Work")
168171
}
169172
for _, work := range works {
170173
d.conf.logger(ctx).Debug("sending start processing for: "+work, slog.String("divider.id", d.conf.instanceID))
171174

172175
err = d.newWork.Publish(ctx, work)
173176
if err != nil {
177+
ObserveInc(PublishAddWorkToDividerError, d.conf.metricsName, 1)
174178
return errors.Wrap(err, "failed to publish work start")
175179
}
176180
}
@@ -198,6 +202,7 @@ func (d *dividerWorker) newWorkerEvent(ctx context.Context, key string) {
198202
d.conf.logger(ctx).Debug("newWorkerEvent triggered: "+key, slog.String("divider.id", d.conf.instanceID))
199203
err := d.rectifyWork(ctx)
200204
if err != nil {
205+
ObserveInc(RectifyWorkError, d.conf.metricsName, 1)
201206
d.conf.logger(ctx).Error("failed to rectify work", slog.String("err.error", err.Error()), slog.String("divider.id", d.conf.instanceID))
202207
}
203208

@@ -219,6 +224,7 @@ func (d *dividerWorker) removeWorkerEvent(ctx context.Context, key string) {
219224
//rectify all of your work.
220225
err := d.rectifyWork(ctx)
221226
if err != nil {
227+
ObserveInc(RectifyWorkError, d.conf.metricsName, 1)
222228
d.conf.logger(ctx).Error("failed to rectify work", slog.String("err.error", err.Error()), slog.String("divider.id", d.conf.instanceID))
223229
}
224230

@@ -236,12 +242,14 @@ func (d *dividerWorker) newWorkEvent(ctx context.Context, key string) {
236242
for _, v := range d.getWorkerNodeKeys() {
237243
inRange, err := d.storage.CheckWorkInKeyRange(ctx, v, key)
238244
if err != nil {
245+
ObserveInc(CheckWorkInKeyRangeError, d.conf.metricsName, 1)
239246
d.conf.logger(ctx).Error("failed to check if work is in the range of this worker", slog.String("err.error", err.Error()), slog.String("divider.id", d.conf.instanceID))
240247
return
241248
}
242249
if inRange {
243250
err = d.conf.starter(ctx, key)
244251
if err != nil {
252+
ObserveInc(StartWorkExternalError, d.conf.metricsName, 1)
245253
d.conf.logger(ctx).Error("failed to execute starter", slog.String("err.error", err.Error()), slog.String("divider.id", d.conf.instanceID))
246254
return
247255
}
@@ -480,6 +488,8 @@ func (d *dividerWorker) rectifyWork(ctx context.Context) (err error) {
480488
defer can()
481489
err = d.conf.stopper(ctx2, key)
482490
if err != nil {
491+
ObserveInc(StopWorkExternalError, d.conf.metricsName, 1)
492+
483493
d.conf.logger(ctx2).Error("failed to execute stopper, not removing from known work", slog.String("err.error", err.Error()), slog.String("divider.id", d.conf.instanceID))
484494
continue
485495
}
@@ -492,6 +502,7 @@ func (d *dividerWorker) rectifyWork(ctx context.Context) (err error) {
492502
defer can()
493503
err = d.conf.starter(ctx2, key)
494504
if err != nil {
505+
ObserveInc(StartWorkExternalError, d.conf.metricsName, 1)
495506
d.conf.logger(ctx2).Error("failed to execute starter, not adding to known work", slog.String("err.error", err.Error()), slog.String("divider.id", d.conf.instanceID))
496507
continue
497508
}

redisconsistent/workStorage.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,8 @@ func (r *workStorageImpl) GetWorkFromKeyToNextWorkerKey(ctx context.Context, key
181181

182182
//if the scores are properly ordered, return that set of work.
183183
if startScore < nextWorkScore {
184-
dat, err := r.client.ZRangeByScore(ctx, r.getDataKey(), &redis.ZRangeBy{
184+
var dat []string
185+
dat, err = r.client.ZRangeByScore(ctx, r.getDataKey(), &redis.ZRangeBy{
185186
Min: fmt.Sprintf("%f", startScore),
186187
Max: fmt.Sprintf("%f", nextWorkScore),
187188
}).Result()

0 commit comments

Comments
 (0)