Skip to content

Commit

Permalink
fix: use exponential backoff to attempt reconnect to tsdb (#1267)
Browse files Browse the repository at this point in the history
* fix: use exponential backoff to attempt reconnect to tsdb
  • Loading branch information
kasteph authored Nov 20, 2023
1 parent 755a42d commit 6050165
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 2 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ require k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9

require (
github.com/DataDog/zstd v1.4.5
github.com/cenkalti/backoff/v4 v4.2.1
github.com/filecoin-project/go-amt-ipld/v4 v4.1.1-0.20230511024927-ecc426107fac
github.com/hibiken/asynq v0.23.0
github.com/hibiken/asynq/x v0.0.0-20220413130846-5c723f597e01
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQt
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
Expand Down
13 changes: 11 additions & 2 deletions storage/sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"sort"
"strings"

backoff "github.com/cenkalti/backoff/v4"
"github.com/go-pg/pg/v10"
"github.com/go-pg/pg/v10/orm"
"github.com/go-pg/pg/v10/types"
Expand Down Expand Up @@ -228,8 +229,16 @@ func connect(ctx context.Context, opt *pg.Options) (*pg.DB, error) {
// db.AddQueryHook(&pgext.OpenTelemetryHook{})

// Check if connection credentials are valid and PostgreSQL is up and running.
if err := db.Ping(ctx); err != nil {
return nil, fmt.Errorf("ping database: %w", err)
operation := func() error {
if err := db.Ping(ctx); err != nil {
return fmt.Errorf("ping database: %w", err)
}
return nil
}

retryErr := backoff.Retry(operation, backoff.NewExponentialBackOff())
if retryErr != nil {
return nil, fmt.Errorf("ping database retry attempt: %w", retryErr)
}

// Acquire a shared lock on the schema to notify other instances that we are running
Expand Down

0 comments on commit 6050165

Please sign in to comment.