diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8889e9e..6554580 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ name: Build and test on: [push, pull_request] env: - GOLANGCI_LINT_VERSION: v1.54.1 + GOLANGCI_LINT_VERSION: v1.54.2 jobs: build: diff --git a/.gitignore b/.gitignore index 9f11b75..2bf8fbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ .idea/ +sentinel1.conf +sentinel2.conf +sentinel3.conf diff --git a/Makefile b/Makefile index e8c3cea..6abf201 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all +m.PHONY: all all: validate test clean ## Run validates @@ -17,8 +17,14 @@ test: test-start-stack: docker-compose -f script/docker-compose.yml up --wait + PORT=26379 envsubst < ./script/conf/sentinel_template.conf > ./script/conf/sentinel1.conf + PORT=36379 envsubst < ./script/conf/sentinel_template.conf > ./script/conf/sentinel2.conf + PORT=46379 envsubst < ./script/conf/sentinel_template.conf > ./script/conf/sentinel3.conf + docker-compose -f script/docker-compose-sentinel.yml up --wait + ## Clean local data .PHONY: clean clean: docker-compose -f script/docker-compose.yml down + docker-compose -f script/docker-compose-sentinel.yml down $(RM) goverage.report $(shell find . -type f -name *.out) diff --git a/go.mod b/go.mod index 58e6909..3ec10ca 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/kvtools/redis go 1.19 require ( - github.com/go-redis/redis/v8 v8.11.5 github.com/kvtools/valkeyrie v1.0.0 + github.com/redis/go-redis/v9 v9.2.1 github.com/stretchr/testify v1.8.4 ) diff --git a/go.sum b/go.sum index 272ba75..29624ef 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -5,27 +7,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kvtools/valkeyrie v1.0.0 h1:LAITop2wPoYCMitR24GZZsW0b57hmI+ePD18VRTtOf0= github.com/kvtools/valkeyrie v1.0.0/go.mod h1:bDi/OdhJCSbGPMsCgUQl881yuEweKCSItAtTBI+ZjpU= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= +github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/redis.go b/redis.go index 5cdeb5d..d9c060d 100644 --- a/redis.go +++ b/redis.go @@ -10,9 +10,9 @@ import ( "strings" "time" - "github.com/go-redis/redis/v8" "github.com/kvtools/valkeyrie" "github.com/kvtools/valkeyrie/store" + "github.com/redis/go-redis/v9" ) // StoreName the name of the store. @@ -32,6 +32,14 @@ var ( // by sending a signal to the stop chan, // this is used to verify if the operation succeeded. ErrAbortTryLock = errors.New("redis: lock operation aborted") + + // ErrMasterSetMustBeProvided is thrown when Redis Sentinel is enabled + // and the MasterName option is undefined. + ErrMasterSetMustBeProvided = errors.New("master set name must be provided") + + // ErrInvalidRoutesOptions is thrown when Redis Sentinel is enabled + // with RouteByLatency & RouteRandomly options without the ClusterClient. + ErrInvalidRoutesOptions = errors.New("RouteByLatency and RouteRandomly options are only allowed with the ClusterClient") ) // registers Redis to Valkeyrie. @@ -45,6 +53,32 @@ type Config struct { Username string Password string DB int + Sentinel *Sentinel +} + +// Sentinel holds the Redis Sentinel configuration. +type Sentinel struct { + MasterName string + Username string + Password string + + // ClusterClient indicates whether to use the NewFailoverClusterClient to build the client. + ClusterClient bool + + // Allows routing read-only commands to the closest master or replica node. + // This option only works with NewFailoverClusterClient. + RouteByLatency bool + + // Allows routing read-only commands to the random master or replica node. + // This option only works with NewFailoverClusterClient. + RouteRandomly bool + + // Route all commands to replica read-only nodes. + ReplicaOnly bool + + // Use replicas disconnected with master when cannot get connected replicas + // Now, this option only works in RandomReplicaAddr function. + UseDisconnectedReplicas bool } func newStore(ctx context.Context, endpoints []string, options valkeyrie.Config) (store.Store, error) { @@ -58,7 +92,7 @@ func newStore(ctx context.Context, endpoints []string, options valkeyrie.Config) // Store implements the store.Store interface. type Store struct { - client *redis.Client + client redis.UniversalClient script *redis.Script codec Codec } @@ -70,14 +104,54 @@ func New(ctx context.Context, endpoints []string, options *Config) (*Store, erro // NewWithCodec creates a new Redis client with codec config. func NewWithCodec(ctx context.Context, endpoints []string, options *Config, codec Codec) (*Store, error) { - if len(endpoints) > 1 { - return nil, ErrMultipleEndpointsUnsupported + client, err := newClient(endpoints, options) + if err != nil { + return nil, err } - return newRedis(ctx, endpoints, options, codec), nil + return makeStore(ctx, client, codec), nil } -func newRedis(ctx context.Context, endpoints []string, options *Config, codec Codec) *Store { +func newClient(endpoints []string, options *Config) (redis.UniversalClient, error) { + if options != nil && options.Sentinel != nil { + if options.Sentinel.MasterName == "" { + return nil, ErrMasterSetMustBeProvided + } + + if !options.Sentinel.ClusterClient && (options.Sentinel.RouteByLatency || options.Sentinel.RouteRandomly) { + return nil, ErrInvalidRoutesOptions + } + + cfg := &redis.FailoverOptions{ + SentinelAddrs: endpoints, + SentinelUsername: options.Sentinel.Username, + SentinelPassword: options.Sentinel.Password, + MasterName: options.Sentinel.MasterName, + RouteByLatency: options.Sentinel.RouteByLatency, + RouteRandomly: options.Sentinel.RouteRandomly, + ReplicaOnly: options.Sentinel.ReplicaOnly, + UseDisconnectedReplicas: options.Sentinel.UseDisconnectedReplicas, + Username: options.Username, + Password: options.Password, + DB: options.DB, + DialTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + ContextTimeoutEnabled: true, + TLSConfig: options.TLS, + } + + if options.Sentinel.ClusterClient { + return redis.NewFailoverClusterClient(cfg), nil + } + + return redis.NewFailoverClient(cfg), nil + } + + if len(endpoints) > 1 { + return nil, ErrMultipleEndpointsUnsupported + } + opt := &redis.Options{ Addr: endpoints[0], DialTimeout: 5 * time.Second, @@ -93,8 +167,10 @@ func newRedis(ctx context.Context, endpoints []string, options *Config, codec Co } // TODO: use *redis.ClusterClient if we support multiple endpoints. - client := redis.NewClient(opt) + return redis.NewClient(opt), nil +} +func makeStore(ctx context.Context, client redis.UniversalClient, codec Codec) *Store { // Listen to Keyspace events. client.ConfigSet(ctx, "notify-keyspace-events", "KEA") @@ -513,7 +589,7 @@ type subscribe struct { closeCh chan struct{} } -func newSubscribe(ctx context.Context, client *redis.Client, regex string) *subscribe { +func newSubscribe(ctx context.Context, client redis.UniversalClient, regex string) *subscribe { return &subscribe{ pubsub: client.PSubscribe(ctx, regex), closeCh: make(chan struct{}), diff --git a/redis_test.go b/redis_test.go index 509b4c7..98ffa0c 100644 --- a/redis_test.go +++ b/redis_test.go @@ -14,15 +14,16 @@ import ( const testTimeout = 60 * time.Second -const client = "localhost:6379" +const testAddress = "localhost:6379" -func makeRedisClient(t *testing.T) store.Store { +func makeRedisClient(t *testing.T, endpoints []string, config *Config) store.Store { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - kv := newRedis(ctx, []string{client}, nil, nil) + kv, err := NewWithCodec(ctx, endpoints, config, nil) + require.NoError(t, err) // NOTE: please turn on redis's notification // before you using watch/watchtree/lock related features. @@ -35,7 +36,7 @@ func TestRegister(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testTimeout) defer cancel() - kv, err := valkeyrie.NewStore(ctx, StoreName, []string{client}, nil) + kv, err := valkeyrie.NewStore(ctx, StoreName, []string{testAddress}, nil) require.NoError(t, err) assert.NotNil(t, kv) @@ -43,9 +44,49 @@ func TestRegister(t *testing.T) { } func TestRedisStore(t *testing.T) { - kv := makeRedisClient(t) - lockTTL := makeRedisClient(t) - kvTTL := makeRedisClient(t) + kv := makeRedisClient(t, []string{testAddress}, nil) + lockTTL := makeRedisClient(t, []string{testAddress}, nil) + kvTTL := makeRedisClient(t, []string{testAddress}, nil) + + t.Cleanup(func() { + testsuite.RunCleanup(t, kv) + }) + + testsuite.RunTestCommon(t, kv) + testsuite.RunTestAtomic(t, kv) + testsuite.RunTestWatch(t, kv) + testsuite.RunTestLock(t, kv) + testsuite.RunTestLockTTL(t, kv, lockTTL) + testsuite.RunTestTTL(t, kv, kvTTL) +} + +func TestRedisSentinelStore(t *testing.T) { + endpoints := []string{"localhost:26379", "localhost:36379", "localhost:46379"} + config := &Config{Sentinel: &Sentinel{MasterName: "mymaster"}} + + kv := makeRedisClient(t, endpoints, config) + lockTTL := makeRedisClient(t, endpoints, config) + kvTTL := makeRedisClient(t, endpoints, config) + + t.Cleanup(func() { + testsuite.RunCleanup(t, kv) + }) + + testsuite.RunTestCommon(t, kv) + testsuite.RunTestAtomic(t, kv) + testsuite.RunTestWatch(t, kv) + testsuite.RunTestLock(t, kv) + testsuite.RunTestLockTTL(t, kv, lockTTL) + testsuite.RunTestTTL(t, kv, kvTTL) +} + +func TestRedisSentinelStore_withClientCluster(t *testing.T) { + endpoints := []string{"localhost:26379", "localhost:36379", "localhost:46379"} + config := &Config{Sentinel: &Sentinel{MasterName: "mymaster", ClusterClient: true}} + + kv := makeRedisClient(t, endpoints, config) + lockTTL := makeRedisClient(t, endpoints, config) + kvTTL := makeRedisClient(t, endpoints, config) t.Cleanup(func() { testsuite.RunCleanup(t, kv) diff --git a/script/conf/sentinel_template.conf b/script/conf/sentinel_template.conf new file mode 100644 index 0000000..7772fd2 --- /dev/null +++ b/script/conf/sentinel_template.conf @@ -0,0 +1,5 @@ +port ${PORT} +dir "/tmp" +sentinel resolve-hostnames yes +sentinel monitor mymaster master 6380 2 +sentinel deny-scripts-reconfig yes diff --git a/script/docker-compose-sentinel.yml b/script/docker-compose-sentinel.yml new file mode 100644 index 0000000..a735518 --- /dev/null +++ b/script/docker-compose-sentinel.yml @@ -0,0 +1,60 @@ +version: "3.9" + +# when running test local, you can specify the image version using the env var REDIS_VERSION. +# Example: REDIS_VERSION=7.0.5 make + +services: + master: + image: redis:${REDIS_VERSION:-7.2.1} + container_name: redis-master + command: redis-server --port 6380 + ports: + - 6380:6380 + healthcheck: + test: redis-cli -p 6380 ping + node1: + image: redis:${REDIS_VERSION:-7.2.1} + container_name: redis-node-1 + ports: + - 6381:6381 + command: redis-server --port 6381 --slaveof redis-master 6380 + healthcheck: + test: redis-cli -p 6381 ping + node2: + image: redis:${REDIS_VERSION:-7.2.1} + container_name: redis-node-2 + ports: + - 6382:6382 + command: redis-server --port 6382 --slaveof redis-master 6380 + healthcheck: + test: redis-cli -p 6382 ping + sentinel1: + image: redis:${REDIS_VERSION:-7.2.1} + container_name: redis-sentinel-1 + ports: + - 26379:26379 + command: redis-sentinel /usr/local/etc/redis/conf/sentinel1.conf + healthcheck: + test: redis-cli -p 26379 ping + volumes: + - ./conf:/usr/local/etc/redis/conf + sentinel2: + image: redis:${REDIS_VERSION:-7.2.1} + container_name: redis-sentinel-2 + ports: + - 36379:26379 + command: redis-sentinel /usr/local/etc/redis/conf/sentinel2.conf + healthcheck: + test: redis-cli -p 36379 ping + volumes: + - ./conf:/usr/local/etc/redis/conf + sentinel3: + image: redis:${REDIS_VERSION:-7.2.1} + container_name: redis-sentinel-3 + ports: + - 46379:26379 + command: redis-sentinel /usr/local/etc/redis/conf/sentinel3.conf + healthcheck: + test: redis-cli -p 46379 ping + volumes: + - ./conf:/usr/local/etc/redis/conf diff --git a/script/docker-compose.yml b/script/docker-compose.yml index 7f1cdbc..4da616a 100644 --- a/script/docker-compose.yml +++ b/script/docker-compose.yml @@ -5,7 +5,7 @@ version: "3.9" services: redis: - image: redis:${REDIS_VERSION:-4.0.10} + image: redis:${REDIS_VERSION:-7.2.1} container_name: redis healthcheck: test: redis-cli ping