Improved Redis connection pool setup. Further logging on keyrange in core pkg

Author: filipecosta90

db/redis/db.go

@@ -264,6 +264,7 @@ const (
 	redisReadTimeout           = "redis.read_timeout"
 	redisWriteTimeout          = "redis.write_timeout"
 	redisPoolSize              = "redis.pool_size"
+	redisPoolSizeDefault              = 0
 	redisMinIdleConns          = "redis.min_idle_conns"
 	redisMaxConnAge            = "redis.max_conn_age"
 	redisPoolTimeout           = "redis.pool_timeout"
@@ -292,23 +293,28 @@ func parseTLS(p *properties.Properties) *tls.Config {
 
 func getOptionsSingle(p *properties.Properties) *goredis.Options {
 	opts := &goredis.Options{}
-	opts.Network = p.GetString(redisNetwork, redisNetworkDefault)
+
 	opts.Addr = p.GetString(redisAddr, redisAddrDefault)
-	opts.Password, _ = p.Get(redisPassword)
 	opts.DB = p.GetInt(redisDB, 0)
+	opts.Network = p.GetString(redisNetwork, redisNetworkDefault)
+	opts.Password, _ = p.Get(redisPassword)
 	opts.MaxRetries = p.GetInt(redisMaxRetries, 0)
 	opts.MinRetryBackoff = p.GetDuration(redisMinRetryBackoff, time.Millisecond*8)
 	opts.MaxRetryBackoff = p.GetDuration(redisMaxRetryBackoff, time.Millisecond*512)
 	opts.DialTimeout = p.GetDuration(redisDialTimeout, time.Second*5)
 	opts.ReadTimeout = p.GetDuration(redisReadTimeout, time.Second*3)
 	opts.WriteTimeout = p.GetDuration(redisWriteTimeout, opts.ReadTimeout)
-	opts.PoolSize = p.GetInt(redisPoolSize, 10)
+	opts.PoolSize = p.GetInt(redisPoolSize, redisPoolSizeDefault)
+	threadCount := p.MustGetInt("threadcount")
+	if opts.PoolSize == 0 {
+		opts.PoolSize = threadCount
+		fmt.Println(fmt.Sprintf("Setting %s=%d (from <threadcount>) given you haven't specified a value.", redisPoolSize, opts.PoolSize))
+	}
 	opts.MinIdleConns = p.GetInt(redisMinIdleConns, 0)
 	opts.MaxConnAge = p.GetDuration(redisMaxConnAge, 0)
 	opts.PoolTimeout = p.GetDuration(redisPoolTimeout, time.Second+opts.ReadTimeout)
 	opts.IdleTimeout = p.GetDuration(redisIdleTimeout, time.Minute*5)
 	opts.IdleCheckFrequency = p.GetDuration(redisIdleCheckFreq, time.Minute)
-
 	opts.TLSConfig = parseTLS(p)
 
 	return opts
@@ -330,7 +336,12 @@ func getOptionsCluster(p *properties.Properties) *goredis.ClusterOptions {
 	opts.DialTimeout = p.GetDuration(redisDialTimeout, time.Second*5)
 	opts.ReadTimeout = p.GetDuration(redisReadTimeout, time.Second*3)
 	opts.WriteTimeout = p.GetDuration(redisWriteTimeout, opts.ReadTimeout)
-	opts.PoolSize = p.GetInt(redisPoolSize, 10)
+	opts.PoolSize = p.GetInt(redisPoolSize, redisPoolSizeDefault)
+	threadCount := p.MustGetInt("threadcount")
+	if opts.PoolSize == 0 {
+		opts.PoolSize = threadCount
+		fmt.Println(fmt.Sprintf("Setting %s=%d (from <threadcount>) given you haven't specified a value.", redisPoolSize, opts.PoolSize))
+	}
 	opts.MinIdleConns = p.GetInt(redisMinIdleConns, 0)
 	opts.MaxConnAge = p.GetDuration(redisMaxConnAge, 0)
 	opts.PoolTimeout = p.GetDuration(redisPoolTimeout, time.Second+opts.ReadTimeout)

pkg/workload/core.go

@@ -408,10 +408,7 @@ func (c *core) nextKeyNum(state *coreState) int64 {
 			keyNum = c.transactionInsertKeySequence.Last() - c.keyChooser.Next(r)
 		}
 	} else {
-		keyNum = math.MaxInt64
-		for keyNum > c.transactionInsertKeySequence.Last() {
-			keyNum = c.keyChooser.Next(r)
-		}
+		keyNum = c.keyChooser.Next(r)
 	}
 	return keyNum
 }
@@ -650,31 +647,35 @@ func (coreCreator) Create(p *properties.Properties) (ycsb.Workload, error) {
 
 	c.keySequence = generator.NewCounter(insertStart)
 	c.operationChooser = createOperationGenerator(p)
+	var keyrangeLowerBound int64 = insertStart
+	var keyrangeUpperBound int64 = insertStart + insertCount - 1
 
 	c.transactionInsertKeySequence = generator.NewAcknowledgedCounter(c.recordCount)
 	switch requestDistrib {
 	case "uniform":
-		c.keyChooser = generator.NewUniform(insertStart, insertStart+insertCount-1)
+		c.keyChooser = generator.NewUniform(keyrangeLowerBound, keyrangeUpperBound)
 	case "sequential":
-		c.keyChooser = generator.NewSequential(insertStart, insertStart+insertCount-1)
+		c.keyChooser = generator.NewSequential(keyrangeLowerBound, keyrangeUpperBound)
 	case "zipfian":
 		insertProportion := p.GetFloat64(prop.InsertProportion, prop.InsertProportionDefault)
 		opCount := p.GetInt64(prop.OperationCount, 0)
 		expectedNewKeys := int64(float64(opCount) * insertProportion * 2.0)
-		c.keyChooser = generator.NewScrambledZipfian(insertStart, insertStart+insertCount+expectedNewKeys, generator.ZipfianConstant)
+		keyrangeUpperBound = insertStart + insertCount + expectedNewKeys
+		c.keyChooser = generator.NewScrambledZipfian(keyrangeLowerBound, keyrangeUpperBound, generator.ZipfianConstant)
 	case "latest":
 		c.keyChooser = generator.NewSkewedLatest(c.transactionInsertKeySequence)
 	case "hotspot":
 		hotsetFraction := p.GetFloat64(prop.HotspotDataFraction, prop.HotspotDataFractionDefault)
 		hotopnFraction := p.GetFloat64(prop.HotspotOpnFraction, prop.HotspotOpnFractionDefault)
-		c.keyChooser = generator.NewHotspot(insertStart, insertStart+insertCount-1, hotsetFraction, hotopnFraction)
+		c.keyChooser = generator.NewHotspot(keyrangeLowerBound, keyrangeUpperBound, hotsetFraction, hotopnFraction)
 	case "exponential":
 		percentile := p.GetFloat64(prop.ExponentialPercentile, prop.ExponentialPercentileDefault)
 		frac := p.GetFloat64(prop.ExponentialFrac, prop.ExponentialFracDefault)
 		c.keyChooser = generator.NewExponential(percentile, float64(c.recordCount)*frac)
 	default:
 		util.Fatalf("unknown request distribution %s", requestDistrib)
 	}
+	fmt.Println(fmt.Sprintf("Using request distribution '%s' a keyrange of [%d %d]", requestDistrib, keyrangeLowerBound, keyrangeUpperBound))
 
 	c.fieldChooser = generator.NewUniform(0, c.fieldCount-1)
 	switch scanLengthDistrib {