[zstd] test low memory options

Let's see how this affects memory usage and performance. Related to buchgr#473.
mostynb · Sep 22, 2021 · 647f07b · 647f07b
1 parent a914031
commit 647f07b
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go
@@ -26,8 +26,8 @@ const (
 
 var zstdFastestLevel = zstd.WithEncoderLevel(zstd.SpeedFastest)
 
-var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel) // TODO: raise WithEncoderConcurrency ?
-var decoder, _ = zstd.NewReader(nil)                   // TODO: raise WithDecoderConcurrency ?
+var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel, zstd.WithLowerEncoderMem(true)) // TODO: raise WithEncoderConcurrency ?
+var decoder, _ = zstd.NewReader(nil, zstd.WithDecoderLowmem(true))                     // TODO: raise WithDecoderConcurrency ?
 
 var encoderPool = zstdpool.GetEncoderPool()
 var decoderPool = zstdpool.GetDecoderPool()

diff --git a/server/http.go b/server/http.go
@@ -27,7 +27,7 @@ import (
 
 var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$")
 
-var decoder, _ = zstd.NewReader(nil) // TODO: raise WithDecoderConcurrency ?
+var decoder, _ = zstd.NewReader(nil, zstd.WithDecoderLowmem(true)) // TODO: raise WithDecoderConcurrency ?
 
 // HTTPCache ...
 type HTTPCache interface {

diff --git a/utils/zstdpool/zstdpool.go b/utils/zstdpool/zstdpool.go
@@ -17,7 +17,8 @@ func GetEncoderPool() *sync.Pool {
 	onceEncPool.Do(func() {
 		encoderPool = syncpool.NewEncoderPool(
 			zstd.WithEncoderConcurrency(1),
-			zstd.WithEncoderLevel(zstd.SpeedFastest))
+			zstd.WithEncoderLevel(zstd.SpeedFastest),
+			zstd.WithLowerEncoderMem(true))
 	})
 
 	return encoderPool
@@ -26,7 +27,8 @@ func GetEncoderPool() *sync.Pool {
 func GetDecoderPool() *sync.Pool {
 	onceDecPool.Do(func() {
 		decoderPool = syncpool.NewDecoderPool(
-			zstd.WithDecoderConcurrency(1))
+			zstd.WithDecoderConcurrency(1),
+			zstd.WithDecoderLowmem(true))
 	})
 
 	return decoderPool