fuse: increase max readers for better throughput

Previously maximum number of requests readers was hard-coded to 2.
Setting it to max(2, min(16, GOMAXPROCS)) improves the throughput on
systems with more cores available. Going beyond 16 readers seems to hurt
performance even if there are more cores available.

Benchmark GoFuseRead can be used for demonstrating the effects of this
variable. The benchmark reads 32 streams in parallel from a dummy
filesystem (read operations immediately return zeros). Example results
from an i7-8550U (8 cores):

| Max readers | Total throughput |
| ----------: | ---------------: |
|           2 |       13217 MB/s |
|           4 |       19202 MB/s |
|           8 |       19973 MB/s |
|          16 |       18994 MB/s |

On a 96 core system:

| Max readers | Total throughput |
| ----------: | ---------------: |
|           2 |       11490 MB/s |
|           4 |       16129 MB/s |
|           8 |       24263 MB/s |
|          16 |       29568 MB/s |
|          32 |       28262 MB/s |

Note that improvements won't be as dramatic for real filesytem
implementations. In benchmarks for a filesystem doing real work I see a
30-40% improvement (8.3 -> 11.4 GB/s) on the 96 core system.

Also tweaked some of the other benchmarks so they don't leave behind
mountpoints.

Fixes hanwen#388.

Change-Id: Ibff17d7fc92195f078a9ccff818a31f3a58873f2

AUTHORS

@@ -16,5 +16,6 @@ Patrick Crosby <[email protected]>
 Paul Jolly <[email protected]>
 Paul Warren <[email protected]>
 Shayan Pooya <[email protected]>
+Tommy Lindgren <[email protected]>
 Valient Gough <[email protected]>
 Yongwoo Park <[email protected]>

benchmark/read_test.go

@@ -0,0 +1,58 @@
+// Copyright 2021 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package benchmark
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/hanwen/go-fuse/v2/internal/testutil"
+	"golang.org/x/sync/errgroup"
+)
+
+func BenchmarkGoFuseRead(b *testing.B) {
+	fs := &readFS{}
+	wd, clean := setupFs(fs, b.N)
+	defer clean()
+
+	jobs := 32
+	blockSize := 64 * 1024
+
+	cmds := make([]*exec.Cmd, jobs)
+
+	for i := 0; i < jobs; i++ {
+		cmds[i] = exec.Command("dd",
+			fmt.Sprintf("if=%s/foo.txt", wd),
+			"iflag=direct",
+			"of=/dev/null",
+			fmt.Sprintf("bs=%d", blockSize),
+			fmt.Sprintf("count=%d", b.N))
+		if testutil.VerboseTest() {
+			cmds[i].Stdout = os.Stdout
+			cmds[i].Stderr = os.Stderr
+		}
+	}
+
+	b.SetBytes(int64(jobs * blockSize))
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	var eg errgroup.Group
+
+	for i := 0; i < jobs; i++ {
+		i := i
+		eg.Go(func() error {
+			return cmds[i].Run()
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		b.Fatalf("dd failed: %v", err)
+	}
+
+	b.StopTimer()
+}

benchmark/readfs.go

@@ -0,0 +1,49 @@
+// Copyright 2021 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package benchmark
+
+import (
+	"context"
+	"syscall"
+	"time"
+
+	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+const fileSize = 2 << 60
+
+// readFS is a filesystem that always and immediately returns zeros on read
+// operations. Useful when benchmarking the raw throughput with go-fuse.
+type readFS struct {
+	fs.Inode
+}
+
+var _ = (fs.NodeLookuper)((*readFS)(nil))
+
+func (n *readFS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
+	sattr := fs.StableAttr{Mode: fuse.S_IFREG}
+	return n.NewInode(ctx, &readFS{}, sattr), fs.OK
+}
+
+var _ = (fs.NodeGetattrer)((*readFS)(nil))
+
+func (n *readFS) Getattr(ctx context.Context, f fs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	out.Size = fileSize
+	out.SetTimeout(time.Hour)
+	return fs.OK
+}
+
+var _ = (fs.NodeOpener)((*readFS)(nil))
+
+func (n *readFS) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
+	return &readFS{}, fuse.FOPEN_DIRECT_IO, fs.OK
+}
+
+var _ = (fs.FileReader)((*readFS)(nil))
+
+func (n *readFS) Read(ctx context.Context, dest []byte, offset int64) (fuse.ReadResult, syscall.Errno) {
+	return fuse.ReadResultData(dest), fs.OK
+}

benchmark/stat_test.go

@@ -129,7 +129,7 @@ func BenchmarkGoFuseStat(b *testing.B) {
 
 	threads := runtime.GOMAXPROCS(0)
 	if err := TestingBOnePass(b, threads, fileList, wd); err != nil {
-		log.Fatalf("TestingBOnePass %v8", err)
+		b.Fatalf("TestingBOnePass %v8", err)
 	}
 }
 
@@ -252,6 +252,8 @@ func BenchmarkCFuseThreadedStat(b *testing.B) {
 	f.Close()
 
 	mountPoint := testutil.TempDir()
+	defer os.RemoveAll(mountPoint)
+
 	cmd := exec.Command(wd+"/cstatfs",
 		"-o",
 		"entry_timeout=0.0,attr_timeout=0.0,ac_attr_timeout=0.0,negative_timeout=0.0",
@@ -274,6 +276,6 @@ func BenchmarkCFuseThreadedStat(b *testing.B) {
 	os.Lstat(mountPoint)
 	threads := runtime.GOMAXPROCS(0)
 	if err := TestingBOnePass(b, threads, fileList, mountPoint); err != nil {
-		log.Fatalf("TestingBOnePass %v", err)
+		b.Fatalf("TestingBOnePass %v", err)
 	}
 }

fuse/server.go

@@ -21,6 +21,9 @@ import (
 const (
 	// The kernel caps writes at 128k.
 	MAX_KERNEL_WRITE = 128 * 1024
+
+	minMaxReaders = 2
+	maxMaxReaders = 16
 )
 
 // Server contains the logic for reading from the FUSE device and
@@ -40,6 +43,9 @@ type Server struct {
 
 	opts *MountOptions
 
+	// maxReaders is the maximum number of goroutines reading requests
+	maxReaders int
+
 	// Pools for []byte
 	buffers bufferPool
 
@@ -161,9 +167,17 @@ func NewServer(fs RawFileSystem, mountPoint string, opts *MountOptions) (*Server
 		}
 	}
 
+	maxReaders := runtime.GOMAXPROCS(0)
+	if maxReaders < minMaxReaders {
+		maxReaders = minMaxReaders
+	} else if maxReaders > maxMaxReaders {
+		maxReaders = maxMaxReaders
+	}
+
 	ms := &Server{
 		fileSystem:  fs,
 		opts:        &o,
+		maxReaders:  maxReaders,
 		retrieveTab: make(map[uint64]*retrieveCacheRequest),
 		// OSX has races when multiple routines read from the
 		// FUSE device: on unmount, sometime some reads do not
@@ -238,9 +252,6 @@ func (ms *Server) DebugData() string {
 	return fmt.Sprintf("readers: %d", r)
 }
 
-// What is a good number?  Maybe the number of CPUs?
-const _MAX_READERS = 2
-
 // handleEINTR retries the given function until it doesn't return syscall.EINTR.
 // This is similar to the HANDLE_EINTR() macro from Chromium ( see
 // https://code.google.com/p/chromium/codesearch#chromium/src/base/posix/eintr_wrapper.h
@@ -267,7 +278,7 @@ func (ms *Server) readRequest(exitIdle bool) (req *request, code Status) {
 	dest := ms.readPool.Get().([]byte)
 
 	ms.reqMu.Lock()
-	if ms.reqReaders > _MAX_READERS {
+	if ms.reqReaders > ms.maxReaders {
 		ms.reqMu.Unlock()
 		return nil, OK
 	}

go.mod

@@ -3,6 +3,7 @@ module github.com/hanwen/go-fuse/v2
 require (
 	github.com/hanwen/go-fuse v1.0.0
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/sys v0.0.0-20180830151530-49385e6e1522
 )
 

go.sum

@@ -2,5 +2,7 @@ github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
 github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=