From 0f728ba15b38579efefc3dc47821882ca18ffea7 Mon Sep 17 00:00:00 2001 From: Tommy Lindgren Date: Thu, 28 Jan 2021 13:36:41 +0100 Subject: [PATCH] fuse: increase max readers for better throughput Previously maximum number of requests readers was hard-coded to 2. Setting it to max(2, min(16, GOMAXPROCS)) improves the throughput on systems with more cores available. Going beyond 16 readers seems to hurt performance even if there are more cores available. Benchmark GoFuseRead can be used for demonstrating the effects of this variable. The benchmark reads 32 streams in parallel from a dummy filesystem (read operations immediately return zeros). Example results from an i7-8550U (8 cores): | Max readers | Total throughput | | ----------: | ---------------: | | 2 | 13217 MB/s | | 4 | 19202 MB/s | | 8 | 19973 MB/s | | 16 | 18994 MB/s | On a 96 core system: | Max readers | Total throughput | | ----------: | ---------------: | | 2 | 11490 MB/s | | 4 | 16129 MB/s | | 8 | 24263 MB/s | | 16 | 29568 MB/s | | 32 | 28262 MB/s | Note that improvements won't be as dramatic for real filesytem implementations. In benchmarks for a filesystem doing real work I see a 30-40% improvement (8.3 -> 11.4 GB/s) on the 96 core system. Also tweaked some of the other benchmarks so they don't leave behind mountpoints. Fixes #388. Change-Id: Ibff17d7fc92195f078a9ccff818a31f3a58873f2 --- AUTHORS | 1 + benchmark/read_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++ benchmark/readfs.go | 49 +++++++++++++++++++++++++++++++++++ benchmark/stat_test.go | 6 +++-- fuse/server.go | 19 +++++++++++--- go.mod | 1 + go.sum | 2 ++ 7 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 benchmark/read_test.go create mode 100644 benchmark/readfs.go diff --git a/AUTHORS b/AUTHORS index 052128d30..49cafa6a7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -16,5 +16,6 @@ Patrick Crosby Paul Jolly Paul Warren Shayan Pooya +Tommy Lindgren Valient Gough Yongwoo Park diff --git a/benchmark/read_test.go b/benchmark/read_test.go new file mode 100644 index 000000000..12bd3374c --- /dev/null +++ b/benchmark/read_test.go @@ -0,0 +1,58 @@ +// Copyright 2021 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package benchmark + +import ( + "fmt" + "os" + "os/exec" + "testing" + + "github.com/hanwen/go-fuse/v2/internal/testutil" + "golang.org/x/sync/errgroup" +) + +func BenchmarkGoFuseRead(b *testing.B) { + fs := &readFS{} + wd, clean := setupFs(fs, b.N) + defer clean() + + jobs := 32 + blockSize := 64 * 1024 + + cmds := make([]*exec.Cmd, jobs) + + for i := 0; i < jobs; i++ { + cmds[i] = exec.Command("dd", + fmt.Sprintf("if=%s/foo.txt", wd), + "iflag=direct", + "of=/dev/null", + fmt.Sprintf("bs=%d", blockSize), + fmt.Sprintf("count=%d", b.N)) + if testutil.VerboseTest() { + cmds[i].Stdout = os.Stdout + cmds[i].Stderr = os.Stderr + } + } + + b.SetBytes(int64(jobs * blockSize)) + b.ReportAllocs() + b.ResetTimer() + + var eg errgroup.Group + + for i := 0; i < jobs; i++ { + i := i + eg.Go(func() error { + return cmds[i].Run() + }) + } + + if err := eg.Wait(); err != nil { + b.Fatalf("dd failed: %v", err) + } + + b.StopTimer() +} diff --git a/benchmark/readfs.go b/benchmark/readfs.go new file mode 100644 index 000000000..dbc555c5f --- /dev/null +++ b/benchmark/readfs.go @@ -0,0 +1,49 @@ +// Copyright 2021 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package benchmark + +import ( + "context" + "syscall" + "time" + + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" +) + +const fileSize = 2 << 60 + +// readFS is a filesystem that always and immediately returns zeros on read +// operations. Useful when benchmarking the raw throughput with go-fuse. +type readFS struct { + fs.Inode +} + +var _ = (fs.NodeLookuper)((*readFS)(nil)) + +func (n *readFS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { + sattr := fs.StableAttr{Mode: fuse.S_IFREG} + return n.NewInode(ctx, &readFS{}, sattr), fs.OK +} + +var _ = (fs.NodeGetattrer)((*readFS)(nil)) + +func (n *readFS) Getattr(ctx context.Context, f fs.FileHandle, out *fuse.AttrOut) syscall.Errno { + out.Size = fileSize + out.SetTimeout(time.Hour) + return fs.OK +} + +var _ = (fs.NodeOpener)((*readFS)(nil)) + +func (n *readFS) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { + return &readFS{}, fuse.FOPEN_DIRECT_IO, fs.OK +} + +var _ = (fs.FileReader)((*readFS)(nil)) + +func (n *readFS) Read(ctx context.Context, dest []byte, offset int64) (fuse.ReadResult, syscall.Errno) { + return fuse.ReadResultData(dest), fs.OK +} diff --git a/benchmark/stat_test.go b/benchmark/stat_test.go index be5eec06e..4b45708c7 100644 --- a/benchmark/stat_test.go +++ b/benchmark/stat_test.go @@ -129,7 +129,7 @@ func BenchmarkGoFuseStat(b *testing.B) { threads := runtime.GOMAXPROCS(0) if err := TestingBOnePass(b, threads, fileList, wd); err != nil { - log.Fatalf("TestingBOnePass %v8", err) + b.Fatalf("TestingBOnePass %v8", err) } } @@ -252,6 +252,8 @@ func BenchmarkCFuseThreadedStat(b *testing.B) { f.Close() mountPoint := testutil.TempDir() + defer os.RemoveAll(mountPoint) + cmd := exec.Command(wd+"/cstatfs", "-o", "entry_timeout=0.0,attr_timeout=0.0,ac_attr_timeout=0.0,negative_timeout=0.0", @@ -274,6 +276,6 @@ func BenchmarkCFuseThreadedStat(b *testing.B) { os.Lstat(mountPoint) threads := runtime.GOMAXPROCS(0) if err := TestingBOnePass(b, threads, fileList, mountPoint); err != nil { - log.Fatalf("TestingBOnePass %v", err) + b.Fatalf("TestingBOnePass %v", err) } } diff --git a/fuse/server.go b/fuse/server.go index 94d6a90ef..4b5c2428c 100644 --- a/fuse/server.go +++ b/fuse/server.go @@ -21,6 +21,9 @@ import ( const ( // The kernel caps writes at 128k. MAX_KERNEL_WRITE = 128 * 1024 + + minMaxReaders = 2 + maxMaxReaders = 16 ) // Server contains the logic for reading from the FUSE device and @@ -40,6 +43,9 @@ type Server struct { opts *MountOptions + // maxReaders is the maximum number of goroutines reading requests + maxReaders int + // Pools for []byte buffers bufferPool @@ -161,9 +167,17 @@ func NewServer(fs RawFileSystem, mountPoint string, opts *MountOptions) (*Server } } + maxReaders := runtime.GOMAXPROCS(0) + if maxReaders < minMaxReaders { + maxReaders = minMaxReaders + } else if maxReaders > maxMaxReaders { + maxReaders = maxMaxReaders + } + ms := &Server{ fileSystem: fs, opts: &o, + maxReaders: maxReaders, retrieveTab: make(map[uint64]*retrieveCacheRequest), // OSX has races when multiple routines read from the // FUSE device: on unmount, sometime some reads do not @@ -238,9 +252,6 @@ func (ms *Server) DebugData() string { return fmt.Sprintf("readers: %d", r) } -// What is a good number? Maybe the number of CPUs? -const _MAX_READERS = 2 - // handleEINTR retries the given function until it doesn't return syscall.EINTR. // This is similar to the HANDLE_EINTR() macro from Chromium ( see // https://code.google.com/p/chromium/codesearch#chromium/src/base/posix/eintr_wrapper.h @@ -267,7 +278,7 @@ func (ms *Server) readRequest(exitIdle bool) (req *request, code Status) { dest := ms.readPool.Get().([]byte) ms.reqMu.Lock() - if ms.reqReaders > _MAX_READERS { + if ms.reqReaders > ms.maxReaders { ms.reqMu.Unlock() return nil, OK } diff --git a/go.mod b/go.mod index 9c696c231..5531c7082 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/hanwen/go-fuse/v2 require ( github.com/hanwen/go-fuse v1.0.0 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 + golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 ) diff --git a/go.sum b/go.sum index ece416d2b..5e7bf8a0d 100644 --- a/go.sum +++ b/go.sum @@ -2,5 +2,7 @@ github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=