diff --git a/benchmark/spliceread_linux_test.go b/benchmark/spliceread_linux_test.go new file mode 100644 index 000000000..53433a08f --- /dev/null +++ b/benchmark/spliceread_linux_test.go @@ -0,0 +1,213 @@ +// Copyright 2026 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package benchmark + +import ( + "context" + "fmt" + "io" + "os" + "syscall" + "testing" + "time" + "unsafe" + + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" + "github.com/hanwen/go-fuse/v2/internal/testutil" + "github.com/hanwen/go-fuse/v2/splice" + "golang.org/x/sys/unix" +) + +// spliceFS is a fake filesystem serving reads by splicing from a backing file +type spliceFS struct { + fs.Inode + fd uintptr + size int64 + flags int +} + +// spliceReadSize is fixed rather than scaled by b.N, so the reported MiB mean the +// same thing however many iterations the framework picks. +const spliceReadSize = 32 << 20 + +var _ = (fs.NodeLookuper)((*spliceFS)(nil)) + +func (n *spliceFS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { + child := &spliceFS{fd: n.fd, size: n.size, flags: n.flags} + out.Attr.Mode = fuse.S_IFREG | 0444 + out.Attr.Size = uint64(n.size) + out.SetEntryTimeout(time.Hour) + out.SetAttrTimeout(time.Hour) + return n.NewInode(ctx, child, fs.StableAttr{Mode: fuse.S_IFREG}), fs.OK +} + +var _ = (fs.NodeOpener)((*spliceFS)(nil)) + +func (n *spliceFS) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, syscall.Errno) { + return nil, fuse.FOPEN_KEEP_CACHE, fs.OK +} + +var _ = (fs.NodeReader)((*spliceFS)(nil)) + +func (n *spliceFS) Read(ctx context.Context, f fs.FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { + if off >= n.size { + return fuse.ReadResultData(nil), fs.OK + } + total := int(min(off+int64(len(dest)), n.size) - off) + + pair, err := splice.Get() + if err != nil { + return nil, syscall.EIO + } + if err := pair.Grow(total); err != nil { + splice.Done(pair) + return nil, syscall.EIO + } + if _, err := pair.LoadFromAt(n.fd, total, off); err != nil { + splice.Done(pair) + return nil, syscall.EIO + } + return fuse.ReadResultPipeFlags(pair, total, n.flags), fs.OK +} + +func BenchmarkSpliceRead(b *testing.B) { + benchmarkSpliceRead(b, 0) +} + +// BenchmarkSpliceReadMove is BenchmarkSpliceRead with SPLICE_F_MOVE. Read the two +// cache metrics, not the timing: copying caches the payload twice, once in the +// backing file and once in the fuse inode, and moving relocates the page instead, +// so backing-MiB should fall to zero and the total halve. +// +// The kernel only moves whole single pages, so a backing filesystem that hands +// splice a large folio moves nothing and both benchmarks report the same numbers. +// On 6.12 that means ext4 halves and xfs does not. TMPDIR picks the filesystem. +func BenchmarkSpliceReadMove(b *testing.B) { + benchmarkSpliceRead(b, unix.SPLICE_F_MOVE) +} + +func benchmarkSpliceRead(b *testing.B, flags int) { + dir := b.TempDir() + + // Whether the kernel can steal a page depends on the filesystem holding the + // backing file, so it belongs in the name next to the numbers it produced. + b.Run(fsType(b, dir), func(b *testing.B) { + var backing, inode float64 + + // Only the read is timed; the mount and the 32 MiB of setup around it cost + // more than the read does. + b.SetBytes(spliceReadSize) + b.StopTimer() + for i := 0; i < b.N; i++ { + path := fmt.Sprintf("%s/backing-%d", dir, i) + f := cachedFile(b, path, spliceReadSize) + mnt, unmount := mountSpliceFS(b, &spliceFS{fd: f.Fd(), size: spliceReadSize, flags: flags}) + src, err := os.Open(mnt + "/file") + if err != nil { + b.Fatal(err) + } + b.StartTimer() + + n, err := io.CopyBuffer(io.Discard, src, make([]byte, blockSize)) + + b.StopTimer() + if err != nil { + b.Fatal(err) + } + if n != spliceReadSize { + b.Fatalf("read %d bytes, want %d", n, spliceReadSize) + } + backing += residentMiB(b, f) + inode += residentMiB(b, src) + + src.Close() + unmount() + f.Close() + if err := os.Remove(path); err != nil { + b.Fatal(err) + } + } + + b.ReportMetric(backing/float64(b.N), "backing-MiB") + b.ReportMetric(inode/float64(b.N), "inode-MiB") + }) +} + +// fsType names the filesystem holding dir. One magic number covers ext2, ext3 and +// ext4, hence the range in the name. +func fsType(b *testing.B, dir string) string { + var st unix.Statfs_t + if err := unix.Statfs(dir, &st); err != nil { + b.Fatal(err) + } + switch st.Type { + case unix.EXT4_SUPER_MAGIC: + return "ext2-4" + case unix.XFS_SUPER_MAGIC: + return "xfs" + case unix.BTRFS_SUPER_MAGIC: + return "btrfs" + case unix.TMPFS_MAGIC: + return "tmpfs" + case unix.OVERLAYFS_SUPER_MAGIC: + return "overlayfs" + } + return fmt.Sprintf("%#x", st.Type) +} + +// cachedFile returns a file of size bytes whose pages are in the page cache and +// clean, the state a cache file is in when a filesystem serves a read from it. +func cachedFile(b *testing.B, path string, size int64) *os.File { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0644) + if err != nil { + b.Fatal(err) + } + if err := f.Truncate(size); err != nil { + b.Fatal(err) + } + if _, err := io.Copy(io.Discard, io.NewSectionReader(f, 0, size)); err != nil { + b.Fatal(err) + } + return f +} + +func mountSpliceFS(b *testing.B, root *spliceFS) (string, func()) { + opts := &fs.Options{} + opts.Debug = testutil.VerboseTest() + mnt := b.TempDir() + server, err := fs.Mount(mnt, root, opts) + if err != nil { + b.Fatal(err) + } + return mnt, func() { + if err := server.Unmount(); err != nil { + b.Fatal(err) + } + } +} + +// residentMiB is how much of f is in the page cache, from mincore(2). +func residentMiB(b *testing.B, f *os.File) float64 { + data, err := unix.Mmap(int(f.Fd()), 0, spliceReadSize, unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + b.Fatal(err) + } + defer unix.Munmap(data) + + pageSize := os.Getpagesize() + vec := make([]byte, (len(data)+pageSize-1)/pageSize) + if _, _, errno := unix.Syscall(unix.SYS_MINCORE, uintptr(unsafe.Pointer(&data[0])), + uintptr(len(data)), uintptr(unsafe.Pointer(&vec[0]))); errno != 0 { + b.Fatal(errno) + } + resident := 0 + for _, v := range vec { + resident += int(v & 1) + } + return float64(resident) * float64(pageSize) / (1 << 20) +} diff --git a/fs/splicemove_linux_test.go b/fs/splicemove_linux_test.go new file mode 100644 index 000000000..f6ce91328 --- /dev/null +++ b/fs/splicemove_linux_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package fs + +import ( + "bytes" + "context" + "encoding/binary" + "os" + "syscall" + "testing" + "time" + + "github.com/hanwen/go-fuse/v2/fuse" + "github.com/hanwen/go-fuse/v2/internal/testutil" + "github.com/hanwen/go-fuse/v2/splice" + "golang.org/x/sys/unix" +) + +// spliceMoveNode splices reads out of a backing file, asking the kernel to move +// the pages instead of copying them. +type spliceMoveNode struct { + Inode + fd uintptr + promise int64 // size told to the kernel; past EOF it forces the short-read fixup +} + +func (n *spliceMoveNode) Open(ctx context.Context, flags uint32) (FileHandle, uint32, syscall.Errno) { + return nil, fuse.FOPEN_KEEP_CACHE, 0 +} + +func (n *spliceMoveNode) Getattr(ctx context.Context, fh FileHandle, out *fuse.AttrOut) syscall.Errno { + out.Mode = 0444 + out.Size = uint64(n.promise) + return 0 +} + +func (n *spliceMoveNode) Read(ctx context.Context, fh FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { + if off >= n.promise { + return fuse.ReadResultData(nil), 0 + } + total := int(min(off+int64(len(dest)), n.promise) - off) + + pair, err := splice.Get() + if err != nil { + return nil, syscall.EIO + } + if err := pair.Grow(total); err != nil { + splice.Done(pair) + return nil, syscall.EIO + } + if _, err := pair.LoadFromAt(n.fd, total, off); err != nil { + splice.Done(pair) + return nil, syscall.EIO + } + return fuse.ReadResultPipeFlags(pair, total, unix.SPLICE_F_MOVE), 0 +} + +// TestReadResultPipeSpliceMove reads a file served with SPLICE_F_MOVE. A moved +// page leaves the backing file's cache, so a broken move corrupts or loses the +// payload instead of returning an error. +func TestReadResultPipeSpliceMove(t *testing.T) { + // Page aligned, or a promise past EOF leaves the kernel a partial page to + // zero-fill and the read comes back longer than the file. Not a multiple of + // the 128 KiB read window, so that promise makes the last read straddle EOF. + const size = 1<<20 - 4096 + + // Each 8 bytes hold their own offset, so shifted or duplicated data fails. + want := make([]byte, size) + for i := 0; i < len(want); i += 8 { + binary.LittleEndian.PutUint64(want[i:], uint64(i)) + } + + for _, tc := range []struct { + name string + promise int64 + }{ + {"exact", size}, + {"short read", size + 4096}, + } { + t.Run(tc.name, func(t *testing.T) { + backing := t.TempDir() + "/backing" + if err := os.WriteFile(backing, want, 0644); err != nil { + t.Fatal(err) + } + f, err := os.Open(backing) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + root := &Inode{} + node := &spliceMoveNode{fd: f.Fd(), promise: tc.promise} + sec := time.Second + opts := &Options{ + FirstAutomaticIno: 1, + EntryTimeout: &sec, + AttrTimeout: &sec, + OnAdd: func(ctx context.Context) { + n := root.EmbeddedInode() + n.AddChild("file", n.NewPersistentInode(ctx, node, StableAttr{}), false) + }, + } + opts.Debug = testutil.VerboseTest() + + mnt := t.TempDir() + server, err := Mount(mnt, root, opts) + if err != nil { + t.Fatal(err) + } + defer server.Unmount() + + got, err := os.ReadFile(mnt + "/file") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("read %d bytes, want %d", len(got), len(want)) + } + }) + } +} diff --git a/fuse/read.go b/fuse/read.go index 8421f2764..991683237 100644 --- a/fuse/read.go +++ b/fuse/read.go @@ -42,6 +42,12 @@ type statefulResult interface { Stateful() (fd uintptr, sz int) } +// spliceFlaggedResult is a ReadResult carrying splice(2) flags for the write to +// /dev/fuse. +type spliceFlaggedResult interface { + SpliceFlags() int +} + // ReadResultFd is the read return for zero-copy file data. type readResultFd struct { // Splice from the following file. diff --git a/fuse/splice_linux.go b/fuse/splice_linux.go index adf19810f..a4508ccdb 100644 --- a/fuse/splice_linux.go +++ b/fuse/splice_linux.go @@ -35,6 +35,11 @@ func (r *fuseFD) trySplice(req *request, readResult ReadResult) error { // readResult.Size(), so req.outHeaderBuf is correct for the optimistic case. total := len(req.outHeaderBuf) + len(req.outDataBuf) + readResult.Size() + spliceFlags := 0 + if f, ok := readResult.(spliceFlaggedResult); ok { + spliceFlags = f.SpliceFlags() + } + pair, err := splice.Get() if err != nil { return err @@ -90,12 +95,12 @@ func (r *fuseFD) trySplice(req *request, readResult ReadResult) error { // New length. req.serializeHeader(payloadLen) - return r.trySplice(req, ReadResultPipe(pair, payloadLen)) + return r.trySplice(req, ReadResultPipeFlags(pair, payloadLen, spliceFlags)) } // Write header + payload to /dev/fuse. if cerr := r.withFD(func(fd int) { - _, err = pair.WriteTo(uintptr(fd), total) + _, err = pair.WriteToFlags(uintptr(fd), total, spliceFlags) }); cerr != nil { return cerr } @@ -103,8 +108,9 @@ func (r *fuseFD) trySplice(req *request, readResult ReadResult) error { } type pipeReadResult struct { - pair *splice.Pair - size int + pair *splice.Pair + size int + flags int } func (r *pipeReadResult) Done() { @@ -128,9 +134,19 @@ func (r *pipeReadResult) Stateful() (fd uintptr, sz int) { return r.pair.ReadFd(), r.size } +func (r *pipeReadResult) SpliceFlags() int { return r.flags } + // ReadResultPipe returns a [ReadResult] of `size` bytes that was preloaded // into the given pipe. The pipe is discarded with splice.Done() // after the read completes. func ReadResultPipe(pipe *splice.Pair, size int) ReadResult { - return &pipeReadResult{pipe, size} + return &pipeReadResult{pair: pipe, size: size} +} + +// ReadResultPipeFlags is [ReadResultPipe] with splice(2) flags. Only +// SPLICE_F_MOVE has an effect on /dev/fuse: it moves each page out of the +// source file's page cache, after waiting on its writeback, so use it only for +// pages the filesystem can lose. Only readahead reads are eligible. +func ReadResultPipeFlags(pipe *splice.Pair, size, flags int) ReadResult { + return &pipeReadResult{pair: pipe, size: size, flags: flags} } diff --git a/splice/pair_linux.go b/splice/pair_linux.go index 46e9f11e4..2d44fae4b 100644 --- a/splice/pair_linux.go +++ b/splice/pair_linux.go @@ -69,11 +69,17 @@ func (p *Pair) LoadFrom(fd uintptr, sz int) (int, error) { } func (p *Pair) WriteTo(fd uintptr, n int) (int, error) { + return p.WriteToFlags(fd, n, 0) +} + +// WriteToFlags is WriteTo with splice(2) flags. /dev/fuse acts on SPLICE_F_MOVE +// even though the generic pipe-to-file path ignores it. +func (p *Pair) WriteToFlags(fd uintptr, n int, flags int) (int, error) { var m int var err error p.rConn.Control(func(rfd uintptr) { var sm int64 - sm, err = syscall.Splice(int(rfd), nil, int(fd), nil, n, 0) + sm, err = syscall.Splice(int(rfd), nil, int(fd), nil, n, flags) m = int(sm) }) if err != nil {