More caching tricks (#12)

Author: luke-lombardi

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e // indirect
+	github.com/djherbis/atime v1.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.6.0 // indirect

go.sum

@@ -73,6 +73,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q=
 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A=
+github.com/djherbis/atime v1.1.0 h1:rgwVbP/5by8BvvjBNrbh64Qz33idKT3pSnMSJsxhi0g=
+github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=

pkg/blobfs.go

@@ -38,6 +38,7 @@ type BlobFsMetadata struct {
 	Padding   uint32 `redis:"padding" json:"padding"`
 	Uid       uint32 `redis:"uid" json:"uid"`
 	Gid       uint32 `redis:"gid" json:"gid"`
+	Gen       uint64 `redis:"gen" json:"gen"`
 }
 
 type StorageLayer interface {
@@ -97,8 +98,8 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err
 	}
 
 	root, _ := blobfs.Root()
-	attrTimeout := time.Second * 60
-	entryTimeout := time.Second * 60
+	attrTimeout := time.Second * 5
+	entryTimeout := time.Second * 5
 	fsOptions := &fs.Options{
 		AttrTimeout:  &attrTimeout,
 		EntryTimeout: &entryTimeout,

pkg/blobfs_node.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"strings"
 	"syscall"
 
 	"github.com/hanwen/go-fuse/v2/fs"
@@ -51,6 +52,9 @@ func (n *FSNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOu
 	out.Mode = node.Attr.Mode
 	out.Nlink = node.Attr.Nlink
 	out.Owner = node.Attr.Owner
+	out.Atimensec = node.Attr.Atimensec
+	out.Mtimensec = node.Attr.Mtimensec
+	out.Ctimensec = node.Attr.Ctimensec
 
 	return fs.OK
 }
@@ -78,21 +82,14 @@ func metaToAttr(metadata *BlobFsMetadata) fuse.Attr {
 	}
 }
 
-func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
-	n.log("Lookup called with name: %s", name)
-
-	// Construct the full of this file path from root
-	fullPath := path.Join(n.bfsNode.Path, name)
-
-	id := GenerateFsID(fullPath)
-	metadata, err := n.filesystem.Metadata.GetFsNode(ctx, id)
+func (n *FSNode) inodeFromFsId(ctx context.Context, fsId string) (*fs.Inode, *fuse.Attr, error) {
+	metadata, err := n.filesystem.Metadata.GetFsNode(ctx, fsId)
 	if err != nil {
-		return nil, syscall.ENOENT
+		return nil, nil, syscall.ENOENT
 	}
 
 	// Fill out the child node's attributes
 	attr := metaToAttr(metadata)
-	out.Attr = attr
 
 	// Create a new Inode on lookup
 	node := n.NewInode(ctx,
@@ -105,9 +102,46 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*
 			Attr:   attr,
 			Target: "",
 		}, attr: attr},
-		fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino},
+		fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino, Gen: metadata.Gen},
 	)
 
+	return node, &attr, nil
+}
+
+func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
+	fullPath := path.Join(n.bfsNode.Path, name) // Construct the full of this file path from root
+	n.log("Lookup called with path: %s", fullPath)
+
+	// Force caching of a specific full path if the path contains a special illegal character '%'
+	// This is a hack to trigger caching from external callers without going through the GRPC service directly
+	if strings.Contains(fullPath, "%") {
+		sourcePath := strings.ReplaceAll(fullPath, "%", "/")
+
+		if !n.filesystem.Client.HostsAvailable() {
+			return nil, syscall.ENOENT
+		}
+
+		n.log("Storing content from source with path: %s", sourcePath)
+		_, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0)
+		if err != nil {
+			return nil, syscall.ENOENT
+		}
+
+		node, attr, err := n.inodeFromFsId(ctx, GenerateFsID(sourcePath))
+		if err != nil {
+			return nil, syscall.ENOENT
+		}
+
+		out.Attr = *attr
+		return node, fs.OK
+	}
+
+	node, attr, err := n.inodeFromFsId(ctx, GenerateFsID(fullPath))
+	if err != nil {
+		return nil, syscall.ENOENT
+	}
+
+	out.Attr = *attr
 	return node, fs.OK
 }
 
@@ -118,6 +152,11 @@ func (n *FSNode) Opendir(ctx context.Context) syscall.Errno {
 
 func (n *FSNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
 	n.log("Open called with flags: %v", flags)
+
+	if !n.filesystem.Client.HostsAvailable() {
+		return nil, 0, syscall.EIO
+	}
+
 	return nil, 0, fs.OK
 }
 

pkg/client.go

@@ -108,6 +108,7 @@ func NewBlobCacheClient(ctx context.Context, cfg BlobCacheConfig) (*BlobCacheCli
 			Config:   cfg,
 			Metadata: metadata,
 			Client:   bc,
+			Verbose:  cfg.DebugMode,
 		})
 		if err != nil {
 			return nil, err

pkg/metadata.go

@@ -3,11 +3,13 @@ package blobcache
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/djherbis/atime"
 	"github.com/hanwen/go-fuse/v2/fuse"
 	redis "github.com/redis/go-redis/v9"
 )
@@ -151,10 +153,10 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin
 			return err
 		}
 
+		// Initialize default metadata
 		now := time.Now()
 		nowSec := uint64(now.Unix())
 		nowNsec := uint32(now.Nanosecond())
-
 		metadata := &BlobFsMetadata{
 			PID:       previousParentId,
 			ID:        currentNodeId,
@@ -170,18 +172,43 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin
 			Ctimensec: nowNsec,
 		}
 
-		// Since this is the last file, store as a file, not a dir
-		if path == currentPath {
-			metadata.Mode = fuse.S_IFREG | 0755
-			metadata.Hash = hash
-			metadata.Size = size
+		// If currentPath matches the input path, use the actual file info
+		if currentPath == path {
+			fileInfo, err := os.Stat(currentPath)
+			if err != nil {
+				return err
+			}
+
+			// Update metadata fields with actual file info values
+			modTime := fileInfo.ModTime()
+			accessTime := atime.Get(fileInfo)
+			metadata.Mode = uint32(fileInfo.Mode())
+			metadata.Atime = uint64(accessTime.Unix())
+			metadata.Atimensec = uint32(accessTime.Nanosecond())
+			metadata.Mtime = uint64(modTime.Unix())
+			metadata.Mtimensec = uint32(modTime.Nanosecond())
+
+			// Since we cannot get Ctime in a platform-independent way, set it to ModTime
+			metadata.Ctime = uint64(modTime.Unix())
+			metadata.Ctimensec = uint32(modTime.Nanosecond())
+
+			metadata.Size = uint64(fileInfo.Size())
+			if fileInfo.IsDir() {
+				metadata.Hash = GenerateFsID(currentPath)
+				metadata.Size = 0
+			} else {
+				metadata.Hash = hash
+				metadata.Size = size
+			}
 		}
 
+		// Set metadata
 		err = m.SetFsNode(ctx, currentNodeId, metadata)
 		if err != nil {
 			return err
 		}
 
+		// Add the current node as a child of the previous node
 		err = m.AddFsNodeChild(ctx, previousParentId, currentNodeId)
 		if err != nil {
 			return err
@@ -196,7 +223,7 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin
 func (m *BlobCacheMetadata) GetFsNode(ctx context.Context, id string) (*BlobFsMetadata, error) {
 	key := MetadataKeys.MetadataFsNode(id)
 
-	res, err := m.rdb.HGetAll(context.TODO(), key).Result()
+	res, err := m.rdb.HGetAll(ctx, key).Result()
 	if err != nil && err != redis.Nil {
 		return nil, err
 	}
@@ -216,7 +243,22 @@ func (m *BlobCacheMetadata) GetFsNode(ctx context.Context, id string) (*BlobFsMe
 func (m *BlobCacheMetadata) SetFsNode(ctx context.Context, id string, metadata *BlobFsMetadata) error {
 	key := MetadataKeys.MetadataFsNode(id)
 
-	err := m.rdb.HSet(ctx, key, ToSlice(metadata)).Err()
+	// If metadata exists, increment inode generation #
+	res, err := m.rdb.HGetAll(ctx, key).Result()
+	if err != nil && err != redis.Nil {
+		return err
+	}
+
+	if len(res) > 0 {
+		existingMetadata := &BlobFsMetadata{}
+		if err = ToStruct(res, existingMetadata); err != nil {
+			return err
+		}
+
+		metadata.Gen = existingMetadata.Gen + 1
+	}
+
+	err = m.rdb.HSet(ctx, key, ToSlice(metadata)).Err()
 	if err != nil {
 		return fmt.Errorf("failed to set blobfs node metadata <%v>: %w", key, err)
 	}