diff --git a/CHANGELOG.md b/CHANGELOG.md index b49aa89..59d297b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ -## [v0.2.0](https://github.com/AlekSi/hardcache/releases/tag/v0.2.0) (2025-12-07) +## v0.3.0 (unreleased) -## What's Changed +## [v0.2.0](https://github.com/AlekSi/hardcache/releases/tag/v0.2.0) (2025-12-07) * Windows support. * Support continuous trimming with `hardcache local trimd`. diff --git a/Makefile b/Makefile index efd99a1..e9f8fe6 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,4 @@ run: go build -race -o bin/ bin/hardcache --help mkdir -p tmp/cache + env GORACE=halt_on_error=1 GOCACHEPROG='bin/hardcache local use --dir=tmp/cache --debug' go build -o bin/ diff --git a/README.md b/README.md index 66c1267..7c75da7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # hardcache +[![Go Reference](https://pkg.go.dev/badge/github.com/AlekSi/hardcache.svg)](https://pkg.go.dev/github.com/AlekSi/hardcache) + ![Hardcache logo](hardcache.jpeg) Hardcache is a tool for managing the Go build cache. diff --git a/internal/prog/prog.go b/internal/prog/prog.go new file mode 100644 index 0000000..a3e29ff --- /dev/null +++ b/internal/prog/prog.go @@ -0,0 +1,266 @@ +// Package prog provides GOCACHEPROG protocol wrapper for a given cache. +package prog + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "sync" + "time" + + "github.com/AlekSi/hardcache/internal/go/cache" + "github.com/AlekSi/hardcache/internal/go/cacheprog" +) + +// Prog provides GOCACHEPROG implementation for the given [cache.Cache] +// (that should safe to use from multiple goroutines and multiple processes). +// +//nolint:vet // for readability +type Prog struct { + c cache.Cache + l *slog.Logger + closed bool + + inD *json.Decoder + + outM sync.Mutex + outC io.Closer + outW *bufio.Writer + outE *json.Encoder +} + +// New creates a new [Prog]. +// It takes over the cache and reader/writer. +func New(c cache.Cache, l *slog.Logger, r io.Reader, w io.WriteCloser) *Prog { + dec := json.NewDecoder(bufio.NewReader(r)) + dec.DisallowUnknownFields() + + outW := bufio.NewWriter(w) + outE := json.NewEncoder(outW) + + return &Prog{ + c: c, + l: l, + inD: dec, + outC: w, + outW: outW, + outE: outE, + } +} + +// Run runs the Prog until ctx is canceled, close message is received and handled, +// or an error occurs. +// On exit, it closes the cache. +func (p *Prog) Run(ctx context.Context) (err error) { + defer func() { + if e := p.c.Close(); err == nil { + err = e + } + if e := p.outC.Close(); err == nil { + err = e + } + }() + + err = p.send(&cacheprog.Response{ + KnownCommands: []cacheprog.Cmd{cacheprog.CmdGet, cacheprog.CmdPut, cacheprog.CmdClose}, + }) + if err != nil { + return + } + + for { + // Go tool does send multiple requests in a pipeline; we should handle that. + + var req cacheprog.Request + if err = p.inD.Decode(&req); err != nil { + if errors.Is(err, io.EOF) && p.closed { + p.l.Debug("Exiting due to EOF after close") + err = nil + return + } + + p.l.Warn("Exiting due to error", slog.Bool("closed", p.closed)) + return + } + + args := []any{ + slog.Int64("id", req.ID), + slog.String("command", string(req.Command)), + } + + if req.ActionID != nil { + args = append(args, slog.String("actionID", fmt.Sprintf("%x", req.ActionID))) + } + + if req.OutputID != nil { + args = append(args, slog.String("outputID", fmt.Sprintf("%x", req.OutputID))) + } + + args = append(args, slog.Int64("bodySize", req.BodySize)) + + p.l.DebugContext(ctx, "Request", args...) + + var resp *cacheprog.Response + + switch req.Command { + case cacheprog.CmdGet: + resp, err = p.handleGet(&req) + + case cacheprog.CmdPut: + resp, err = p.handlePut(&req) + + case cacheprog.CmdClose: + resp, err = p.handleClose(&req) + + default: + resp = &cacheprog.Response{ + ID: req.ID, + Err: fmt.Sprintf("hardcache: unknown command %q", req.Command), + } + } + + if err != nil { + return + } + + args = []any{slog.Int64("id", resp.ID)} + + if resp.Err != "" { + args = append(args, slog.String("error", resp.Err)) + } + + args = append(args, slog.Bool("miss", resp.Miss)) + + if resp.OutputID != nil { + args = append(args, slog.String("outputID", fmt.Sprintf("%x", resp.OutputID))) + } + + args = append(args, slog.Int64("size", resp.Size)) + + if resp.Time != nil { + args = append(args, slog.String("time", resp.Time.Format(time.RFC3339Nano))) + } + + args = append(args, slog.String("diskPath", resp.DiskPath)) + + p.l.DebugContext(ctx, "Response", args...) + + if err = p.send(resp); err != nil { + return + } + } +} + +// handleGet handles the `get` command. +// Returned error is something fatal. +func (p *Prog) handleGet(req *cacheprog.Request) (*cacheprog.Response, error) { + resp := cacheprog.Response{ + ID: req.ID, + } + + if len(req.ActionID) != cache.HashSize { + resp.Err = fmt.Sprintf("hardcache: get: invalid action ID size %d, expected %d", len(req.ActionID), cache.HashSize) + return &resp, nil + } + + if req.OutputID != nil { + resp.Err = "hardcache: get: unexpected output ID" + return &resp, nil + } + + entry, err := p.c.Get(cache.ActionID(req.ActionID)) + if err == nil { + resp.OutputID = entry.OutputID[:] + resp.Size = entry.Size + resp.Time = &entry.Time + resp.DiskPath = p.c.OutputFile(entry.OutputID) + + return &resp, nil + } + + var notFound *cache.EntryNotFoundError + if errors.As(err, ¬Found) { + resp.Miss = true + return &resp, nil + } + + // it is not entirely clear what errors are possible there and how to handle them + // (should we send it in resp.Err? treat as fatal?) + p.l.Warn(fmt.Sprintf("get: %[1]s (%[1]T)", err)) + resp.Miss = true + return &resp, nil +} + +// handlePut handles the `put` command. +// Returned error is something fatal. +func (p *Prog) handlePut(req *cacheprog.Request) (*cacheprog.Response, error) { + resp := cacheprog.Response{ + ID: req.ID, + } + + if len(req.ActionID) != cache.HashSize { + resp.Err = fmt.Sprintf("hardcache: put: invalid action ID size %d, expected %d", len(req.ActionID), cache.HashSize) + return &resp, nil + } + + if len(req.OutputID) != cache.HashSize { + resp.Err = fmt.Sprintf("hardcache: put: invalid output ID size %d, expected %d", len(req.OutputID), cache.HashSize) + return &resp, nil + } + + var b []byte + if req.BodySize > 0 { + // currently, there is no way to make JSON decoder use preallocated slice capacity + if err := p.inD.Decode(&b); err != nil { + return nil, err + } + + if l := len(b); int(req.BodySize) != l { + return nil, fmt.Errorf("put: expected body size %d, got %d", req.BodySize, l) + } + } + + outputID, size, err := p.c.Put(cache.ActionID(req.ActionID), bytes.NewReader(b)) + if err != nil { + resp.Err = err.Error() + return &resp, nil + } + + if id := cache.OutputID(req.OutputID); id != outputID { + resp.Err = fmt.Sprintf("hardcache: put: expected output ID %x, got %x", id, outputID) + return &resp, nil + } + + if req.BodySize != size { + resp.Err = fmt.Sprintf("hardcache: put: expected size %d, got %d", req.BodySize, size) + return &resp, nil + } + + resp.DiskPath = p.c.OutputFile(outputID) + return &resp, nil +} + +// handleClose handles the `close` command. +// Returned error is something fatal. +func (p *Prog) handleClose(req *cacheprog.Request) (*cacheprog.Response, error) { + p.closed = true + return &cacheprog.Response{ID: req.ID}, nil +} + +// send sends the given response. +// Returned error is something fatal. +func (p *Prog) send(resp *cacheprog.Response) error { + p.outM.Lock() + defer p.outM.Unlock() + + if err := p.outE.Encode(resp); err != nil { + return err + } + + return p.outW.Flush() +} diff --git a/internal/prog/prog_client_test.go b/internal/prog/prog_client_test.go new file mode 100644 index 0000000..1d5e1d1 --- /dev/null +++ b/internal/prog/prog_client_test.go @@ -0,0 +1,110 @@ +package prog + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "sync" + + "github.com/AlekSi/lazyerrors" + + "github.com/AlekSi/hardcache/internal/go/cacheprog" +) + +// client is a simple GOCACHEPROG client for use in tests. +// +//nolint:vet // for readability +type client struct { + inM sync.Mutex + inD *json.Decoder + + outM sync.Mutex + outC io.Closer + outE *json.Encoder +} + +// newClient creates a new client connected to the given reader and writer. +func newClient(r io.Reader, w io.WriteCloser) (*client, error) { + c := &client{ + inD: json.NewDecoder(r), + outC: w, + outE: json.NewEncoder(w), + } + + resp, err := c.recv() + if err != nil { + _ = w.Close() + return nil, lazyerrors.Error(err) + } + + expected := &cacheprog.Response{ + KnownCommands: []cacheprog.Cmd{cacheprog.CmdGet, cacheprog.CmdPut, cacheprog.CmdClose}, + } + if !reflect.DeepEqual(resp, expected) { + _ = w.Close() + return nil, fmt.Errorf("client.newClient: expected initial response %+v, got %+v", expected, resp) + } + + return c, nil +} + +// close correctly closes the client connection. +func (c *client) close() (err error) { + defer func() { + if e := c.outC.Close(); e != nil && err == nil { + err = lazyerrors.Error(e) + } + + if _, e := c.recv(); !errors.Is(e, io.EOF) && err == nil { + err = lazyerrors.Error(e) + } + }() + + err = c.send(&cacheprog.Request{ + ID: 100500, + Command: cacheprog.CmdClose, + }) + if err != nil { + err = lazyerrors.Error(err) + return + } + + var resp *cacheprog.Response + if resp, err = c.recv(); err != nil { + err = lazyerrors.Error(err) + return + } + + expected := &cacheprog.Response{ + ID: 100500, + } + if !reflect.DeepEqual(resp, expected) { + err = fmt.Errorf("client.close: expected empty response, got %+v", resp) + return + } + + return +} + +// send sends a request. +func (c *client) send(req *cacheprog.Request) error { + c.outM.Lock() + defer c.outM.Unlock() + + return c.outE.Encode(req) +} + +// recv receives a response. +func (c *client) recv() (*cacheprog.Response, error) { + c.inM.Lock() + defer c.inM.Unlock() + + var resp cacheprog.Response + if err := c.inD.Decode(&resp); err != nil { + return nil, lazyerrors.Error(err) + } + + return &resp, nil +} diff --git a/internal/prog/prog_test.go b/internal/prog/prog_test.go new file mode 100644 index 0000000..486087c --- /dev/null +++ b/internal/prog/prog_test.go @@ -0,0 +1,79 @@ +package prog + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/go/cacheprog" +) + +// logger returns a [slog.Logger] for the given test. +func logger(t testing.TB) *slog.Logger { + t.Helper() + + opts := &slog.HandlerOptions{ + AddSource: true, + Level: slog.LevelDebug, + } + return slog.New(slog.NewTextHandler(t.Output(), opts)) +} + +func TestProg(t *testing.T) { + cache, err := local.New(t.TempDir(), nil, nil, logger(t)) + require.NoError(t, err) + + var p *Prog + var c *client + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + + { + pr, cw := io.Pipe() + cr, pw := io.Pipe() + + p = New(cache, l, pr, pw) + + go func() { + assert.NoError(t, p.Run(ctx)) + close(done) + }() + + c, err = newClient(cr, cw) + require.NoError(t, err) + } + + t.Cleanup(func() { + assert.NoError(t, c.close()) + + // TODO check sending command after close + + <-done + cancel() + }) + + t.Run("GetInvalidActionID", func(t *testing.T) { + t.Parallel() + + require.NoError(t, c.send(&cacheprog.Request{ + ID: 100, + Command: cacheprog.CmdGet, + ActionID: []byte("missing"), + })) + + resp, err := c.recv() + require.NoError(t, err) + + expected := &cacheprog.Response{ + ID: 100, + Err: "hardcache: get: invalid action ID size 7, expected 32", + } + assert.Equal(t, expected, resp) + }) +} diff --git a/main.go b/main.go index 030c259..2fee596 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,8 @@ import ( "github.com/alecthomas/kong" "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/caches/stat" + "github.com/AlekSi/hardcache/internal/prog" "github.com/AlekSi/hardcache/internal/sigterm" "github.com/AlekSi/hardcache/internal/unit" ) @@ -27,6 +29,7 @@ var cli struct { UnusedFor unit.Duration `default:"5d" help:"Always remove entries unused for this duration. Pass 0 to disable."` MaxSize string `default:"0GB" help:"${local_max_size_help}"` + Use struct{} `cmd:"" hidden:""` Trim struct{} `cmd:"" help:"Trim local cache."` Trimd struct { Interval unit.Duration `short:"i" default:"1h" help:"Interval between trimmings."` @@ -52,10 +55,10 @@ var GOCACHE = sync.OnceValue(func() string { return strings.TrimSpace(string(b)) }) -// localTrim force-trims local cache according to CLI flags. -func localTrim(l *slog.Logger) error { +// localCache creates local cache according to CLI flags. +func localCache(l *slog.Logger) (*local.Cache, error) { if cli.Local.UnusedFor < 0 { - return fmt.Errorf("--unused-for cannot be negative: %d", cli.Local.UnusedFor) + return nil, fmt.Errorf("--unused-for cannot be negative: %d", cli.Local.UnusedFor) } var cutoff *time.Time @@ -68,12 +71,12 @@ func localTrim(l *slog.Logger) error { if strings.HasSuffix(cli.Local.MaxSize, "%") { var p unit.Percentage if err := p.UnmarshalText([]byte(cli.Local.MaxSize)); err != nil { - return err + return nil, err } total, _, err := local.DiskInfo(cli.Local.Dir) if err != nil { - return err + return nil, err } b = unit.Bytes(total / 100 * int64(p)) @@ -87,14 +90,14 @@ func localTrim(l *slog.Logger) error { ) } else { if err := b.UnmarshalText([]byte(cli.Local.MaxSize)); err != nil { - return err + return nil, err } l.Debug("Max size", slog.Int64("max_size_bytes", int64(b)), slog.String("max_size", b.String())) } if b < 0 { - return fmt.Errorf("--max-size cannot be negative: %d", b) + return nil, fmt.Errorf("--max-size cannot be negative: %d", b) } var maxSize *int64 @@ -102,7 +105,12 @@ func localTrim(l *slog.Logger) error { maxSize = (*int64)(&b) } - c, err := local.New(cli.Local.Dir, cutoff, maxSize, l) + return local.New(cli.Local.Dir, cutoff, maxSize, l) +} + +// localTrim force-trims local cache according to CLI flags. +func localTrim(l *slog.Logger) error { + c, err := localCache(l) if err != nil { return err } @@ -151,6 +159,14 @@ func main() { defer cancel() switch kongCtx.Command() { + case "local use": + c, err := localCache(l) + kongCtx.FatalIfErrorf(err) + + p := prog.New(stat.New(c, l), l, os.Stdin, os.Stdout) + err = p.Run(ctx) + kongCtx.FatalIfErrorf(err) + case "local trim": if time.Duration(cli.Local.UnusedFor) > 5*24*time.Hour { l.Info("Note: this command should be invoked more often than once per day to keep the cache.")