From f70d9cb2258056de4b8f0aa3159ce031c0724d60 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:28:29 -0400 Subject: [PATCH 01/16] feat: add encrypted file bundles --- CHANGELOG.md | 1 + backup/backup.go | 36 +++- backup/backup_test.go | 33 +++- backup/files.go | 437 ++++++++++++++++++++++++++++++++++++++++++ backup/files_test.go | 131 +++++++++++++ backup/history.go | 26 +++ 6 files changed, 658 insertions(+), 6 deletions(-) create mode 100644 backup/files.go create mode 100644 backup/files_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d0b0d24..3398709 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add content-deduplicated encrypted file bundles with safe current and historical restore for crawler backups. - Add shared SQL LIKE literal escaping for crawler search queries. - Preserve Unicode combining marks in tokenized FTS5 queries. - Add a safe tokenized FTS5 query helper for arbitrary user search input. diff --git a/backup/backup.go b/backup/backup.go index 91f1468..56d2b3a 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -30,6 +30,7 @@ type Manifest struct { Recipients []string `json:"recipients,omitempty"` Counts map[string]int `json:"counts"` Shards []ShardEntry `json:"shards"` + Files []FileEntry `json:"files,omitempty"` } type Shard struct { @@ -47,12 +48,24 @@ type ShardEntry struct { Bytes int64 `json:"bytes"` } +type FileEntry struct { + Path string `json:"path"` + Shard string `json:"shard"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Bytes int64 `json:"bytes"` +} + type DecodedShard struct { Entry ShardEntry Plaintext []byte } func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest) (Manifest, error) { + return WriteSnapshotWithFiles(ctx, cfg, shards, nil, old) +} + +func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, files []File, old Manifest) (Manifest, error) { if err := ctx.Err(); err != nil { return Manifest{}, err } @@ -90,6 +103,11 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest manifest.Counts[countKey] += rows manifest.Shards = append(manifest.Shards, entry) } + filesManifest, err := writeFiles(ctx, cfg, old, files, reuseEncrypted) + if err != nil { + return Manifest{}, err + } + manifest.Files = filesManifest sort.Slice(manifest.Shards, func(i, j int) bool { return manifest.Shards[i].Path < manifest.Shards[j].Path }) if EquivalentManifest(old, manifest) { return old, nil @@ -103,7 +121,7 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest if err := ctx.Err(); err != nil { return Manifest{}, err } - if err := removeStaleShards(ctx, cfg.Repo, manifest.Shards); err != nil { + if err := removeStaleBackupFiles(ctx, cfg.Repo, manifest.Shards, manifest.Files); err != nil { return Manifest{}, err } return manifest, nil @@ -371,7 +389,7 @@ func (m Manifest) Entry(path string) (ShardEntry, bool) { } func EquivalentManifest(a, b Manifest) bool { - if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) { + if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) || len(a.Files) != len(b.Files) { return false } for i := range a.Shards { @@ -381,6 +399,13 @@ func EquivalentManifest(a, b Manifest) bool { return false } } + for i := range a.Files { + left, right := a.Files[i], b.Files[i] + left.Bytes, right.Bytes = 0, 0 + if left != right { + return false + } + } return true } @@ -389,6 +414,10 @@ func RemoveStaleShards(repo string, shards []ShardEntry) error { } func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) error { + return removeStaleBackupFiles(ctx, repo, shards, nil) +} + +func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntry, files []FileEntry) error { if err := ctx.Err(); err != nil { return err } @@ -396,6 +425,9 @@ func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) er for _, shard := range shards { keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(shard.Path)))] = struct{}{} } + for _, file := range files { + keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(file.Shard)))] = struct{}{} + } root := filepath.Join(repo, "data") if _, err := os.Stat(root); os.IsNotExist(err) { return nil diff --git a/backup/backup_test.go b/backup/backup_test.go index 3ba487d..65c8752 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -231,9 +231,13 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) { if err := mirror.EnsureRepo(ctx, mirrorOpts); err != nil { t.Fatal(err) } - firstManifest, err := WriteSnapshot(ctx, cfg, []Shard{ + mediaSource := filepath.Join(dir, "media.txt") + if err := os.WriteFile(mediaSource, []byte("first media"), 0o600); err != nil { + t.Fatal(err) + } + firstManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{ {Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "first"}}}, - }, Manifest{}) + }, []File{{Path: "media/file.txt", Source: mediaSource}}, Manifest{}) if err != nil { t.Fatal(err) } @@ -247,9 +251,12 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) { if _, err := mirror.CreateImmutableTag(ctx, mirrorOpts, "snapshot/one"); err != nil { t.Fatal(err) } - secondManifest, err := WriteSnapshot(ctx, cfg, []Shard{ + if err := os.WriteFile(mediaSource, []byte("second media"), 0o600); err != nil { + t.Fatal(err) + } + secondManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{ {Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "second"}}}, - }, firstManifest) + }, []File{{Path: "media/file.txt", Source: mediaSource}}, firstManifest) if err != nil { t.Fatal(err) } @@ -289,6 +296,24 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) { if len(historicalRows) != 1 || historicalRows[0].Body != "first" { t.Fatalf("historical rows = %#v", historicalRows) } + restoreRoot := filepath.Join(dir, "historical-restore") + if err := os.MkdirAll(restoreRoot, 0o700); err != nil { + t.Fatal(err) + } + restored, restoredRef, err := RestoreFilesAt(ctx, cfg, mirrorOpts, manifest, "snapshot/one", restoreRoot) + if err != nil { + t.Fatal(err) + } + if restored != 1 || restoredRef != first { + t.Fatalf("historical files = %d at %s", restored, restoredRef) + } + mediaBody, err := os.ReadFile(filepath.Join(restoreRoot, "media", "file.txt")) + if err != nil { + t.Fatal(err) + } + if string(mediaBody) != "first media" { + t.Fatalf("historical media = %q", mediaBody) + } if secondManifest.Counts["messages"] != 1 { t.Fatalf("second manifest = %#v", secondManifest) } diff --git a/backup/files.go b/backup/files.go new file mode 100644 index 0000000..81e5e58 --- /dev/null +++ b/backup/files.go @@ -0,0 +1,437 @@ +package backup + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "filippo.io/age" +) + +type File struct { + Path string + Source string +} + +type encryptedFileLoader func(FileEntry) (io.ReadCloser, error) + +func CollectFiles(ctx context.Context, root, prefix string) ([]File, error) { + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." { + return nil, fmt.Errorf("file root is required") + } + prefix = strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/") + if prefix != "" { + if _, err := cleanFilePath(prefix); err != nil { + return nil, err + } + } + if _, err := os.Stat(root); os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + var files []File + err := filepath.WalkDir(root, func(source string, entry os.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return walkErr + } + if entry.Type()&os.ModeSymlink != 0 { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, source) + if err != nil { + return err + } + logical := filepath.ToSlash(rel) + if prefix != "" { + logical = path.Join(prefix, logical) + } + logical, err = cleanFilePath(logical) + if err != nil { + return err + } + files = append(files, File{Path: logical, Source: source}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + return files, nil +} + +func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reuseEncrypted bool) ([]FileEntry, error) { + oldByHash := make(map[string]FileEntry, len(old.Files)) + if reuseEncrypted { + for _, entry := range old.Files { + oldByHash[entry.SHA256] = entry + } + } + written := make(map[string]FileEntry, len(files)) + seenPaths := make(map[string]struct{}, len(files)) + out := make([]FileEntry, 0, len(files)) + for _, file := range files { + if err := ctx.Err(); err != nil { + return nil, err + } + logical, err := cleanFilePath(file.Path) + if err != nil { + return nil, err + } + if _, exists := seenPaths[logical]; exists { + return nil, fmt.Errorf("duplicate backup file path: %s", logical) + } + seenPaths[logical] = struct{}{} + info, err := os.Lstat(file.Source) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("backup file is not regular: %s", file.Source) + } + tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file.Source, cfg.Repo, cfg.Recipients) + if err != nil { + return nil, err + } + keepTemp := true + defer func() { + if keepTemp { + _ = os.Remove(tmpPath) + } + }() + if entry, ok := written[hashValue]; ok { + if err := os.Remove(tmpPath); err != nil { + return nil, err + } + keepTemp = false + entry.Path = logical + out = append(out, entry) + continue + } + shard := path.Join("data/files", hashValue[:2], hashValue+".gz.age") + if oldEntry, ok := oldByHash[hashValue]; ok && oldEntry.Shard == shard { + target, resolveErr := ResolveShardPath(cfg.Repo, shard) + if resolveErr != nil { + return nil, resolveErr + } + if encryptedInfo, statErr := os.Stat(target); statErr == nil { + if err := os.Remove(tmpPath); err != nil { + return nil, err + } + keepTemp = false + entry := FileEntry{Path: logical, Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedInfo.Size()} + written[hashValue] = entry + out = append(out, entry) + continue + } + } + target, err := ResolveShardPath(cfg.Repo, shard) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return nil, err + } + if err := os.Rename(tmpPath, target); err != nil { + return nil, err + } + keepTemp = false + if err := syncDir(filepath.Dir(target)); err != nil { + return nil, err + } + entry := FileEntry{Path: logical, Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedSize} + written[hashValue] = entry + out = append(out, entry) + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out, nil +} + +func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot string) (int, error) { + return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(entry FileEntry) (io.ReadCloser, error) { + shard, err := ResolveShardPath(cfg.Repo, entry.Shard) + if err != nil { + return nil, err + } + return os.Open(shard) // #nosec G304 -- ResolveShardPath confines manifest-controlled paths below data/. + }) +} + +func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifest, targetRoot string, load encryptedFileLoader) (int, error) { + if manifest.Format != FormatVersion { + return 0, fmt.Errorf("unsupported backup format %d", manifest.Format) + } + identityData, err := os.ReadFile(expandHome(identityPath)) // #nosec G304 -- path is configured by the caller. + if err != nil { + return 0, err + } + identity, err := parseIdentity(identityData) + if err != nil { + return 0, err + } + seen := make(map[string]struct{}, len(manifest.Files)) + for _, entry := range manifest.Files { + if err := ctx.Err(); err != nil { + return 0, err + } + logical, err := cleanFilePath(entry.Path) + if err != nil { + return 0, err + } + if _, exists := seen[logical]; exists { + return 0, fmt.Errorf("duplicate backup file path: %s", logical) + } + seen[logical] = struct{}{} + if _, err := ResolveShardPath(".", entry.Shard); err != nil { + return 0, err + } + ciphertext, err := load(entry) + if err != nil { + return 0, err + } + err = restoreFile(ctx, identity, ciphertext, targetRoot, logical, entry) + closeErr := ciphertext.Close() + if err != nil { + return 0, err + } + if closeErr != nil { + return 0, closeErr + } + } + return len(manifest.Files), nil +} + +func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext io.Reader, targetRoot, logical string, entry FileEntry) error { + target, err := safeRestoreTarget(targetRoot, logical) + if err != nil { + return err + } + decrypted, err := age.Decrypt(ciphertext, identity) + if err != nil { + return err + } + gz, err := gzip.NewReader(decrypted) + if err != nil { + return err + } + defer gz.Close() + tmp, err := os.CreateTemp(filepath.Dir(target), "."+filepath.Base(target)+".tmp-") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + hasher := sha256.New() + size, err := copyContext(ctx, io.MultiWriter(tmp, hasher), gz) + if err != nil { + _ = tmp.Close() + return err + } + if err := gz.Close(); err != nil { + _ = tmp.Close() + return err + } + if size != entry.Size || hex.EncodeToString(hasher.Sum(nil)) != entry.SHA256 { + _ = tmp.Close() + return fmt.Errorf("backup file verification failed for %s", logical) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpPath, target); err != nil { + return err + } + return syncDir(filepath.Dir(target)) +} + +func encryptFileTemp(ctx context.Context, source, repo string, recipientStrings []string) (string, string, int64, int64, error) { + recipients, err := parseRecipients(recipientStrings) + if err != nil { + return "", "", 0, 0, err + } + tmpDir := filepath.Join(repo, "data", "files") + if err := os.MkdirAll(tmpDir, 0o700); err != nil { + return "", "", 0, 0, err + } + in, err := os.Open(source) // #nosec G304 -- source is explicitly supplied by the caller. + if err != nil { + return "", "", 0, 0, err + } + defer in.Close() + tmp, err := os.CreateTemp(tmpDir, ".file.tmp-") + if err != nil { + return "", "", 0, 0, err + } + tmpPath := tmp.Name() + cleanup := true + defer func() { + if cleanup { + _ = os.Remove(tmpPath) + } + }() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + encrypted, err := age.Encrypt(tmp, recipients...) + if err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + gz := gzip.NewWriter(encrypted) + gz.ModTime = time.Unix(0, 0).UTC() + hasher := sha256.New() + size, err := copyContext(ctx, gz, io.TeeReader(in, hasher)) + if err != nil { + _ = gz.Close() + _ = encrypted.Close() + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := gz.Close(); err != nil { + _ = encrypted.Close() + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := encrypted.Close(); err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := tmp.Close(); err != nil { + return "", "", 0, 0, err + } + info, err := os.Stat(tmpPath) + if err != nil { + return "", "", 0, 0, err + } + cleanup = false + return tmpPath, hex.EncodeToString(hasher.Sum(nil)), size, info.Size(), nil +} + +func safeRestoreTarget(root, logical string) (string, error) { + logical, err := cleanFilePath(logical) + if err != nil { + return "", err + } + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." { + return "", fmt.Errorf("restore root is required") + } + if err := ensureDirectory(root); err != nil { + return "", err + } + parts := strings.Split(logical, "/") + parent := root + for _, part := range parts[:len(parts)-1] { + parent = filepath.Join(parent, part) + if err := ensureDirectory(parent); err != nil { + return "", err + } + } + target := filepath.Join(root, filepath.FromSlash(logical)) + if info, err := os.Lstat(target); err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return "", fmt.Errorf("restore target is not a regular file: %s", logical) + } + } else if !os.IsNotExist(err) { + return "", err + } + return target, nil +} + +func ensureDirectory(dir string) error { + info, err := os.Lstat(dir) + if os.IsNotExist(err) { + return os.Mkdir(dir, 0o700) + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("restore path is not a directory: %s", dir) + } + return nil +} + +func cleanFilePath(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" || strings.Contains(value, "\\") { + return "", fmt.Errorf("invalid backup file path: %s", value) + } + clean := path.Clean(value) + local := filepath.FromSlash(clean) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) || filepath.IsAbs(local) || filepath.VolumeName(local) != "" { + return "", fmt.Errorf("backup file path escapes restore root: %s", value) + } + return clean, nil +} + +func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { + buffer := make([]byte, 256*1024) + var total int64 + for { + if err := ctx.Err(); err != nil { + return total, err + } + read, readErr := src.Read(buffer) + if read > 0 { + written, writeErr := dst.Write(buffer[:read]) + total += int64(written) + if writeErr != nil { + return total, writeErr + } + if written != read { + return total, io.ErrShortWrite + } + } + if readErr == io.EOF { + return total, nil + } + if readErr != nil { + return total, readErr + } + } +} + +func readEncryptedFileBytes(data []byte) io.ReadCloser { + return io.NopCloser(bytes.NewReader(data)) +} diff --git a/backup/files_test.go b/backup/files_test.go new file mode 100644 index 0000000..cbae681 --- /dev/null +++ b/backup/files_test.go @@ -0,0 +1,131 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + source := filepath.Join(dir, "source") + if err := os.MkdirAll(filepath.Join(source, "nested"), 0o700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"photo.jpg", filepath.Join("nested", "copy.jpg")} { + if err := os.WriteFile(filepath.Join(source, name), []byte("same private media"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(filepath.Join(source, "photo.jpg"), filepath.Join(source, "linked.jpg")); err != nil { + t.Fatal(err) + } + files, err := CollectFiles(ctx, source, "media") + if err != nil { + t.Fatal(err) + } + if len(files) != 2 { + t.Fatalf("collected files = %#v", files) + } + identity := filepath.Join(dir, "age.key") + recipient, err := EnsureIdentity(identity) + if err != nil { + t.Fatal(err) + } + cfg := Config{Repo: filepath.Join(dir, "repo"), Identity: identity, Recipients: []string{recipient}} + if err := os.MkdirAll(cfg.Repo, 0o700); err != nil { + t.Fatal(err) + } + manifest, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}) + if err != nil { + t.Fatal(err) + } + if len(manifest.Files) != 2 || manifest.Files[0].Shard != manifest.Files[1].Shard { + t.Fatalf("files were not content-deduplicated: %#v", manifest.Files) + } + ciphertext, err := os.ReadFile(filepath.Join(cfg.Repo, filepath.FromSlash(manifest.Files[0].Shard))) + if err != nil { + t.Fatal(err) + } + if string(ciphertext) == "same private media" { + t.Fatal("backup file was stored as plaintext") + } + second, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, manifest) + if err != nil { + t.Fatal(err) + } + if !EquivalentManifest(manifest, second) || !second.Exported.Equal(manifest.Exported) { + t.Fatalf("unchanged files rewrote manifest: %#v", second) + } + restoreRoot := filepath.Join(dir, "restore") + if err := os.MkdirAll(restoreRoot, 0o700); err != nil { + t.Fatal(err) + } + count, err := RestoreFiles(ctx, cfg, manifest, restoreRoot) + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("restored files = %d", count) + } + for _, name := range []string{filepath.Join("media", "photo.jpg"), filepath.Join("media", "nested", "copy.jpg")} { + body, err := os.ReadFile(filepath.Join(restoreRoot, name)) + if err != nil { + t.Fatal(err) + } + if string(body) != "same private media" { + t.Fatalf("restored %s = %q", name, body) + } + } +} + +func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + identity := filepath.Join(dir, "age.key") + if _, err := EnsureIdentity(identity); err != nil { + t.Fatal(err) + } + cfg := Config{Repo: dir, Identity: identity} + manifest := Manifest{Format: FormatVersion, Files: []FileEntry{{Path: "../escape", Shard: "data/files/aa/file.age"}}} + if _, err := RestoreFiles(ctx, cfg, manifest, filepath.Join(dir, "restore")); err == nil { + t.Fatal("unsafe restore path should fail") + } + + source := filepath.Join(dir, "source") + if err := os.MkdirAll(source, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "file"), []byte("body"), 0o600); err != nil { + t.Fatal(err) + } + files, err := CollectFiles(ctx, source, "media") + if err != nil { + t.Fatal(err) + } + recipient, err := RecipientFromIdentity(identity) + if err != nil { + t.Fatal(err) + } + cfg.Recipients = []string{recipient} + manifest, err = WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}) + if err != nil { + t.Fatal(err) + } + restoreRoot := filepath.Join(dir, "target") + outside := filepath.Join(dir, "outside") + if err := os.MkdirAll(outside, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(restoreRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(restoreRoot, "media")); err != nil { + t.Fatal(err) + } + if _, err := RestoreFiles(ctx, cfg, manifest, restoreRoot); err == nil { + t.Fatal("symlinked restore directory should fail") + } +} diff --git a/backup/history.go b/backup/history.go index 44b21a5..efffe1b 100644 --- a/backup/history.go +++ b/backup/history.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "path" "strings" @@ -85,3 +86,28 @@ func ReadSnapshotAt(ctx context.Context, cfg Config, opts mirror.Options, manife } return shards, commit, nil } + +func RestoreFilesAt(ctx context.Context, cfg Config, opts mirror.Options, manifest Manifest, ref, targetRoot string) (int, string, error) { + commit, err := mirror.ResolveCommit(ctx, opts, ref) + if err != nil { + return 0, "", err + } + count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(entry FileEntry) (io.ReadCloser, error) { + if _, err := ResolveShardPath(cfg.Repo, entry.Shard); err != nil { + return nil, err + } + clean := path.Clean(strings.TrimSpace(entry.Shard)) + ciphertext, resolved, err := mirror.ReadFileAt(ctx, opts, commit, clean) + if err != nil { + return nil, err + } + if resolved != commit { + return nil, fmt.Errorf("backup ref changed while reading %s", clean) + } + return readEncryptedFileBytes(ciphertext), nil + }) + if err != nil { + return 0, "", err + } + return count, commit, nil +} From 1cddbe5f380baac68fd3dab30cab9ab44d406c78 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:31:59 -0400 Subject: [PATCH 02/16] fix: encrypt backup file paths --- backup/backup.go | 17 +++++- backup/files.go | 136 ++++++++++++++++++++++++++++++++++--------- backup/files_test.go | 13 ++++- 3 files changed, 135 insertions(+), 31 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 56d2b3a..99681fd 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -49,7 +49,6 @@ type ShardEntry struct { } type FileEntry struct { - Path string `json:"path"` Shard string `json:"shard"` SHA256 string `json:"sha256"` Size int64 `json:"size"` @@ -103,11 +102,22 @@ func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, fil manifest.Counts[countKey] += rows manifest.Shards = append(manifest.Shards, entry) } - filesManifest, err := writeFiles(ctx, cfg, old, files, reuseEncrypted) + filesManifest, fileIndex, err := writeFiles(ctx, cfg, old, files, reuseEncrypted) if err != nil { return Manifest{}, err } manifest.Files = filesManifest + if len(fileIndex) > 0 { + plaintext, rows, err := encodeJSONL(ctx, fileIndex) + if err != nil { + return Manifest{}, fmt.Errorf("encode backup file index: %w", err) + } + entry, err := writeShard(ctx, cfg, old, fileIndexTable, fileIndexPath, plaintext, rows, reuseEncrypted) + if err != nil { + return Manifest{}, err + } + manifest.Shards = append(manifest.Shards, entry) + } sort.Slice(manifest.Shards, func(i, j int) bool { return manifest.Shards[i].Path < manifest.Shards[j].Path }) if EquivalentManifest(old, manifest) { return old, nil @@ -139,6 +149,9 @@ func readSnapshotWith(manifest Manifest, load func(ShardEntry) ([]byte, error)) } var out []DecodedShard for _, shard := range manifest.Shards { + if shard.Table == fileIndexTable { + continue + } plaintext, err := load(shard) if err != nil { return nil, err diff --git a/backup/files.go b/backup/files.go index 81e5e58..a58c0cf 100644 --- a/backup/files.go +++ b/backup/files.go @@ -23,7 +23,16 @@ type File struct { Source string } -type encryptedFileLoader func(FileEntry) (io.ReadCloser, error) +const ( + fileIndexTable = "_backup_files" + fileIndexPath = "data/files/index.jsonl.gz.age" +) + +type fileIndexRecord struct { + Path string `json:"path"` +} + +type encryptedFileLoader func(string) (io.ReadCloser, error) func CollectFiles(ctx context.Context, root, prefix string) ([]File, error) { root = filepath.Clean(strings.TrimSpace(root)) @@ -87,7 +96,9 @@ func CollectFiles(ctx context.Context, root, prefix string) ([]File, error) { return files, nil } -func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reuseEncrypted bool) ([]FileEntry, error) { +func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reuseEncrypted bool) ([]FileEntry, []fileIndexRecord, error) { + ordered := append([]File(nil), files...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].Path < ordered[j].Path }) oldByHash := make(map[string]FileEntry, len(old.Files)) if reuseEncrypted { for _, entry := range old.Files { @@ -97,28 +108,30 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu written := make(map[string]FileEntry, len(files)) seenPaths := make(map[string]struct{}, len(files)) out := make([]FileEntry, 0, len(files)) - for _, file := range files { + index := make([]fileIndexRecord, 0, len(files)) + for _, file := range ordered { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } logical, err := cleanFilePath(file.Path) if err != nil { - return nil, err + return nil, nil, err } if _, exists := seenPaths[logical]; exists { - return nil, fmt.Errorf("duplicate backup file path: %s", logical) + return nil, nil, fmt.Errorf("duplicate backup file path: %s", logical) } seenPaths[logical] = struct{}{} + index = append(index, fileIndexRecord{Path: logical}) info, err := os.Lstat(file.Source) if err != nil { - return nil, err + return nil, nil, err } if !info.Mode().IsRegular() { - return nil, fmt.Errorf("backup file is not regular: %s", file.Source) + return nil, nil, fmt.Errorf("backup file is not regular: %s", file.Source) } tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file.Source, cfg.Repo, cfg.Recipients) if err != nil { - return nil, err + return nil, nil, err } keepTemp := true defer func() { @@ -128,10 +141,9 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu }() if entry, ok := written[hashValue]; ok { if err := os.Remove(tmpPath); err != nil { - return nil, err + return nil, nil, err } keepTemp = false - entry.Path = logical out = append(out, entry) continue } @@ -139,14 +151,14 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu if oldEntry, ok := oldByHash[hashValue]; ok && oldEntry.Shard == shard { target, resolveErr := ResolveShardPath(cfg.Repo, shard) if resolveErr != nil { - return nil, resolveErr + return nil, nil, resolveErr } if encryptedInfo, statErr := os.Stat(target); statErr == nil { if err := os.Remove(tmpPath); err != nil { - return nil, err + return nil, nil, err } keepTemp = false - entry := FileEntry{Path: logical, Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedInfo.Size()} + entry := FileEntry{Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedInfo.Size()} written[hashValue] = entry out = append(out, entry) continue @@ -154,29 +166,28 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu } target, err := ResolveShardPath(cfg.Repo, shard) if err != nil { - return nil, err + return nil, nil, err } if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { - return nil, err + return nil, nil, err } if err := os.Rename(tmpPath, target); err != nil { - return nil, err + return nil, nil, err } keepTemp = false if err := syncDir(filepath.Dir(target)); err != nil { - return nil, err + return nil, nil, err } - entry := FileEntry{Path: logical, Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedSize} + entry := FileEntry{Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedSize} written[hashValue] = entry out = append(out, entry) } - sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) - return out, nil + return out, index, nil } func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot string) (int, error) { - return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(entry FileEntry) (io.ReadCloser, error) { - shard, err := ResolveShardPath(cfg.Repo, entry.Shard) + return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(rel string) (io.ReadCloser, error) { + shard, err := ResolveShardPath(cfg.Repo, rel) if err != nil { return nil, err } @@ -196,12 +207,17 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes if err != nil { return 0, err } + paths, err := readFileIndex(identity, manifest, load) + if err != nil { + return 0, err + } seen := make(map[string]struct{}, len(manifest.Files)) - for _, entry := range manifest.Files { + for index, entry := range manifest.Files { if err := ctx.Err(); err != nil { return 0, err } - logical, err := cleanFilePath(entry.Path) + logical := paths[index] + logical, err = cleanFilePath(logical) if err != nil { return 0, err } @@ -212,7 +228,7 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes if _, err := ResolveShardPath(".", entry.Shard); err != nil { return 0, err } - ciphertext, err := load(entry) + ciphertext, err := load(entry.Shard) if err != nil { return 0, err } @@ -228,6 +244,74 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes return len(manifest.Files), nil } +func readFileIndex(identity *age.X25519Identity, manifest Manifest, load encryptedFileLoader) ([]string, error) { + if len(manifest.Files) == 0 { + return []string{}, nil + } + var indexEntry *ShardEntry + for index := range manifest.Shards { + if manifest.Shards[index].Table != fileIndexTable { + continue + } + if indexEntry != nil { + return nil, fmt.Errorf("backup contains multiple file indexes") + } + indexEntry = &manifest.Shards[index] + } + if indexEntry == nil { + return nil, fmt.Errorf("backup file index is missing") + } + ciphertext, err := load(indexEntry.Path) + if err != nil { + return nil, err + } + plaintext, decryptErr := decryptFilePayload(identity, ciphertext) + closeErr := ciphertext.Close() + if decryptErr != nil { + return nil, decryptErr + } + if closeErr != nil { + return nil, closeErr + } + if SHA256Hex(plaintext) != indexEntry.SHA256 { + return nil, fmt.Errorf("backup file index hash mismatch") + } + var records []fileIndexRecord + if err := DecodeJSONL(plaintext, &records); err != nil { + return nil, fmt.Errorf("decode backup file index: %w", err) + } + if len(records) != len(manifest.Files) { + return nil, fmt.Errorf("backup file index count mismatch") + } + paths := make([]string, 0, len(records)) + seen := make(map[string]struct{}, len(records)) + for _, record := range records { + logical, err := cleanFilePath(record.Path) + if err != nil { + return nil, err + } + if _, exists := seen[logical]; exists { + return nil, fmt.Errorf("duplicate backup file index path: %s", logical) + } + seen[logical] = struct{}{} + paths = append(paths, logical) + } + return paths, nil +} + +func decryptFilePayload(identity *age.X25519Identity, ciphertext io.Reader) ([]byte, error) { + decrypted, err := age.Decrypt(ciphertext, identity) + if err != nil { + return nil, err + } + gz, err := gzip.NewReader(decrypted) + if err != nil { + return nil, err + } + defer gz.Close() + return io.ReadAll(gz) +} + func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext io.Reader, targetRoot, logical string, entry FileEntry) error { target, err := safeRestoreTarget(targetRoot, logical) if err != nil { diff --git a/backup/files_test.go b/backup/files_test.go index cbae681..87c4b40 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" ) @@ -45,6 +46,13 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { if len(manifest.Files) != 2 || manifest.Files[0].Shard != manifest.Files[1].Shard { t.Fatalf("files were not content-deduplicated: %#v", manifest.Files) } + manifestBody, err := os.ReadFile(filepath.Join(cfg.Repo, "manifest.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(manifestBody), "photo.jpg") || strings.Contains(string(manifestBody), "copy.jpg") { + t.Fatalf("manifest exposes logical media paths: %s", manifestBody) + } ciphertext, err := os.ReadFile(filepath.Join(cfg.Repo, filepath.FromSlash(manifest.Files[0].Shard))) if err != nil { t.Fatal(err) @@ -89,8 +97,7 @@ func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { t.Fatal(err) } cfg := Config{Repo: dir, Identity: identity} - manifest := Manifest{Format: FormatVersion, Files: []FileEntry{{Path: "../escape", Shard: "data/files/aa/file.age"}}} - if _, err := RestoreFiles(ctx, cfg, manifest, filepath.Join(dir, "restore")); err == nil { + if _, err := safeRestoreTarget(filepath.Join(dir, "restore"), "../escape"); err == nil { t.Fatal("unsafe restore path should fail") } @@ -110,7 +117,7 @@ func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { t.Fatal(err) } cfg.Recipients = []string{recipient} - manifest, err = WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}) + manifest, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}) if err != nil { t.Fatal(err) } From c17b8e88fb337c06a2cc0f3087ca44a0be1b5fc6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:32:06 -0400 Subject: [PATCH 03/16] fix: restore encrypted file indexes at refs --- backup/history.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backup/history.go b/backup/history.go index efffe1b..ca450c9 100644 --- a/backup/history.go +++ b/backup/history.go @@ -92,11 +92,11 @@ func RestoreFilesAt(ctx context.Context, cfg Config, opts mirror.Options, manife if err != nil { return 0, "", err } - count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(entry FileEntry) (io.ReadCloser, error) { - if _, err := ResolveShardPath(cfg.Repo, entry.Shard); err != nil { + count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(rel string) (io.ReadCloser, error) { + if _, err := ResolveShardPath(cfg.Repo, rel); err != nil { return nil, err } - clean := path.Clean(strings.TrimSpace(entry.Shard)) + clean := path.Clean(strings.TrimSpace(rel)) ciphertext, resolved, err := mirror.ReadFileAt(ctx, opts, commit, clean) if err != nil { return nil, err From 735163c76eb3a51012b8bbb7452ca50ff706835a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:35:14 -0400 Subject: [PATCH 04/16] fix: skip directory sync on Windows --- backup/backup.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backup/backup.go b/backup/backup.go index 99681fd..c8f94c9 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "reflect" + "runtime" "sort" "strings" "time" @@ -384,6 +385,9 @@ func writeFileAtomicContext(ctx context.Context, target string, data []byte, per } func syncDir(dir string) error { + if runtime.GOOS == "windows" { + return nil + } file, err := os.Open(dir) if err != nil { return err From fb3715ad16df6532275f208548a0e1851d11a042 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:36:28 -0400 Subject: [PATCH 05/16] perf: reuse unchanged encrypted files --- backup/files.go | 69 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/backup/files.go b/backup/files.go index a58c0cf..ec695e8 100644 --- a/backup/files.go +++ b/backup/files.go @@ -129,6 +129,16 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu if !info.Mode().IsRegular() { return nil, nil, fmt.Errorf("backup file is not regular: %s", file.Source) } + hashValue, size, err := hashFile(ctx, file.Source) + if err != nil { + return nil, nil, err + } + if entry, ok, err := reusableFileEntry(cfg, oldByHash, written, hashValue, size); err != nil { + return nil, nil, err + } else if ok { + out = append(out, entry) + continue + } tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file.Source, cfg.Repo, cfg.Recipients) if err != nil { return nil, nil, err @@ -139,7 +149,9 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu _ = os.Remove(tmpPath) } }() - if entry, ok := written[hashValue]; ok { + if entry, ok, err := reusableFileEntry(cfg, oldByHash, written, hashValue, size); err != nil { + return nil, nil, err + } else if ok { if err := os.Remove(tmpPath); err != nil { return nil, nil, err } @@ -148,22 +160,6 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu continue } shard := path.Join("data/files", hashValue[:2], hashValue+".gz.age") - if oldEntry, ok := oldByHash[hashValue]; ok && oldEntry.Shard == shard { - target, resolveErr := ResolveShardPath(cfg.Repo, shard) - if resolveErr != nil { - return nil, nil, resolveErr - } - if encryptedInfo, statErr := os.Stat(target); statErr == nil { - if err := os.Remove(tmpPath); err != nil { - return nil, nil, err - } - keepTemp = false - entry := FileEntry{Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedInfo.Size()} - written[hashValue] = entry - out = append(out, entry) - continue - } - } target, err := ResolveShardPath(cfg.Repo, shard) if err != nil { return nil, nil, err @@ -185,6 +181,31 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu return out, index, nil } +func reusableFileEntry(cfg Config, oldByHash, written map[string]FileEntry, hashValue string, size int64) (FileEntry, bool, error) { + if entry, ok := written[hashValue]; ok { + return entry, true, nil + } + shard := path.Join("data/files", hashValue[:2], hashValue+".gz.age") + oldEntry, ok := oldByHash[hashValue] + if !ok || oldEntry.Shard != shard || oldEntry.Size != size { + return FileEntry{}, false, nil + } + target, err := ResolveShardPath(cfg.Repo, shard) + if err != nil { + return FileEntry{}, false, err + } + info, err := os.Stat(target) + if err != nil { + if os.IsNotExist(err) { + return FileEntry{}, false, nil + } + return FileEntry{}, false, err + } + entry := FileEntry{Shard: shard, SHA256: hashValue, Size: size, Bytes: info.Size()} + written[hashValue] = entry + return entry, true, nil +} + func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot string) (int, error) { return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(rel string) (io.ReadCloser, error) { shard, err := ResolveShardPath(cfg.Repo, rel) @@ -431,6 +452,20 @@ func encryptFileTemp(ctx context.Context, source, repo string, recipientStrings return tmpPath, hex.EncodeToString(hasher.Sum(nil)), size, info.Size(), nil } +func hashFile(ctx context.Context, source string) (string, int64, error) { + file, err := os.Open(source) // #nosec G304 -- source is explicitly supplied by the caller. + if err != nil { + return "", 0, err + } + defer file.Close() + hasher := sha256.New() + size, err := copyContext(ctx, hasher, file) + if err != nil { + return "", 0, err + } + return hex.EncodeToString(hasher.Sum(nil)), size, nil +} + func safeRestoreTarget(root, logical string) (string, error) { logical, err := cleanFilePath(logical) if err != nil { From ed684b31aed4662c05c7ceaa592f260646bf9246 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:38:58 -0400 Subject: [PATCH 06/16] fix: bind encrypted files to collected sources --- backup/files.go | 56 ++++++++++++++++++++++++++++++-------------- backup/files_test.go | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 18 deletions(-) diff --git a/backup/files.go b/backup/files.go index ec695e8..285d135 100644 --- a/backup/files.go +++ b/backup/files.go @@ -21,6 +21,7 @@ import ( type File struct { Path string Source string + info os.FileInfo } const ( @@ -86,7 +87,7 @@ func CollectFiles(ctx context.Context, root, prefix string) ([]File, error) { if err != nil { return err } - files = append(files, File{Path: logical, Source: source}) + files = append(files, File{Path: logical, Source: source, info: info}) return nil }) if err != nil { @@ -122,14 +123,7 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu } seenPaths[logical] = struct{}{} index = append(index, fileIndexRecord{Path: logical}) - info, err := os.Lstat(file.Source) - if err != nil { - return nil, nil, err - } - if !info.Mode().IsRegular() { - return nil, nil, fmt.Errorf("backup file is not regular: %s", file.Source) - } - hashValue, size, err := hashFile(ctx, file.Source) + hashValue, size, sourceInfo, err := hashFile(ctx, file) if err != nil { return nil, nil, err } @@ -139,7 +133,7 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu out = append(out, entry) continue } - tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file.Source, cfg.Repo, cfg.Recipients) + tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file, sourceInfo, cfg.Repo, cfg.Recipients) if err != nil { return nil, nil, err } @@ -384,7 +378,7 @@ func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext i return syncDir(filepath.Dir(target)) } -func encryptFileTemp(ctx context.Context, source, repo string, recipientStrings []string) (string, string, int64, int64, error) { +func encryptFileTemp(ctx context.Context, source File, expected os.FileInfo, repo string, recipientStrings []string) (string, string, int64, int64, error) { recipients, err := parseRecipients(recipientStrings) if err != nil { return "", "", 0, 0, err @@ -393,7 +387,7 @@ func encryptFileTemp(ctx context.Context, source, repo string, recipientStrings if err := os.MkdirAll(tmpDir, 0o700); err != nil { return "", "", 0, 0, err } - in, err := os.Open(source) // #nosec G304 -- source is explicitly supplied by the caller. + in, _, err := openSourceFile(source, expected) if err != nil { return "", "", 0, 0, err } @@ -452,18 +446,45 @@ func encryptFileTemp(ctx context.Context, source, repo string, recipientStrings return tmpPath, hex.EncodeToString(hasher.Sum(nil)), size, info.Size(), nil } -func hashFile(ctx context.Context, source string) (string, int64, error) { - file, err := os.Open(source) // #nosec G304 -- source is explicitly supplied by the caller. +func hashFile(ctx context.Context, source File) (string, int64, os.FileInfo, error) { + file, info, err := openSourceFile(source, source.info) if err != nil { - return "", 0, err + return "", 0, nil, err } defer file.Close() hasher := sha256.New() size, err := copyContext(ctx, hasher, file) if err != nil { - return "", 0, err + return "", 0, nil, err + } + return hex.EncodeToString(hasher.Sum(nil)), size, info, nil +} + +func openSourceFile(source File, expected os.FileInfo) (*os.File, os.FileInfo, error) { + before, err := os.Lstat(source.Source) + if err != nil { + return nil, nil, err + } + if !before.Mode().IsRegular() { + return nil, nil, fmt.Errorf("backup file is not regular: %s", source.Source) + } + if expected != nil && !os.SameFile(expected, before) { + return nil, nil, fmt.Errorf("backup file changed during collection: %s", source.Source) + } + file, err := os.Open(source.Source) // #nosec G304 -- identity checks bind the opened file to the collected regular file before reads. + if err != nil { + return nil, nil, err + } + after, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, nil, err + } + if !after.Mode().IsRegular() || !os.SameFile(before, after) || (expected != nil && !os.SameFile(expected, after)) { + _ = file.Close() + return nil, nil, fmt.Errorf("backup file changed before open: %s", source.Source) } - return hex.EncodeToString(hasher.Sum(nil)), size, nil + return file, after, nil } func safeRestoreTarget(root, logical string) (string, error) { @@ -512,7 +533,6 @@ func ensureDirectory(dir string) error { } func cleanFilePath(value string) (string, error) { - value = strings.TrimSpace(value) if value == "" || strings.Contains(value, "\\") { return "", fmt.Errorf("invalid backup file path: %s", value) } diff --git a/backup/files_test.go b/backup/files_test.go index 87c4b40..c10a12f 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -136,3 +136,47 @@ func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { t.Fatal("symlinked restore directory should fail") } } + +func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + root := filepath.Join(dir, "source") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"name", "name "} { + if err := os.WriteFile(filepath.Join(root, name), []byte(name), 0o600); err != nil { + t.Fatal(err) + } + } + files, err := CollectFiles(ctx, root, "media") + if err != nil { + t.Fatal(err) + } + if len(files) != 2 || files[0].Path != "media/name" || files[1].Path != "media/name " { + t.Fatalf("whitespace paths changed: %#v", files) + } + + outside := filepath.Join(dir, "outside") + if err := os.WriteFile(outside, []byte("outside secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Remove(files[0].Source); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, files[0].Source); err != nil { + t.Fatal(err) + } + identity := filepath.Join(dir, "age.key") + recipient, err := EnsureIdentity(identity) + if err != nil { + t.Fatal(err) + } + cfg := Config{Repo: filepath.Join(dir, "repo"), Identity: identity, Recipients: []string{recipient}} + if _, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}); err == nil { + t.Fatal("symlink-swapped source should fail") + } + if _, err := os.Stat(filepath.Join(cfg.Repo, "manifest.json")); !os.IsNotExist(err) { + t.Fatalf("failed backup wrote a manifest: %v", err) + } +} From 2deaaac48320518457a9ed446d72722f6fec023a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:41:07 -0400 Subject: [PATCH 07/16] fix: reserve encrypted file index namespace --- backup/backup.go | 5 +++++ backup/files_test.go | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index c8f94c9..6dbfe18 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -69,6 +69,11 @@ func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, fil if err := ctx.Err(); err != nil { return Manifest{}, err } + for _, shard := range shards { + if shard.Table == fileIndexTable || path.Clean(strings.TrimSpace(shard.Path)) == fileIndexPath { + return Manifest{}, fmt.Errorf("backup shard uses reserved file index namespace: %s", shard.Path) + } + } recipients := normalizedStrings(cfg.Recipients) reuseEncrypted := sameStrings(old.Recipients, recipients) manifest := Manifest{ diff --git a/backup/files_test.go b/backup/files_test.go index c10a12f..606888e 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "strings" "testing" ) @@ -144,7 +145,11 @@ func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { if err := os.MkdirAll(root, 0o700); err != nil { t.Fatal(err) } - for _, name := range []string{"name", "name "} { + names := []string{"name"} + if runtime.GOOS != "windows" { + names = append(names, "name ") + } + for _, name := range names { if err := os.WriteFile(filepath.Join(root, name), []byte(name), 0o600); err != nil { t.Fatal(err) } @@ -153,7 +158,7 @@ func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { if err != nil { t.Fatal(err) } - if len(files) != 2 || files[0].Path != "media/name" || files[1].Path != "media/name " { + if len(files) != len(names) || files[0].Path != "media/name" || (runtime.GOOS != "windows" && files[1].Path != "media/name ") { t.Fatalf("whitespace paths changed: %#v", files) } @@ -180,3 +185,15 @@ func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { t.Fatalf("failed backup wrote a manifest: %v", err) } } + +func TestWriteSnapshotRejectsReservedFileIndexNamespace(t *testing.T) { + ctx := context.Background() + for _, shard := range []Shard{ + {Table: fileIndexTable, Path: "data/custom.jsonl.gz.age", Rows: []row{}}, + {Table: "custom", Path: fileIndexPath, Rows: []row{}}, + } { + if _, err := WriteSnapshotWithFiles(ctx, Config{}, []Shard{shard}, nil, Manifest{}); err == nil { + t.Fatalf("reserved shard should fail: %#v", shard) + } + } +} From 595d3f352e2769de5ff944d136d932204804c66c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:42:58 -0400 Subject: [PATCH 08/16] feat: constrain encrypted file restores --- backup/files.go | 18 ++++++++++++++++-- backup/files_test.go | 7 +++++++ backup/history.go | 6 +++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/backup/files.go b/backup/files.go index 285d135..eb5b6bd 100644 --- a/backup/files.go +++ b/backup/files.go @@ -201,7 +201,11 @@ func reusableFileEntry(cfg Config, oldByHash, written map[string]FileEntry, hash } func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot string) (int, error) { - return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(rel string) (io.ReadCloser, error) { + return RestoreFilesUnder(ctx, cfg, manifest, targetRoot, "") +} + +func RestoreFilesUnder(ctx context.Context, cfg Config, manifest Manifest, targetRoot, requiredPrefix string) (int, error) { + return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, requiredPrefix, func(rel string) (io.ReadCloser, error) { shard, err := ResolveShardPath(cfg.Repo, rel) if err != nil { return nil, err @@ -210,7 +214,7 @@ func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot }) } -func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifest, targetRoot string, load encryptedFileLoader) (int, error) { +func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifest, targetRoot, requiredPrefix string, load encryptedFileLoader) (int, error) { if manifest.Format != FormatVersion { return 0, fmt.Errorf("unsupported backup format %d", manifest.Format) } @@ -227,6 +231,13 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes return 0, err } seen := make(map[string]struct{}, len(manifest.Files)) + prefix := "" + if requiredPrefix != "" { + prefix, err = cleanFilePath(requiredPrefix) + if err != nil { + return 0, err + } + } for index, entry := range manifest.Files { if err := ctx.Err(); err != nil { return 0, err @@ -236,6 +247,9 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes if err != nil { return 0, err } + if prefix != "" && logical != prefix && !strings.HasPrefix(logical, prefix+"/") { + return 0, fmt.Errorf("backup file path is outside required prefix %s: %s", prefix, logical) + } if _, exists := seen[logical]; exists { return 0, fmt.Errorf("duplicate backup file path: %s", logical) } diff --git a/backup/files_test.go b/backup/files_test.go index 606888e..3e4347c 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -88,6 +88,13 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { t.Fatalf("restored %s = %q", name, body) } } + confinedRoot := filepath.Join(dir, "confined") + if err := os.MkdirAll(confinedRoot, 0o700); err != nil { + t.Fatal(err) + } + if _, err := RestoreFilesUnder(ctx, cfg, manifest, confinedRoot, "attachments"); err == nil { + t.Fatal("restore outside required prefix should fail") + } } func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { diff --git a/backup/history.go b/backup/history.go index ca450c9..7607456 100644 --- a/backup/history.go +++ b/backup/history.go @@ -88,11 +88,15 @@ func ReadSnapshotAt(ctx context.Context, cfg Config, opts mirror.Options, manife } func RestoreFilesAt(ctx context.Context, cfg Config, opts mirror.Options, manifest Manifest, ref, targetRoot string) (int, string, error) { + return RestoreFilesAtUnder(ctx, cfg, opts, manifest, ref, targetRoot, "") +} + +func RestoreFilesAtUnder(ctx context.Context, cfg Config, opts mirror.Options, manifest Manifest, ref, targetRoot, requiredPrefix string) (int, string, error) { commit, err := mirror.ResolveCommit(ctx, opts, ref) if err != nil { return 0, "", err } - count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, func(rel string) (io.ReadCloser, error) { + count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, requiredPrefix, func(rel string) (io.ReadCloser, error) { if _, err := ResolveShardPath(cfg.Repo, rel); err != nil { return nil, err } From 18866d7d75cdd96f3dc14f34dc98399f24b1c9c4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:46:13 -0400 Subject: [PATCH 09/16] fix: preserve bundled files during legacy cleanup --- backup/backup.go | 14 +++++++++++--- backup/files_test.go | 6 ++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 6dbfe18..000a6c1 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -137,7 +137,7 @@ func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, fil if err := ctx.Err(); err != nil { return Manifest{}, err } - if err := removeStaleBackupFiles(ctx, cfg.Repo, manifest.Shards, manifest.Files); err != nil { + if err := removeStaleBackupFiles(ctx, cfg.Repo, manifest.Shards, manifest.Files, true); err != nil { return Manifest{}, err } return manifest, nil @@ -435,11 +435,15 @@ func RemoveStaleShards(repo string, shards []ShardEntry) error { return removeStaleShards(context.Background(), repo, shards) } +func RemoveStaleManifestFiles(repo string, manifest Manifest) error { + return removeStaleBackupFiles(context.Background(), repo, manifest.Shards, manifest.Files, true) +} + func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) error { - return removeStaleBackupFiles(ctx, repo, shards, nil) + return removeStaleBackupFiles(ctx, repo, shards, nil, false) } -func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntry, files []FileEntry) error { +func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntry, files []FileEntry, manageFiles bool) error { if err := ctx.Err(); err != nil { return err } @@ -451,6 +455,7 @@ func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntr keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(file.Shard)))] = struct{}{} } root := filepath.Join(repo, "data") + filesRoot := filepath.Join(root, "files") if _, err := os.Stat(root); os.IsNotExist(err) { return nil } @@ -466,6 +471,9 @@ func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntr return nil } clean := filepath.Clean(path) + if !manageFiles && (clean == filesRoot || strings.HasPrefix(clean, filesRoot+string(filepath.Separator))) { + return nil + } if _, ok := keep[clean]; ok { return nil } diff --git a/backup/files_test.go b/backup/files_test.go index 3e4347c..3ad7bd8 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -61,6 +61,12 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { if string(ciphertext) == "same private media" { t.Fatal("backup file was stored as plaintext") } + if err := RemoveStaleShards(cfg.Repo, manifest.Shards); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(cfg.Repo, filepath.FromSlash(manifest.Files[0].Shard))); err != nil { + t.Fatalf("legacy stale cleanup removed file bundle object: %v", err) + } second, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, manifest) if err != nil { t.Fatal(err) From f9c1c822ef65afaadfbbc759623c73731128ce56 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:49:09 -0400 Subject: [PATCH 10/16] refactor: keep cleanup API minimal --- backup/backup.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 000a6c1..d5893ea 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -435,10 +435,6 @@ func RemoveStaleShards(repo string, shards []ShardEntry) error { return removeStaleShards(context.Background(), repo, shards) } -func RemoveStaleManifestFiles(repo string, manifest Manifest) error { - return removeStaleBackupFiles(context.Background(), repo, manifest.Shards, manifest.Files, true) -} - func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) error { return removeStaleBackupFiles(ctx, repo, shards, nil, false) } From adafd8d3f56d26b99ef26a41232940c60679089e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:52:59 -0400 Subject: [PATCH 11/16] fix: reserve encrypted file object namespace --- backup/backup.go | 3 ++- backup/files_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backup/backup.go b/backup/backup.go index d5893ea..1c41483 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -70,7 +70,8 @@ func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, fil return Manifest{}, err } for _, shard := range shards { - if shard.Table == fileIndexTable || path.Clean(strings.TrimSpace(shard.Path)) == fileIndexPath { + cleanPath := path.Clean(strings.TrimSpace(shard.Path)) + if shard.Table == fileIndexTable || cleanPath == "data/files" || strings.HasPrefix(cleanPath, "data/files/") { return Manifest{}, fmt.Errorf("backup shard uses reserved file index namespace: %s", shard.Path) } } diff --git a/backup/files_test.go b/backup/files_test.go index 3ad7bd8..a1a58de 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -204,6 +204,7 @@ func TestWriteSnapshotRejectsReservedFileIndexNamespace(t *testing.T) { for _, shard := range []Shard{ {Table: fileIndexTable, Path: "data/custom.jsonl.gz.age", Rows: []row{}}, {Table: "custom", Path: fileIndexPath, Rows: []row{}}, + {Table: "custom", Path: "data/files/aa/custom.gz.age", Rows: []row{}}, } { if _, err := WriteSnapshotWithFiles(ctx, Config{}, []Shard{shard}, nil, Manifest{}); err == nil { t.Fatalf("reserved shard should fail: %#v", shard) From 441777525b33de46428c4a6b13ceb833bfa69663 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 13:56:03 -0400 Subject: [PATCH 12/16] fix: hide plaintext media fingerprints --- backup/backup.go | 6 +-- backup/files.go | 114 ++++++++++++++++++++++++++++++++++--------- backup/files_test.go | 3 ++ 3 files changed, 95 insertions(+), 28 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 1c41483..27cbbfe 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -50,10 +50,8 @@ type ShardEntry struct { } type FileEntry struct { - Shard string `json:"shard"` - SHA256 string `json:"sha256"` - Size int64 `json:"size"` - Bytes int64 `json:"bytes"` + Shard string `json:"shard"` + Bytes int64 `json:"bytes"` } type DecodedShard struct { diff --git a/backup/files.go b/backup/files.go index eb5b6bd..fba2da8 100644 --- a/backup/files.go +++ b/backup/files.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "fmt" @@ -30,7 +31,14 @@ const ( ) type fileIndexRecord struct { - Path string `json:"path"` + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +type indexedFile struct { + Entry FileEntry + Record fileIndexRecord } type encryptedFileLoader func(string) (io.ReadCloser, error) @@ -100,10 +108,15 @@ func CollectFiles(ctx context.Context, root, prefix string) ([]File, error) { func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reuseEncrypted bool) ([]FileEntry, []fileIndexRecord, error) { ordered := append([]File(nil), files...) sort.Slice(ordered, func(i, j int) bool { return ordered[i].Path < ordered[j].Path }) - oldByHash := make(map[string]FileEntry, len(old.Files)) + oldByHash := make(map[string]indexedFile, len(old.Files)) if reuseEncrypted { - for _, entry := range old.Files { - oldByHash[entry.SHA256] = entry + records, err := loadLocalFileIndex(cfg, old) + if err != nil { + return nil, nil, err + } + for index, entry := range old.Files { + record := records[index] + oldByHash[record.SHA256] = indexedFile{Entry: entry, Record: record} } } written := make(map[string]FileEntry, len(files)) @@ -122,7 +135,6 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu return nil, nil, fmt.Errorf("duplicate backup file path: %s", logical) } seenPaths[logical] = struct{}{} - index = append(index, fileIndexRecord{Path: logical}) hashValue, size, sourceInfo, err := hashFile(ctx, file) if err != nil { return nil, nil, err @@ -131,6 +143,7 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu return nil, nil, err } else if ok { out = append(out, entry) + index = append(index, fileIndexRecord{Path: logical, SHA256: hashValue, Size: size}) continue } tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file, sourceInfo, cfg.Repo, cfg.Recipients) @@ -151,9 +164,13 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu } keepTemp = false out = append(out, entry) + index = append(index, fileIndexRecord{Path: logical, SHA256: hashValue, Size: size}) continue } - shard := path.Join("data/files", hashValue[:2], hashValue+".gz.age") + shard, err := randomFileShard(cfg.Repo) + if err != nil { + return nil, nil, err + } target, err := ResolveShardPath(cfg.Repo, shard) if err != nil { return nil, nil, err @@ -168,23 +185,23 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu if err := syncDir(filepath.Dir(target)); err != nil { return nil, nil, err } - entry := FileEntry{Shard: shard, SHA256: hashValue, Size: size, Bytes: encryptedSize} + entry := FileEntry{Shard: shard, Bytes: encryptedSize} written[hashValue] = entry out = append(out, entry) + index = append(index, fileIndexRecord{Path: logical, SHA256: hashValue, Size: size}) } return out, index, nil } -func reusableFileEntry(cfg Config, oldByHash, written map[string]FileEntry, hashValue string, size int64) (FileEntry, bool, error) { +func reusableFileEntry(cfg Config, oldByHash map[string]indexedFile, written map[string]FileEntry, hashValue string, size int64) (FileEntry, bool, error) { if entry, ok := written[hashValue]; ok { return entry, true, nil } - shard := path.Join("data/files", hashValue[:2], hashValue+".gz.age") - oldEntry, ok := oldByHash[hashValue] - if !ok || oldEntry.Shard != shard || oldEntry.Size != size { + oldFile, ok := oldByHash[hashValue] + if !ok || oldFile.Record.Size != size { return FileEntry{}, false, nil } - target, err := ResolveShardPath(cfg.Repo, shard) + target, err := ResolveShardPath(cfg.Repo, oldFile.Entry.Shard) if err != nil { return FileEntry{}, false, err } @@ -195,11 +212,52 @@ func reusableFileEntry(cfg Config, oldByHash, written map[string]FileEntry, hash } return FileEntry{}, false, err } - entry := FileEntry{Shard: shard, SHA256: hashValue, Size: size, Bytes: info.Size()} + entry := FileEntry{Shard: oldFile.Entry.Shard, Bytes: info.Size()} written[hashValue] = entry return entry, true, nil } +func loadLocalFileIndex(cfg Config, manifest Manifest) ([]fileIndexRecord, error) { + if len(manifest.Files) == 0 { + return nil, nil + } + identityData, err := os.ReadFile(expandHome(cfg.Identity)) // #nosec G304 -- path is configured by the caller. + if err != nil { + return nil, err + } + identity, err := parseIdentity(identityData) + if err != nil { + return nil, err + } + return readFileIndex(identity, manifest, func(rel string) (io.ReadCloser, error) { + shard, err := ResolveShardPath(cfg.Repo, rel) + if err != nil { + return nil, err + } + return os.Open(shard) // #nosec G304 -- ResolveShardPath confines manifest-controlled paths below data/. + }) +} + +func randomFileShard(repo string) (string, error) { + for range 10 { + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return "", err + } + rel := path.Join("data/files/objects", hex.EncodeToString(id[:])+".gz.age") + target, err := ResolveShardPath(repo, rel) + if err != nil { + return "", err + } + if _, err := os.Stat(target); os.IsNotExist(err) { + return rel, nil + } else if err != nil { + return "", err + } + } + return "", fmt.Errorf("generate unique backup file object path") +} + func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot string) (int, error) { return RestoreFilesUnder(ctx, cfg, manifest, targetRoot, "") } @@ -226,7 +284,7 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes if err != nil { return 0, err } - paths, err := readFileIndex(identity, manifest, load) + records, err := readFileIndex(identity, manifest, load) if err != nil { return 0, err } @@ -242,7 +300,8 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes if err := ctx.Err(); err != nil { return 0, err } - logical := paths[index] + record := records[index] + logical := record.Path logical, err = cleanFilePath(logical) if err != nil { return 0, err @@ -261,7 +320,7 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes if err != nil { return 0, err } - err = restoreFile(ctx, identity, ciphertext, targetRoot, logical, entry) + err = restoreFile(ctx, identity, ciphertext, targetRoot, logical, record) closeErr := ciphertext.Close() if err != nil { return 0, err @@ -273,9 +332,9 @@ func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifes return len(manifest.Files), nil } -func readFileIndex(identity *age.X25519Identity, manifest Manifest, load encryptedFileLoader) ([]string, error) { +func readFileIndex(identity *age.X25519Identity, manifest Manifest, load encryptedFileLoader) ([]fileIndexRecord, error) { if len(manifest.Files) == 0 { - return []string{}, nil + return []fileIndexRecord{}, nil } var indexEntry *ShardEntry for index := range manifest.Shards { @@ -312,9 +371,9 @@ func readFileIndex(identity *age.X25519Identity, manifest Manifest, load encrypt if len(records) != len(manifest.Files) { return nil, fmt.Errorf("backup file index count mismatch") } - paths := make([]string, 0, len(records)) seen := make(map[string]struct{}, len(records)) - for _, record := range records { + for index := range records { + record := &records[index] logical, err := cleanFilePath(record.Path) if err != nil { return nil, err @@ -323,9 +382,16 @@ func readFileIndex(identity *age.X25519Identity, manifest Manifest, load encrypt return nil, fmt.Errorf("duplicate backup file index path: %s", logical) } seen[logical] = struct{}{} - paths = append(paths, logical) + record.Path = logical + if record.Size < 0 { + return nil, fmt.Errorf("invalid backup file size for %s", logical) + } + hashBytes, err := hex.DecodeString(record.SHA256) + if err != nil || len(hashBytes) != sha256.Size { + return nil, fmt.Errorf("invalid backup file hash for %s", logical) + } } - return paths, nil + return records, nil } func decryptFilePayload(identity *age.X25519Identity, ciphertext io.Reader) ([]byte, error) { @@ -341,7 +407,7 @@ func decryptFilePayload(identity *age.X25519Identity, ciphertext io.Reader) ([]b return io.ReadAll(gz) } -func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext io.Reader, targetRoot, logical string, entry FileEntry) error { +func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext io.Reader, targetRoot, logical string, record fileIndexRecord) error { target, err := safeRestoreTarget(targetRoot, logical) if err != nil { return err @@ -375,7 +441,7 @@ func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext i _ = tmp.Close() return err } - if size != entry.Size || hex.EncodeToString(hasher.Sum(nil)) != entry.SHA256 { + if size != record.Size || hex.EncodeToString(hasher.Sum(nil)) != record.SHA256 { _ = tmp.Close() return fmt.Errorf("backup file verification failed for %s", logical) } diff --git a/backup/files_test.go b/backup/files_test.go index a1a58de..85bad0a 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -54,6 +54,9 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { if strings.Contains(string(manifestBody), "photo.jpg") || strings.Contains(string(manifestBody), "copy.jpg") { t.Fatalf("manifest exposes logical media paths: %s", manifestBody) } + if strings.Contains(string(manifestBody), SHA256Hex([]byte("same private media"))) { + t.Fatalf("manifest exposes plaintext media hash: %s", manifestBody) + } ciphertext, err := os.ReadFile(filepath.Join(cfg.Repo, filepath.FromSlash(manifest.Files[0].Shard))) if err != nil { t.Fatal(err) From d9c6440db799d6a13990460a8f24a3b07953d8df Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 14:00:03 -0400 Subject: [PATCH 13/16] fix: make encrypted file reuse optional --- backup/files.go | 13 ++++++------- backup/files_test.go | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/backup/files.go b/backup/files.go index fba2da8..6b27bab 100644 --- a/backup/files.go +++ b/backup/files.go @@ -109,14 +109,13 @@ func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reu ordered := append([]File(nil), files...) sort.Slice(ordered, func(i, j int) bool { return ordered[i].Path < ordered[j].Path }) oldByHash := make(map[string]indexedFile, len(old.Files)) - if reuseEncrypted { + if reuseEncrypted && len(files) > 0 { records, err := loadLocalFileIndex(cfg, old) - if err != nil { - return nil, nil, err - } - for index, entry := range old.Files { - record := records[index] - oldByHash[record.SHA256] = indexedFile{Entry: entry, Record: record} + if err == nil { + for index, entry := range old.Files { + record := records[index] + oldByHash[record.SHA256] = indexedFile{Entry: entry, Record: record} + } } } written := make(map[string]FileEntry, len(files)) diff --git a/backup/files_test.go b/backup/files_test.go index 85bad0a..52dc445 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -77,6 +77,26 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { if !EquivalentManifest(manifest, second) || !second.Exported.Equal(manifest.Exported) { t.Fatalf("unchanged files rewrote manifest: %#v", second) } + publicOnly := cfg + publicOnly.Identity = "" + publicOnly.Repo = filepath.Join(dir, "public-only") + if err := os.MkdirAll(publicOnly.Repo, 0o700); err != nil { + t.Fatal(err) + } + withoutReuse, err := WriteSnapshotWithFiles(ctx, publicOnly, nil, files, manifest) + if err != nil { + t.Fatalf("public-recipient-only writer should re-encrypt files: %v", err) + } + if len(withoutReuse.Files) != len(files) { + t.Fatalf("public-only file manifest = %#v", withoutReuse.Files) + } + withoutFiles, err := WriteSnapshotWithFiles(ctx, publicOnly, nil, nil, withoutReuse) + if err != nil { + t.Fatalf("removing files should not require an identity: %v", err) + } + if len(withoutFiles.Files) != 0 { + t.Fatalf("removed file manifest = %#v", withoutFiles.Files) + } restoreRoot := filepath.Join(dir, "restore") if err := os.MkdirAll(restoreRoot, 0o700); err != nil { t.Fatal(err) From dd65c31001195d0e4ce5771611c85a1f3ef32e1c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 14:05:14 -0400 Subject: [PATCH 14/16] test: cover repeated file restore --- backup/files.go | 2 ++ backup/files_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/backup/files.go b/backup/files.go index 6b27bab..bc59354 100644 --- a/backup/files.go +++ b/backup/files.go @@ -451,6 +451,8 @@ func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext i if err := tmp.Close(); err != nil { return err } + // os.Rename replaces regular files, including via MoveFileEx with + // MOVEFILE_REPLACE_EXISTING on Windows, without first hiding the old file. if err := os.Rename(tmpPath, target); err != nil { return err } diff --git a/backup/files_test.go b/backup/files_test.go index 52dc445..764111a 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -117,6 +117,16 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { t.Fatalf("restored %s = %q", name, body) } } + photo := filepath.Join(restoreRoot, "media", "photo.jpg") + if err := os.WriteFile(photo, []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := RestoreFiles(ctx, cfg, manifest, restoreRoot); err != nil { + t.Fatalf("repeat restore: %v", err) + } + if body, err := os.ReadFile(photo); err != nil || string(body) != "same private media" { + t.Fatalf("repeat restored photo = %q err=%v", body, err) + } confinedRoot := filepath.Join(dir, "confined") if err := os.MkdirAll(confinedRoot, 0o700); err != nil { t.Fatal(err) From 5880b7fd6d0eb3c52ea8519da3f687218c6a35c7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 14:08:17 -0400 Subject: [PATCH 15/16] test: tolerate unavailable Windows symlinks --- backup/files_test.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/backup/files_test.go b/backup/files_test.go index 764111a..e872396 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -21,7 +21,7 @@ func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { t.Fatal(err) } } - if err := os.Symlink(filepath.Join(source, "photo.jpg"), filepath.Join(source, "linked.jpg")); err != nil { + if err := os.Symlink(filepath.Join(source, "photo.jpg"), filepath.Join(source, "linked.jpg")); err != nil && runtime.GOOS != "windows" { t.Fatal(err) } files, err := CollectFiles(ctx, source, "media") @@ -176,9 +176,7 @@ func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { if err := os.MkdirAll(restoreRoot, 0o700); err != nil { t.Fatal(err) } - if err := os.Symlink(outside, filepath.Join(restoreRoot, "media")); err != nil { - t.Fatal(err) - } + symlinkOrSkip(t, outside, filepath.Join(restoreRoot, "media")) if _, err := RestoreFiles(ctx, cfg, manifest, restoreRoot); err == nil { t.Fatal("symlinked restore directory should fail") } @@ -215,9 +213,7 @@ func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { if err := os.Remove(files[0].Source); err != nil { t.Fatal(err) } - if err := os.Symlink(outside, files[0].Source); err != nil { - t.Fatal(err) - } + symlinkOrSkip(t, outside, files[0].Source) identity := filepath.Join(dir, "age.key") recipient, err := EnsureIdentity(identity) if err != nil { @@ -232,6 +228,16 @@ func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { } } +func symlinkOrSkip(t *testing.T, oldname, newname string) { + t.Helper() + if err := os.Symlink(oldname, newname); err != nil { + if runtime.GOOS == "windows" { + t.Skipf("symlink unavailable: %v", err) + } + t.Fatal(err) + } +} + func TestWriteSnapshotRejectsReservedFileIndexNamespace(t *testing.T) { ctx := context.Background() for _, shard := range []Shard{ From d82a7559ef7caa600d2093b26a586d4f706629b1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 14:12:04 -0400 Subject: [PATCH 16/16] fix: reject noncanonical shard paths --- backup/backup.go | 28 +++++++++++++++++++++------- backup/backup_test.go | 2 +- backup/files_test.go | 2 ++ backup/history.go | 9 ++++----- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 27cbbfe..2983911 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -68,7 +68,10 @@ func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, fil return Manifest{}, err } for _, shard := range shards { - cleanPath := path.Clean(strings.TrimSpace(shard.Path)) + cleanPath, err := cleanShardPath(shard.Path) + if err != nil { + return Manifest{}, err + } if shard.Table == fileIndexTable || cleanPath == "data/files" || strings.HasPrefix(cleanPath, "data/files/") { return Manifest{}, fmt.Errorf("backup shard uses reserved file index namespace: %s", shard.Path) } @@ -249,12 +252,9 @@ func DecryptShardFile(cfg Config, shard ShardEntry) ([]byte, error) { } func ResolveShardPath(repo, rel string) (string, error) { - clean := path.Clean(strings.TrimSpace(rel)) - if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) { - return "", fmt.Errorf("backup shard path escapes backup root: %s", rel) - } - if !strings.HasPrefix(clean, "data/") || !strings.HasSuffix(clean, ".age") { - return "", fmt.Errorf("invalid backup shard path: %s", rel) + clean, err := cleanShardPath(rel) + if err != nil { + return "", err } full := filepath.Join(repo, filepath.FromSlash(clean)) root := filepath.Clean(filepath.Join(repo, "data")) @@ -265,6 +265,20 @@ func ResolveShardPath(repo, rel string) (string, error) { return full, nil } +func cleanShardPath(rel string) (string, error) { + if strings.ContainsRune(rel, '\\') { + return "", fmt.Errorf("invalid backup shard path: %s", rel) + } + clean := path.Clean(strings.TrimSpace(rel)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) { + return "", fmt.Errorf("backup shard path escapes backup root: %s", rel) + } + if !strings.HasPrefix(clean, "data/") || !strings.HasSuffix(clean, ".age") { + return "", fmt.Errorf("invalid backup shard path: %s", rel) + } + return clean, nil +} + func EncodeJSONL(rows any) ([]byte, int, error) { return encodeJSONL(context.Background(), rows) } diff --git a/backup/backup_test.go b/backup/backup_test.go index 65c8752..64a35ea 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -211,7 +211,7 @@ func TestPublicBackupHelpers(t *testing.T) { } func TestResolveShardPathRejectsEscapes(t *testing.T) { - for _, rel := range []string{"../x.age", "data/../x.age", "data/x.txt", "/data/x.age"} { + for _, rel := range []string{"../x.age", "data/../x.age", "data/x.txt", "/data/x.age", `data\files\index.jsonl.gz.age`, `data/files\index.jsonl.gz.age`} { if _, err := ResolveShardPath(t.TempDir(), rel); err == nil { t.Fatalf("expected error for %q", rel) } diff --git a/backup/files_test.go b/backup/files_test.go index e872396..0e2bcb5 100644 --- a/backup/files_test.go +++ b/backup/files_test.go @@ -244,6 +244,8 @@ func TestWriteSnapshotRejectsReservedFileIndexNamespace(t *testing.T) { {Table: fileIndexTable, Path: "data/custom.jsonl.gz.age", Rows: []row{}}, {Table: "custom", Path: fileIndexPath, Rows: []row{}}, {Table: "custom", Path: "data/files/aa/custom.gz.age", Rows: []row{}}, + {Table: "custom", Path: `data\files\index.jsonl.gz.age`, Rows: []row{}}, + {Table: "custom", Path: `data/files\index.jsonl.gz.age`, Rows: []row{}}, } { if _, err := WriteSnapshotWithFiles(ctx, Config{}, []Shard{shard}, nil, Manifest{}); err == nil { t.Fatalf("reserved shard should fail: %#v", shard) diff --git a/backup/history.go b/backup/history.go index 7607456..9f435b9 100644 --- a/backup/history.go +++ b/backup/history.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "path" "strings" "github.com/openclaw/crawlkit/mirror" @@ -68,10 +67,10 @@ func ReadSnapshotAt(ctx context.Context, cfg Config, opts mirror.Options, manife return nil, "", err } shards, err := readSnapshotWith(manifest, func(shard ShardEntry) ([]byte, error) { - if _, err := ResolveShardPath(cfg.Repo, shard.Path); err != nil { + clean, err := cleanShardPath(shard.Path) + if err != nil { return nil, err } - clean := path.Clean(strings.TrimSpace(shard.Path)) ciphertext, resolved, err := mirror.ReadFileAt(ctx, opts, commit, clean) if err != nil { return nil, err @@ -97,10 +96,10 @@ func RestoreFilesAtUnder(ctx context.Context, cfg Config, opts mirror.Options, m return 0, "", err } count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, requiredPrefix, func(rel string) (io.ReadCloser, error) { - if _, err := ResolveShardPath(cfg.Repo, rel); err != nil { + clean, err := cleanShardPath(rel) + if err != nil { return nil, err } - clean := path.Clean(strings.TrimSpace(rel)) ciphertext, resolved, err := mirror.ReadFileAt(ctx, opts, commit, clean) if err != nil { return nil, err