[codex] Add encrypted backup file bundles

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add content-deduplicated encrypted file bundles with safe current and historical restore for crawler backups.
 - Add shared SQL LIKE literal escaping for crawler search queries.
 - Preserve Unicode combining marks in tokenized FTS5 queries.
 - Add a safe tokenized FTS5 query helper for arbitrary user search input.

backup/backup.go

@@ -10,6 +10,7 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -30,6 +31,7 @@ type Manifest struct {
 	Recipients []string       `json:"recipients,omitempty"`
 	Counts     map[string]int `json:"counts"`
 	Shards     []ShardEntry   `json:"shards"`
+	Files      []FileEntry    `json:"files,omitempty"`
 }
 
 type Shard struct {
@@ -47,15 +49,31 @@ type ShardEntry struct {
 	Bytes  int64  `json:"bytes"`
 }
 
+type FileEntry struct {
+	Shard  string `json:"shard"`
+	SHA256 string `json:"sha256"`
+	Size   int64  `json:"size"`
+	Bytes  int64  `json:"bytes"`
+}
+
 type DecodedShard struct {
 	Entry     ShardEntry
 	Plaintext []byte
 }
 
 func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest) (Manifest, error) {
+	return WriteSnapshotWithFiles(ctx, cfg, shards, nil, old)
+}
+
+func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, files []File, old Manifest) (Manifest, error) {
 	if err := ctx.Err(); err != nil {
 		return Manifest{}, err
 	}
+	for _, shard := range shards {
+		if shard.Table == fileIndexTable || path.Clean(strings.TrimSpace(shard.Path)) == fileIndexPath {
+			return Manifest{}, fmt.Errorf("backup shard uses reserved file index namespace: %s", shard.Path)
+		}
+	}
 	recipients := normalizedStrings(cfg.Recipients)
 	reuseEncrypted := sameStrings(old.Recipients, recipients)
 	manifest := Manifest{
@@ -90,6 +108,22 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest
 		manifest.Counts[countKey] += rows
 		manifest.Shards = append(manifest.Shards, entry)
 	}
+	filesManifest, fileIndex, err := writeFiles(ctx, cfg, old, files, reuseEncrypted)
+	if err != nil {
+		return Manifest{}, err
+	}
+	manifest.Files = filesManifest
+	if len(fileIndex) > 0 {
+		plaintext, rows, err := encodeJSONL(ctx, fileIndex)
+		if err != nil {
+			return Manifest{}, fmt.Errorf("encode backup file index: %w", err)
+		}
+		entry, err := writeShard(ctx, cfg, old, fileIndexTable, fileIndexPath, plaintext, rows, reuseEncrypted)
+		if err != nil {
+			return Manifest{}, err
+		}
+		manifest.Shards = append(manifest.Shards, entry)
+	}
 	sort.Slice(manifest.Shards, func(i, j int) bool { return manifest.Shards[i].Path < manifest.Shards[j].Path })
 	if EquivalentManifest(old, manifest) {
 		return old, nil
@@ -103,7 +137,7 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest
 	if err := ctx.Err(); err != nil {
 		return Manifest{}, err
 	}
-	if err := removeStaleShards(ctx, cfg.Repo, manifest.Shards); err != nil {
+	if err := removeStaleBackupFiles(ctx, cfg.Repo, manifest.Shards, manifest.Files, true); err != nil {
 		return Manifest{}, err
 	}
 	return manifest, nil
@@ -121,6 +155,9 @@ func readSnapshotWith(manifest Manifest, load func(ShardEntry) ([]byte, error))
 	}
 	var out []DecodedShard
 	for _, shard := range manifest.Shards {
+		if shard.Table == fileIndexTable {
+			continue
+		}
 		plaintext, err := load(shard)
 		if err != nil {
 			return nil, err
@@ -353,6 +390,9 @@ func writeFileAtomicContext(ctx context.Context, target string, data []byte, per
 }
 
 func syncDir(dir string) error {
+	if runtime.GOOS == "windows" {
+		return nil
+	}
 	file, err := os.Open(dir)
 	if err != nil {
 		return err
@@ -371,7 +411,7 @@ func (m Manifest) Entry(path string) (ShardEntry, bool) {
 }
 
 func EquivalentManifest(a, b Manifest) bool {
-	if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) {
+	if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) || len(a.Files) != len(b.Files) {
 		return false
 	}
 	for i := range a.Shards {
@@ -381,6 +421,13 @@ func EquivalentManifest(a, b Manifest) bool {
 			return false
 		}
 	}
+	for i := range a.Files {
+		left, right := a.Files[i], b.Files[i]
+		left.Bytes, right.Bytes = 0, 0
+		if left != right {
+			return false
+		}
+	}
 	return true
 }
 
@@ -389,14 +436,22 @@ func RemoveStaleShards(repo string, shards []ShardEntry) error {
 }
 
 func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) error {
+	return removeStaleBackupFiles(ctx, repo, shards, nil, false)
+}
+
+func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntry, files []FileEntry, manageFiles bool) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
 	keep := map[string]struct{}{}
 	for _, shard := range shards {
 		keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(shard.Path)))] = struct{}{}
 	}
+	for _, file := range files {
+		keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(file.Shard)))] = struct{}{}
+	}
 	root := filepath.Join(repo, "data")
+	filesRoot := filepath.Join(root, "files")
 	if _, err := os.Stat(root); os.IsNotExist(err) {
 		return nil
 	}
@@ -412,6 +467,9 @@ func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) er
 			return nil
 		}
 		clean := filepath.Clean(path)
+		if !manageFiles && (clean == filesRoot || strings.HasPrefix(clean, filesRoot+string(filepath.Separator))) {
+			return nil
+		}
 		if _, ok := keep[clean]; ok {
 			return nil
 		}

backup/backup_test.go

@@ -231,9 +231,13 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) {
 	if err := mirror.EnsureRepo(ctx, mirrorOpts); err != nil {
 		t.Fatal(err)
 	}
-	firstManifest, err := WriteSnapshot(ctx, cfg, []Shard{
+	mediaSource := filepath.Join(dir, "media.txt")
+	if err := os.WriteFile(mediaSource, []byte("first media"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	firstManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{
 		{Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "first"}}},
-	}, Manifest{})
+	}, []File{{Path: "media/file.txt", Source: mediaSource}}, Manifest{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -247,9 +251,12 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) {
 	if _, err := mirror.CreateImmutableTag(ctx, mirrorOpts, "snapshot/one"); err != nil {
 		t.Fatal(err)
 	}
-	secondManifest, err := WriteSnapshot(ctx, cfg, []Shard{
+	if err := os.WriteFile(mediaSource, []byte("second media"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	secondManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{
 		{Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "second"}}},
-	}, firstManifest)
+	}, []File{{Path: "media/file.txt", Source: mediaSource}}, firstManifest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -289,6 +296,24 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) {
 	if len(historicalRows) != 1 || historicalRows[0].Body != "first" {
 		t.Fatalf("historical rows = %#v", historicalRows)
 	}
+	restoreRoot := filepath.Join(dir, "historical-restore")
+	if err := os.MkdirAll(restoreRoot, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	restored, restoredRef, err := RestoreFilesAt(ctx, cfg, mirrorOpts, manifest, "snapshot/one", restoreRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if restored != 1 || restoredRef != first {
+		t.Fatalf("historical files = %d at %s", restored, restoredRef)
+	}
+	mediaBody, err := os.ReadFile(filepath.Join(restoreRoot, "media", "file.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(mediaBody) != "first media" {
+		t.Fatalf("historical media = %q", mediaBody)
+	}
 	if secondManifest.Counts["messages"] != 1 {
 		t.Fatalf("second manifest = %#v", secondManifest)
 	}