[codex] Add encrypted backup file bundles

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add content-deduplicated encrypted file bundles with safe current and historical restore for crawler backups.
 - Add shared SQL LIKE literal escaping for crawler search queries.
 - Preserve Unicode combining marks in tokenized FTS5 queries.
 - Add a safe tokenized FTS5 query helper for arbitrary user search input.

backup/backup.go

@@ -30,6 +30,7 @@ type Manifest struct {
 	Recipients []string       `json:"recipients,omitempty"`
 	Counts     map[string]int `json:"counts"`
 	Shards     []ShardEntry   `json:"shards"`
+	Files      []FileEntry    `json:"files,omitempty"`
 }
 
 type Shard struct {
@@ -47,12 +48,24 @@ type ShardEntry struct {
 	Bytes  int64  `json:"bytes"`
 }
 
+type FileEntry struct {
+	Path   string `json:"path"`
+	Shard  string `json:"shard"`
+	SHA256 string `json:"sha256"`
+	Size   int64  `json:"size"`
+	Bytes  int64  `json:"bytes"`
+}
+
 type DecodedShard struct {
 	Entry     ShardEntry
 	Plaintext []byte
 }
 
 func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest) (Manifest, error) {
+	return WriteSnapshotWithFiles(ctx, cfg, shards, nil, old)
+}
+
+func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, files []File, old Manifest) (Manifest, error) {
 	if err := ctx.Err(); err != nil {
 		return Manifest{}, err
 	}
@@ -90,6 +103,11 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest
 		manifest.Counts[countKey] += rows
 		manifest.Shards = append(manifest.Shards, entry)
 	}
+	filesManifest, err := writeFiles(ctx, cfg, old, files, reuseEncrypted)
+	if err != nil {
+		return Manifest{}, err
+	}
+	manifest.Files = filesManifest
 	sort.Slice(manifest.Shards, func(i, j int) bool { return manifest.Shards[i].Path < manifest.Shards[j].Path })
 	if EquivalentManifest(old, manifest) {
 		return old, nil
@@ -103,7 +121,7 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest
 	if err := ctx.Err(); err != nil {
 		return Manifest{}, err
 	}
-	if err := removeStaleShards(ctx, cfg.Repo, manifest.Shards); err != nil {
+	if err := removeStaleBackupFiles(ctx, cfg.Repo, manifest.Shards, manifest.Files); err != nil {
 		return Manifest{}, err
 	}
 	return manifest, nil
@@ -371,7 +389,7 @@ func (m Manifest) Entry(path string) (ShardEntry, bool) {
 }
 
 func EquivalentManifest(a, b Manifest) bool {
-	if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) {
+	if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) || len(a.Files) != len(b.Files) {
 		return false
 	}
 	for i := range a.Shards {
@@ -381,6 +399,13 @@ func EquivalentManifest(a, b Manifest) bool {
 			return false
 		}
 	}
+	for i := range a.Files {
+		left, right := a.Files[i], b.Files[i]
+		left.Bytes, right.Bytes = 0, 0
+		if left != right {
+			return false
+		}
+	}
 	return true
 }
 
@@ -389,13 +414,20 @@ func RemoveStaleShards(repo string, shards []ShardEntry) error {
 }
 
 func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) error {
+	return removeStaleBackupFiles(ctx, repo, shards, nil)
+}
+
+func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntry, files []FileEntry) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
 	keep := map[string]struct{}{}
 	for _, shard := range shards {
 		keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(shard.Path)))] = struct{}{}
 	}
+	for _, file := range files {
+		keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(file.Shard)))] = struct{}{}
+	}
 	root := filepath.Join(repo, "data")
 	if _, err := os.Stat(root); os.IsNotExist(err) {
 		return nil

backup/backup_test.go

@@ -231,9 +231,13 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) {
 	if err := mirror.EnsureRepo(ctx, mirrorOpts); err != nil {
 		t.Fatal(err)
 	}
-	firstManifest, err := WriteSnapshot(ctx, cfg, []Shard{
+	mediaSource := filepath.Join(dir, "media.txt")
+	if err := os.WriteFile(mediaSource, []byte("first media"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	firstManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{
 		{Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "first"}}},
-	}, Manifest{})
+	}, []File{{Path: "media/file.txt", Source: mediaSource}}, Manifest{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -247,9 +251,12 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) {
 	if _, err := mirror.CreateImmutableTag(ctx, mirrorOpts, "snapshot/one"); err != nil {
 		t.Fatal(err)
 	}
-	secondManifest, err := WriteSnapshot(ctx, cfg, []Shard{
+	if err := os.WriteFile(mediaSource, []byte("second media"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	secondManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{
 		{Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "second"}}},
-	}, firstManifest)
+	}, []File{{Path: "media/file.txt", Source: mediaSource}}, firstManifest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -289,6 +296,24 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) {
 	if len(historicalRows) != 1 || historicalRows[0].Body != "first" {
 		t.Fatalf("historical rows = %#v", historicalRows)
 	}
+	restoreRoot := filepath.Join(dir, "historical-restore")
+	if err := os.MkdirAll(restoreRoot, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	restored, restoredRef, err := RestoreFilesAt(ctx, cfg, mirrorOpts, manifest, "snapshot/one", restoreRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if restored != 1 || restoredRef != first {
+		t.Fatalf("historical files = %d at %s", restored, restoredRef)
+	}
+	mediaBody, err := os.ReadFile(filepath.Join(restoreRoot, "media", "file.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(mediaBody) != "first media" {
+		t.Fatalf("historical media = %q", mediaBody)
+	}
 	if secondManifest.Counts["messages"] != 1 {
 		t.Fatalf("second manifest = %#v", secondManifest)
 	}