diff --git a/CHANGELOG.md b/CHANGELOG.md index d0b0d24..3398709 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add content-deduplicated encrypted file bundles with safe current and historical restore for crawler backups. - Add shared SQL LIKE literal escaping for crawler search queries. - Preserve Unicode combining marks in tokenized FTS5 queries. - Add a safe tokenized FTS5 query helper for arbitrary user search input. diff --git a/backup/backup.go b/backup/backup.go index 91f1468..2983911 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "reflect" + "runtime" "sort" "strings" "time" @@ -30,6 +31,7 @@ type Manifest struct { Recipients []string `json:"recipients,omitempty"` Counts map[string]int `json:"counts"` Shards []ShardEntry `json:"shards"` + Files []FileEntry `json:"files,omitempty"` } type Shard struct { @@ -47,15 +49,33 @@ type ShardEntry struct { Bytes int64 `json:"bytes"` } +type FileEntry struct { + Shard string `json:"shard"` + Bytes int64 `json:"bytes"` +} + type DecodedShard struct { Entry ShardEntry Plaintext []byte } func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest) (Manifest, error) { + return WriteSnapshotWithFiles(ctx, cfg, shards, nil, old) +} + +func WriteSnapshotWithFiles(ctx context.Context, cfg Config, shards []Shard, files []File, old Manifest) (Manifest, error) { if err := ctx.Err(); err != nil { return Manifest{}, err } + for _, shard := range shards { + cleanPath, err := cleanShardPath(shard.Path) + if err != nil { + return Manifest{}, err + } + if shard.Table == fileIndexTable || cleanPath == "data/files" || strings.HasPrefix(cleanPath, "data/files/") { + return Manifest{}, fmt.Errorf("backup shard uses reserved file index namespace: %s", shard.Path) + } + } recipients := normalizedStrings(cfg.Recipients) reuseEncrypted := sameStrings(old.Recipients, recipients) manifest := Manifest{ @@ -90,6 +110,22 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest manifest.Counts[countKey] += rows manifest.Shards = append(manifest.Shards, entry) } + filesManifest, fileIndex, err := writeFiles(ctx, cfg, old, files, reuseEncrypted) + if err != nil { + return Manifest{}, err + } + manifest.Files = filesManifest + if len(fileIndex) > 0 { + plaintext, rows, err := encodeJSONL(ctx, fileIndex) + if err != nil { + return Manifest{}, fmt.Errorf("encode backup file index: %w", err) + } + entry, err := writeShard(ctx, cfg, old, fileIndexTable, fileIndexPath, plaintext, rows, reuseEncrypted) + if err != nil { + return Manifest{}, err + } + manifest.Shards = append(manifest.Shards, entry) + } sort.Slice(manifest.Shards, func(i, j int) bool { return manifest.Shards[i].Path < manifest.Shards[j].Path }) if EquivalentManifest(old, manifest) { return old, nil @@ -103,7 +139,7 @@ func WriteSnapshot(ctx context.Context, cfg Config, shards []Shard, old Manifest if err := ctx.Err(); err != nil { return Manifest{}, err } - if err := removeStaleShards(ctx, cfg.Repo, manifest.Shards); err != nil { + if err := removeStaleBackupFiles(ctx, cfg.Repo, manifest.Shards, manifest.Files, true); err != nil { return Manifest{}, err } return manifest, nil @@ -121,6 +157,9 @@ func readSnapshotWith(manifest Manifest, load func(ShardEntry) ([]byte, error)) } var out []DecodedShard for _, shard := range manifest.Shards { + if shard.Table == fileIndexTable { + continue + } plaintext, err := load(shard) if err != nil { return nil, err @@ -213,12 +252,9 @@ func DecryptShardFile(cfg Config, shard ShardEntry) ([]byte, error) { } func ResolveShardPath(repo, rel string) (string, error) { - clean := path.Clean(strings.TrimSpace(rel)) - if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) { - return "", fmt.Errorf("backup shard path escapes backup root: %s", rel) - } - if !strings.HasPrefix(clean, "data/") || !strings.HasSuffix(clean, ".age") { - return "", fmt.Errorf("invalid backup shard path: %s", rel) + clean, err := cleanShardPath(rel) + if err != nil { + return "", err } full := filepath.Join(repo, filepath.FromSlash(clean)) root := filepath.Clean(filepath.Join(repo, "data")) @@ -229,6 +265,20 @@ func ResolveShardPath(repo, rel string) (string, error) { return full, nil } +func cleanShardPath(rel string) (string, error) { + if strings.ContainsRune(rel, '\\') { + return "", fmt.Errorf("invalid backup shard path: %s", rel) + } + clean := path.Clean(strings.TrimSpace(rel)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) { + return "", fmt.Errorf("backup shard path escapes backup root: %s", rel) + } + if !strings.HasPrefix(clean, "data/") || !strings.HasSuffix(clean, ".age") { + return "", fmt.Errorf("invalid backup shard path: %s", rel) + } + return clean, nil +} + func EncodeJSONL(rows any) ([]byte, int, error) { return encodeJSONL(context.Background(), rows) } @@ -353,6 +403,9 @@ func writeFileAtomicContext(ctx context.Context, target string, data []byte, per } func syncDir(dir string) error { + if runtime.GOOS == "windows" { + return nil + } file, err := os.Open(dir) if err != nil { return err @@ -371,7 +424,7 @@ func (m Manifest) Entry(path string) (ShardEntry, bool) { } func EquivalentManifest(a, b Manifest) bool { - if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) { + if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || !sameCounts(a.Counts, b.Counts) || len(a.Shards) != len(b.Shards) || len(a.Files) != len(b.Files) { return false } for i := range a.Shards { @@ -381,6 +434,13 @@ func EquivalentManifest(a, b Manifest) bool { return false } } + for i := range a.Files { + left, right := a.Files[i], b.Files[i] + left.Bytes, right.Bytes = 0, 0 + if left != right { + return false + } + } return true } @@ -389,6 +449,10 @@ func RemoveStaleShards(repo string, shards []ShardEntry) error { } func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) error { + return removeStaleBackupFiles(ctx, repo, shards, nil, false) +} + +func removeStaleBackupFiles(ctx context.Context, repo string, shards []ShardEntry, files []FileEntry, manageFiles bool) error { if err := ctx.Err(); err != nil { return err } @@ -396,7 +460,11 @@ func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) er for _, shard := range shards { keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(shard.Path)))] = struct{}{} } + for _, file := range files { + keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(file.Shard)))] = struct{}{} + } root := filepath.Join(repo, "data") + filesRoot := filepath.Join(root, "files") if _, err := os.Stat(root); os.IsNotExist(err) { return nil } @@ -412,6 +480,9 @@ func removeStaleShards(ctx context.Context, repo string, shards []ShardEntry) er return nil } clean := filepath.Clean(path) + if !manageFiles && (clean == filesRoot || strings.HasPrefix(clean, filesRoot+string(filepath.Separator))) { + return nil + } if _, ok := keep[clean]; ok { return nil } diff --git a/backup/backup_test.go b/backup/backup_test.go index 3ba487d..64a35ea 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -211,7 +211,7 @@ func TestPublicBackupHelpers(t *testing.T) { } func TestResolveShardPathRejectsEscapes(t *testing.T) { - for _, rel := range []string{"../x.age", "data/../x.age", "data/x.txt", "/data/x.age"} { + for _, rel := range []string{"../x.age", "data/../x.age", "data/x.txt", "/data/x.age", `data\files\index.jsonl.gz.age`, `data/files\index.jsonl.gz.age`} { if _, err := ResolveShardPath(t.TempDir(), rel); err == nil { t.Fatalf("expected error for %q", rel) } @@ -231,9 +231,13 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) { if err := mirror.EnsureRepo(ctx, mirrorOpts); err != nil { t.Fatal(err) } - firstManifest, err := WriteSnapshot(ctx, cfg, []Shard{ + mediaSource := filepath.Join(dir, "media.txt") + if err := os.WriteFile(mediaSource, []byte("first media"), 0o600); err != nil { + t.Fatal(err) + } + firstManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{ {Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "first"}}}, - }, Manifest{}) + }, []File{{Path: "media/file.txt", Source: mediaSource}}, Manifest{}) if err != nil { t.Fatal(err) } @@ -247,9 +251,12 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) { if _, err := mirror.CreateImmutableTag(ctx, mirrorOpts, "snapshot/one"); err != nil { t.Fatal(err) } - secondManifest, err := WriteSnapshot(ctx, cfg, []Shard{ + if err := os.WriteFile(mediaSource, []byte("second media"), 0o600); err != nil { + t.Fatal(err) + } + secondManifest, err := WriteSnapshotWithFiles(ctx, cfg, []Shard{ {Table: "messages", Path: "data/messages.jsonl.gz.age", Rows: []row{{ID: "1", Body: "second"}}}, - }, firstManifest) + }, []File{{Path: "media/file.txt", Source: mediaSource}}, firstManifest) if err != nil { t.Fatal(err) } @@ -289,6 +296,24 @@ func TestHistoricalEncryptedSnapshot(t *testing.T) { if len(historicalRows) != 1 || historicalRows[0].Body != "first" { t.Fatalf("historical rows = %#v", historicalRows) } + restoreRoot := filepath.Join(dir, "historical-restore") + if err := os.MkdirAll(restoreRoot, 0o700); err != nil { + t.Fatal(err) + } + restored, restoredRef, err := RestoreFilesAt(ctx, cfg, mirrorOpts, manifest, "snapshot/one", restoreRoot) + if err != nil { + t.Fatal(err) + } + if restored != 1 || restoredRef != first { + t.Fatalf("historical files = %d at %s", restored, restoredRef) + } + mediaBody, err := os.ReadFile(filepath.Join(restoreRoot, "media", "file.txt")) + if err != nil { + t.Fatal(err) + } + if string(mediaBody) != "first media" { + t.Fatalf("historical media = %q", mediaBody) + } if secondManifest.Counts["messages"] != 1 { t.Fatalf("second manifest = %#v", secondManifest) } diff --git a/backup/files.go b/backup/files.go new file mode 100644 index 0000000..bc59354 --- /dev/null +++ b/backup/files.go @@ -0,0 +1,657 @@ +package backup + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "filippo.io/age" +) + +type File struct { + Path string + Source string + info os.FileInfo +} + +const ( + fileIndexTable = "_backup_files" + fileIndexPath = "data/files/index.jsonl.gz.age" +) + +type fileIndexRecord struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +type indexedFile struct { + Entry FileEntry + Record fileIndexRecord +} + +type encryptedFileLoader func(string) (io.ReadCloser, error) + +func CollectFiles(ctx context.Context, root, prefix string) ([]File, error) { + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." { + return nil, fmt.Errorf("file root is required") + } + prefix = strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/") + if prefix != "" { + if _, err := cleanFilePath(prefix); err != nil { + return nil, err + } + } + if _, err := os.Stat(root); os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + var files []File + err := filepath.WalkDir(root, func(source string, entry os.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return walkErr + } + if entry.Type()&os.ModeSymlink != 0 { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, source) + if err != nil { + return err + } + logical := filepath.ToSlash(rel) + if prefix != "" { + logical = path.Join(prefix, logical) + } + logical, err = cleanFilePath(logical) + if err != nil { + return err + } + files = append(files, File{Path: logical, Source: source, info: info}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + return files, nil +} + +func writeFiles(ctx context.Context, cfg Config, old Manifest, files []File, reuseEncrypted bool) ([]FileEntry, []fileIndexRecord, error) { + ordered := append([]File(nil), files...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].Path < ordered[j].Path }) + oldByHash := make(map[string]indexedFile, len(old.Files)) + if reuseEncrypted && len(files) > 0 { + records, err := loadLocalFileIndex(cfg, old) + if err == nil { + for index, entry := range old.Files { + record := records[index] + oldByHash[record.SHA256] = indexedFile{Entry: entry, Record: record} + } + } + } + written := make(map[string]FileEntry, len(files)) + seenPaths := make(map[string]struct{}, len(files)) + out := make([]FileEntry, 0, len(files)) + index := make([]fileIndexRecord, 0, len(files)) + for _, file := range ordered { + if err := ctx.Err(); err != nil { + return nil, nil, err + } + logical, err := cleanFilePath(file.Path) + if err != nil { + return nil, nil, err + } + if _, exists := seenPaths[logical]; exists { + return nil, nil, fmt.Errorf("duplicate backup file path: %s", logical) + } + seenPaths[logical] = struct{}{} + hashValue, size, sourceInfo, err := hashFile(ctx, file) + if err != nil { + return nil, nil, err + } + if entry, ok, err := reusableFileEntry(cfg, oldByHash, written, hashValue, size); err != nil { + return nil, nil, err + } else if ok { + out = append(out, entry) + index = append(index, fileIndexRecord{Path: logical, SHA256: hashValue, Size: size}) + continue + } + tmpPath, hashValue, size, encryptedSize, err := encryptFileTemp(ctx, file, sourceInfo, cfg.Repo, cfg.Recipients) + if err != nil { + return nil, nil, err + } + keepTemp := true + defer func() { + if keepTemp { + _ = os.Remove(tmpPath) + } + }() + if entry, ok, err := reusableFileEntry(cfg, oldByHash, written, hashValue, size); err != nil { + return nil, nil, err + } else if ok { + if err := os.Remove(tmpPath); err != nil { + return nil, nil, err + } + keepTemp = false + out = append(out, entry) + index = append(index, fileIndexRecord{Path: logical, SHA256: hashValue, Size: size}) + continue + } + shard, err := randomFileShard(cfg.Repo) + if err != nil { + return nil, nil, err + } + target, err := ResolveShardPath(cfg.Repo, shard) + if err != nil { + return nil, nil, err + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return nil, nil, err + } + if err := os.Rename(tmpPath, target); err != nil { + return nil, nil, err + } + keepTemp = false + if err := syncDir(filepath.Dir(target)); err != nil { + return nil, nil, err + } + entry := FileEntry{Shard: shard, Bytes: encryptedSize} + written[hashValue] = entry + out = append(out, entry) + index = append(index, fileIndexRecord{Path: logical, SHA256: hashValue, Size: size}) + } + return out, index, nil +} + +func reusableFileEntry(cfg Config, oldByHash map[string]indexedFile, written map[string]FileEntry, hashValue string, size int64) (FileEntry, bool, error) { + if entry, ok := written[hashValue]; ok { + return entry, true, nil + } + oldFile, ok := oldByHash[hashValue] + if !ok || oldFile.Record.Size != size { + return FileEntry{}, false, nil + } + target, err := ResolveShardPath(cfg.Repo, oldFile.Entry.Shard) + if err != nil { + return FileEntry{}, false, err + } + info, err := os.Stat(target) + if err != nil { + if os.IsNotExist(err) { + return FileEntry{}, false, nil + } + return FileEntry{}, false, err + } + entry := FileEntry{Shard: oldFile.Entry.Shard, Bytes: info.Size()} + written[hashValue] = entry + return entry, true, nil +} + +func loadLocalFileIndex(cfg Config, manifest Manifest) ([]fileIndexRecord, error) { + if len(manifest.Files) == 0 { + return nil, nil + } + identityData, err := os.ReadFile(expandHome(cfg.Identity)) // #nosec G304 -- path is configured by the caller. + if err != nil { + return nil, err + } + identity, err := parseIdentity(identityData) + if err != nil { + return nil, err + } + return readFileIndex(identity, manifest, func(rel string) (io.ReadCloser, error) { + shard, err := ResolveShardPath(cfg.Repo, rel) + if err != nil { + return nil, err + } + return os.Open(shard) // #nosec G304 -- ResolveShardPath confines manifest-controlled paths below data/. + }) +} + +func randomFileShard(repo string) (string, error) { + for range 10 { + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return "", err + } + rel := path.Join("data/files/objects", hex.EncodeToString(id[:])+".gz.age") + target, err := ResolveShardPath(repo, rel) + if err != nil { + return "", err + } + if _, err := os.Stat(target); os.IsNotExist(err) { + return rel, nil + } else if err != nil { + return "", err + } + } + return "", fmt.Errorf("generate unique backup file object path") +} + +func RestoreFiles(ctx context.Context, cfg Config, manifest Manifest, targetRoot string) (int, error) { + return RestoreFilesUnder(ctx, cfg, manifest, targetRoot, "") +} + +func RestoreFilesUnder(ctx context.Context, cfg Config, manifest Manifest, targetRoot, requiredPrefix string) (int, error) { + return restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, requiredPrefix, func(rel string) (io.ReadCloser, error) { + shard, err := ResolveShardPath(cfg.Repo, rel) + if err != nil { + return nil, err + } + return os.Open(shard) // #nosec G304 -- ResolveShardPath confines manifest-controlled paths below data/. + }) +} + +func restoreFilesWith(ctx context.Context, identityPath string, manifest Manifest, targetRoot, requiredPrefix string, load encryptedFileLoader) (int, error) { + if manifest.Format != FormatVersion { + return 0, fmt.Errorf("unsupported backup format %d", manifest.Format) + } + identityData, err := os.ReadFile(expandHome(identityPath)) // #nosec G304 -- path is configured by the caller. + if err != nil { + return 0, err + } + identity, err := parseIdentity(identityData) + if err != nil { + return 0, err + } + records, err := readFileIndex(identity, manifest, load) + if err != nil { + return 0, err + } + seen := make(map[string]struct{}, len(manifest.Files)) + prefix := "" + if requiredPrefix != "" { + prefix, err = cleanFilePath(requiredPrefix) + if err != nil { + return 0, err + } + } + for index, entry := range manifest.Files { + if err := ctx.Err(); err != nil { + return 0, err + } + record := records[index] + logical := record.Path + logical, err = cleanFilePath(logical) + if err != nil { + return 0, err + } + if prefix != "" && logical != prefix && !strings.HasPrefix(logical, prefix+"/") { + return 0, fmt.Errorf("backup file path is outside required prefix %s: %s", prefix, logical) + } + if _, exists := seen[logical]; exists { + return 0, fmt.Errorf("duplicate backup file path: %s", logical) + } + seen[logical] = struct{}{} + if _, err := ResolveShardPath(".", entry.Shard); err != nil { + return 0, err + } + ciphertext, err := load(entry.Shard) + if err != nil { + return 0, err + } + err = restoreFile(ctx, identity, ciphertext, targetRoot, logical, record) + closeErr := ciphertext.Close() + if err != nil { + return 0, err + } + if closeErr != nil { + return 0, closeErr + } + } + return len(manifest.Files), nil +} + +func readFileIndex(identity *age.X25519Identity, manifest Manifest, load encryptedFileLoader) ([]fileIndexRecord, error) { + if len(manifest.Files) == 0 { + return []fileIndexRecord{}, nil + } + var indexEntry *ShardEntry + for index := range manifest.Shards { + if manifest.Shards[index].Table != fileIndexTable { + continue + } + if indexEntry != nil { + return nil, fmt.Errorf("backup contains multiple file indexes") + } + indexEntry = &manifest.Shards[index] + } + if indexEntry == nil { + return nil, fmt.Errorf("backup file index is missing") + } + ciphertext, err := load(indexEntry.Path) + if err != nil { + return nil, err + } + plaintext, decryptErr := decryptFilePayload(identity, ciphertext) + closeErr := ciphertext.Close() + if decryptErr != nil { + return nil, decryptErr + } + if closeErr != nil { + return nil, closeErr + } + if SHA256Hex(plaintext) != indexEntry.SHA256 { + return nil, fmt.Errorf("backup file index hash mismatch") + } + var records []fileIndexRecord + if err := DecodeJSONL(plaintext, &records); err != nil { + return nil, fmt.Errorf("decode backup file index: %w", err) + } + if len(records) != len(manifest.Files) { + return nil, fmt.Errorf("backup file index count mismatch") + } + seen := make(map[string]struct{}, len(records)) + for index := range records { + record := &records[index] + logical, err := cleanFilePath(record.Path) + if err != nil { + return nil, err + } + if _, exists := seen[logical]; exists { + return nil, fmt.Errorf("duplicate backup file index path: %s", logical) + } + seen[logical] = struct{}{} + record.Path = logical + if record.Size < 0 { + return nil, fmt.Errorf("invalid backup file size for %s", logical) + } + hashBytes, err := hex.DecodeString(record.SHA256) + if err != nil || len(hashBytes) != sha256.Size { + return nil, fmt.Errorf("invalid backup file hash for %s", logical) + } + } + return records, nil +} + +func decryptFilePayload(identity *age.X25519Identity, ciphertext io.Reader) ([]byte, error) { + decrypted, err := age.Decrypt(ciphertext, identity) + if err != nil { + return nil, err + } + gz, err := gzip.NewReader(decrypted) + if err != nil { + return nil, err + } + defer gz.Close() + return io.ReadAll(gz) +} + +func restoreFile(ctx context.Context, identity *age.X25519Identity, ciphertext io.Reader, targetRoot, logical string, record fileIndexRecord) error { + target, err := safeRestoreTarget(targetRoot, logical) + if err != nil { + return err + } + decrypted, err := age.Decrypt(ciphertext, identity) + if err != nil { + return err + } + gz, err := gzip.NewReader(decrypted) + if err != nil { + return err + } + defer gz.Close() + tmp, err := os.CreateTemp(filepath.Dir(target), "."+filepath.Base(target)+".tmp-") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + hasher := sha256.New() + size, err := copyContext(ctx, io.MultiWriter(tmp, hasher), gz) + if err != nil { + _ = tmp.Close() + return err + } + if err := gz.Close(); err != nil { + _ = tmp.Close() + return err + } + if size != record.Size || hex.EncodeToString(hasher.Sum(nil)) != record.SHA256 { + _ = tmp.Close() + return fmt.Errorf("backup file verification failed for %s", logical) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + // os.Rename replaces regular files, including via MoveFileEx with + // MOVEFILE_REPLACE_EXISTING on Windows, without first hiding the old file. + if err := os.Rename(tmpPath, target); err != nil { + return err + } + return syncDir(filepath.Dir(target)) +} + +func encryptFileTemp(ctx context.Context, source File, expected os.FileInfo, repo string, recipientStrings []string) (string, string, int64, int64, error) { + recipients, err := parseRecipients(recipientStrings) + if err != nil { + return "", "", 0, 0, err + } + tmpDir := filepath.Join(repo, "data", "files") + if err := os.MkdirAll(tmpDir, 0o700); err != nil { + return "", "", 0, 0, err + } + in, _, err := openSourceFile(source, expected) + if err != nil { + return "", "", 0, 0, err + } + defer in.Close() + tmp, err := os.CreateTemp(tmpDir, ".file.tmp-") + if err != nil { + return "", "", 0, 0, err + } + tmpPath := tmp.Name() + cleanup := true + defer func() { + if cleanup { + _ = os.Remove(tmpPath) + } + }() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + encrypted, err := age.Encrypt(tmp, recipients...) + if err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + gz := gzip.NewWriter(encrypted) + gz.ModTime = time.Unix(0, 0).UTC() + hasher := sha256.New() + size, err := copyContext(ctx, gz, io.TeeReader(in, hasher)) + if err != nil { + _ = gz.Close() + _ = encrypted.Close() + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := gz.Close(); err != nil { + _ = encrypted.Close() + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := encrypted.Close(); err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return "", "", 0, 0, err + } + if err := tmp.Close(); err != nil { + return "", "", 0, 0, err + } + info, err := os.Stat(tmpPath) + if err != nil { + return "", "", 0, 0, err + } + cleanup = false + return tmpPath, hex.EncodeToString(hasher.Sum(nil)), size, info.Size(), nil +} + +func hashFile(ctx context.Context, source File) (string, int64, os.FileInfo, error) { + file, info, err := openSourceFile(source, source.info) + if err != nil { + return "", 0, nil, err + } + defer file.Close() + hasher := sha256.New() + size, err := copyContext(ctx, hasher, file) + if err != nil { + return "", 0, nil, err + } + return hex.EncodeToString(hasher.Sum(nil)), size, info, nil +} + +func openSourceFile(source File, expected os.FileInfo) (*os.File, os.FileInfo, error) { + before, err := os.Lstat(source.Source) + if err != nil { + return nil, nil, err + } + if !before.Mode().IsRegular() { + return nil, nil, fmt.Errorf("backup file is not regular: %s", source.Source) + } + if expected != nil && !os.SameFile(expected, before) { + return nil, nil, fmt.Errorf("backup file changed during collection: %s", source.Source) + } + file, err := os.Open(source.Source) // #nosec G304 -- identity checks bind the opened file to the collected regular file before reads. + if err != nil { + return nil, nil, err + } + after, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, nil, err + } + if !after.Mode().IsRegular() || !os.SameFile(before, after) || (expected != nil && !os.SameFile(expected, after)) { + _ = file.Close() + return nil, nil, fmt.Errorf("backup file changed before open: %s", source.Source) + } + return file, after, nil +} + +func safeRestoreTarget(root, logical string) (string, error) { + logical, err := cleanFilePath(logical) + if err != nil { + return "", err + } + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." { + return "", fmt.Errorf("restore root is required") + } + if err := ensureDirectory(root); err != nil { + return "", err + } + parts := strings.Split(logical, "/") + parent := root + for _, part := range parts[:len(parts)-1] { + parent = filepath.Join(parent, part) + if err := ensureDirectory(parent); err != nil { + return "", err + } + } + target := filepath.Join(root, filepath.FromSlash(logical)) + if info, err := os.Lstat(target); err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return "", fmt.Errorf("restore target is not a regular file: %s", logical) + } + } else if !os.IsNotExist(err) { + return "", err + } + return target, nil +} + +func ensureDirectory(dir string) error { + info, err := os.Lstat(dir) + if os.IsNotExist(err) { + return os.Mkdir(dir, 0o700) + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("restore path is not a directory: %s", dir) + } + return nil +} + +func cleanFilePath(value string) (string, error) { + if value == "" || strings.Contains(value, "\\") { + return "", fmt.Errorf("invalid backup file path: %s", value) + } + clean := path.Clean(value) + local := filepath.FromSlash(clean) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) || filepath.IsAbs(local) || filepath.VolumeName(local) != "" { + return "", fmt.Errorf("backup file path escapes restore root: %s", value) + } + return clean, nil +} + +func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { + buffer := make([]byte, 256*1024) + var total int64 + for { + if err := ctx.Err(); err != nil { + return total, err + } + read, readErr := src.Read(buffer) + if read > 0 { + written, writeErr := dst.Write(buffer[:read]) + total += int64(written) + if writeErr != nil { + return total, writeErr + } + if written != read { + return total, io.ErrShortWrite + } + } + if readErr == io.EOF { + return total, nil + } + if readErr != nil { + return total, readErr + } + } +} + +func readEncryptedFileBytes(data []byte) io.ReadCloser { + return io.NopCloser(bytes.NewReader(data)) +} diff --git a/backup/files_test.go b/backup/files_test.go new file mode 100644 index 0000000..0e2bcb5 --- /dev/null +++ b/backup/files_test.go @@ -0,0 +1,254 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestEncryptedSnapshotFilesDeduplicateAndRestore(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + source := filepath.Join(dir, "source") + if err := os.MkdirAll(filepath.Join(source, "nested"), 0o700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"photo.jpg", filepath.Join("nested", "copy.jpg")} { + if err := os.WriteFile(filepath.Join(source, name), []byte("same private media"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(filepath.Join(source, "photo.jpg"), filepath.Join(source, "linked.jpg")); err != nil && runtime.GOOS != "windows" { + t.Fatal(err) + } + files, err := CollectFiles(ctx, source, "media") + if err != nil { + t.Fatal(err) + } + if len(files) != 2 { + t.Fatalf("collected files = %#v", files) + } + identity := filepath.Join(dir, "age.key") + recipient, err := EnsureIdentity(identity) + if err != nil { + t.Fatal(err) + } + cfg := Config{Repo: filepath.Join(dir, "repo"), Identity: identity, Recipients: []string{recipient}} + if err := os.MkdirAll(cfg.Repo, 0o700); err != nil { + t.Fatal(err) + } + manifest, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}) + if err != nil { + t.Fatal(err) + } + if len(manifest.Files) != 2 || manifest.Files[0].Shard != manifest.Files[1].Shard { + t.Fatalf("files were not content-deduplicated: %#v", manifest.Files) + } + manifestBody, err := os.ReadFile(filepath.Join(cfg.Repo, "manifest.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(manifestBody), "photo.jpg") || strings.Contains(string(manifestBody), "copy.jpg") { + t.Fatalf("manifest exposes logical media paths: %s", manifestBody) + } + if strings.Contains(string(manifestBody), SHA256Hex([]byte("same private media"))) { + t.Fatalf("manifest exposes plaintext media hash: %s", manifestBody) + } + ciphertext, err := os.ReadFile(filepath.Join(cfg.Repo, filepath.FromSlash(manifest.Files[0].Shard))) + if err != nil { + t.Fatal(err) + } + if string(ciphertext) == "same private media" { + t.Fatal("backup file was stored as plaintext") + } + if err := RemoveStaleShards(cfg.Repo, manifest.Shards); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(cfg.Repo, filepath.FromSlash(manifest.Files[0].Shard))); err != nil { + t.Fatalf("legacy stale cleanup removed file bundle object: %v", err) + } + second, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, manifest) + if err != nil { + t.Fatal(err) + } + if !EquivalentManifest(manifest, second) || !second.Exported.Equal(manifest.Exported) { + t.Fatalf("unchanged files rewrote manifest: %#v", second) + } + publicOnly := cfg + publicOnly.Identity = "" + publicOnly.Repo = filepath.Join(dir, "public-only") + if err := os.MkdirAll(publicOnly.Repo, 0o700); err != nil { + t.Fatal(err) + } + withoutReuse, err := WriteSnapshotWithFiles(ctx, publicOnly, nil, files, manifest) + if err != nil { + t.Fatalf("public-recipient-only writer should re-encrypt files: %v", err) + } + if len(withoutReuse.Files) != len(files) { + t.Fatalf("public-only file manifest = %#v", withoutReuse.Files) + } + withoutFiles, err := WriteSnapshotWithFiles(ctx, publicOnly, nil, nil, withoutReuse) + if err != nil { + t.Fatalf("removing files should not require an identity: %v", err) + } + if len(withoutFiles.Files) != 0 { + t.Fatalf("removed file manifest = %#v", withoutFiles.Files) + } + restoreRoot := filepath.Join(dir, "restore") + if err := os.MkdirAll(restoreRoot, 0o700); err != nil { + t.Fatal(err) + } + count, err := RestoreFiles(ctx, cfg, manifest, restoreRoot) + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("restored files = %d", count) + } + for _, name := range []string{filepath.Join("media", "photo.jpg"), filepath.Join("media", "nested", "copy.jpg")} { + body, err := os.ReadFile(filepath.Join(restoreRoot, name)) + if err != nil { + t.Fatal(err) + } + if string(body) != "same private media" { + t.Fatalf("restored %s = %q", name, body) + } + } + photo := filepath.Join(restoreRoot, "media", "photo.jpg") + if err := os.WriteFile(photo, []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := RestoreFiles(ctx, cfg, manifest, restoreRoot); err != nil { + t.Fatalf("repeat restore: %v", err) + } + if body, err := os.ReadFile(photo); err != nil || string(body) != "same private media" { + t.Fatalf("repeat restored photo = %q err=%v", body, err) + } + confinedRoot := filepath.Join(dir, "confined") + if err := os.MkdirAll(confinedRoot, 0o700); err != nil { + t.Fatal(err) + } + if _, err := RestoreFilesUnder(ctx, cfg, manifest, confinedRoot, "attachments"); err == nil { + t.Fatal("restore outside required prefix should fail") + } +} + +func TestRestoreFilesRejectsUnsafePaths(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + identity := filepath.Join(dir, "age.key") + if _, err := EnsureIdentity(identity); err != nil { + t.Fatal(err) + } + cfg := Config{Repo: dir, Identity: identity} + if _, err := safeRestoreTarget(filepath.Join(dir, "restore"), "../escape"); err == nil { + t.Fatal("unsafe restore path should fail") + } + + source := filepath.Join(dir, "source") + if err := os.MkdirAll(source, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "file"), []byte("body"), 0o600); err != nil { + t.Fatal(err) + } + files, err := CollectFiles(ctx, source, "media") + if err != nil { + t.Fatal(err) + } + recipient, err := RecipientFromIdentity(identity) + if err != nil { + t.Fatal(err) + } + cfg.Recipients = []string{recipient} + manifest, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}) + if err != nil { + t.Fatal(err) + } + restoreRoot := filepath.Join(dir, "target") + outside := filepath.Join(dir, "outside") + if err := os.MkdirAll(outside, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(restoreRoot, 0o700); err != nil { + t.Fatal(err) + } + symlinkOrSkip(t, outside, filepath.Join(restoreRoot, "media")) + if _, err := RestoreFiles(ctx, cfg, manifest, restoreRoot); err == nil { + t.Fatal("symlinked restore directory should fail") + } +} + +func TestCollectFilesPreservesWhitespaceAndRejectsSwappedSymlink(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + root := filepath.Join(dir, "source") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + names := []string{"name"} + if runtime.GOOS != "windows" { + names = append(names, "name ") + } + for _, name := range names { + if err := os.WriteFile(filepath.Join(root, name), []byte(name), 0o600); err != nil { + t.Fatal(err) + } + } + files, err := CollectFiles(ctx, root, "media") + if err != nil { + t.Fatal(err) + } + if len(files) != len(names) || files[0].Path != "media/name" || (runtime.GOOS != "windows" && files[1].Path != "media/name ") { + t.Fatalf("whitespace paths changed: %#v", files) + } + + outside := filepath.Join(dir, "outside") + if err := os.WriteFile(outside, []byte("outside secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Remove(files[0].Source); err != nil { + t.Fatal(err) + } + symlinkOrSkip(t, outside, files[0].Source) + identity := filepath.Join(dir, "age.key") + recipient, err := EnsureIdentity(identity) + if err != nil { + t.Fatal(err) + } + cfg := Config{Repo: filepath.Join(dir, "repo"), Identity: identity, Recipients: []string{recipient}} + if _, err := WriteSnapshotWithFiles(ctx, cfg, nil, files, Manifest{}); err == nil { + t.Fatal("symlink-swapped source should fail") + } + if _, err := os.Stat(filepath.Join(cfg.Repo, "manifest.json")); !os.IsNotExist(err) { + t.Fatalf("failed backup wrote a manifest: %v", err) + } +} + +func symlinkOrSkip(t *testing.T, oldname, newname string) { + t.Helper() + if err := os.Symlink(oldname, newname); err != nil { + if runtime.GOOS == "windows" { + t.Skipf("symlink unavailable: %v", err) + } + t.Fatal(err) + } +} + +func TestWriteSnapshotRejectsReservedFileIndexNamespace(t *testing.T) { + ctx := context.Background() + for _, shard := range []Shard{ + {Table: fileIndexTable, Path: "data/custom.jsonl.gz.age", Rows: []row{}}, + {Table: "custom", Path: fileIndexPath, Rows: []row{}}, + {Table: "custom", Path: "data/files/aa/custom.gz.age", Rows: []row{}}, + {Table: "custom", Path: `data\files\index.jsonl.gz.age`, Rows: []row{}}, + {Table: "custom", Path: `data/files\index.jsonl.gz.age`, Rows: []row{}}, + } { + if _, err := WriteSnapshotWithFiles(ctx, Config{}, []Shard{shard}, nil, Manifest{}); err == nil { + t.Fatalf("reserved shard should fail: %#v", shard) + } + } +} diff --git a/backup/history.go b/backup/history.go index 44b21a5..9f435b9 100644 --- a/backup/history.go +++ b/backup/history.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "path" + "io" "strings" "github.com/openclaw/crawlkit/mirror" @@ -67,10 +67,10 @@ func ReadSnapshotAt(ctx context.Context, cfg Config, opts mirror.Options, manife return nil, "", err } shards, err := readSnapshotWith(manifest, func(shard ShardEntry) ([]byte, error) { - if _, err := ResolveShardPath(cfg.Repo, shard.Path); err != nil { + clean, err := cleanShardPath(shard.Path) + if err != nil { return nil, err } - clean := path.Clean(strings.TrimSpace(shard.Path)) ciphertext, resolved, err := mirror.ReadFileAt(ctx, opts, commit, clean) if err != nil { return nil, err @@ -85,3 +85,32 @@ func ReadSnapshotAt(ctx context.Context, cfg Config, opts mirror.Options, manife } return shards, commit, nil } + +func RestoreFilesAt(ctx context.Context, cfg Config, opts mirror.Options, manifest Manifest, ref, targetRoot string) (int, string, error) { + return RestoreFilesAtUnder(ctx, cfg, opts, manifest, ref, targetRoot, "") +} + +func RestoreFilesAtUnder(ctx context.Context, cfg Config, opts mirror.Options, manifest Manifest, ref, targetRoot, requiredPrefix string) (int, string, error) { + commit, err := mirror.ResolveCommit(ctx, opts, ref) + if err != nil { + return 0, "", err + } + count, err := restoreFilesWith(ctx, cfg.Identity, manifest, targetRoot, requiredPrefix, func(rel string) (io.ReadCloser, error) { + clean, err := cleanShardPath(rel) + if err != nil { + return nil, err + } + ciphertext, resolved, err := mirror.ReadFileAt(ctx, opts, commit, clean) + if err != nil { + return nil, err + } + if resolved != commit { + return nil, fmt.Errorf("backup ref changed while reading %s", clean) + } + return readEncryptedFileBytes(ciphertext), nil + }) + if err != nil { + return 0, "", err + } + return count, commit, nil +}