diff --git a/CHANGELOG.md b/CHANGELOG.md index 0657c1a21..25f652999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The following emojis are used to highlight certain changes: ### Added +- `files`, `ipld/unixfs`, `mfs` and `tar` now support optional UnixFS 1.5 mode and modification time metadata + ### Changed ### Removed diff --git a/examples/go.mod b/examples/go.mod index 615b39356..40d3c3696 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -3,13 +3,13 @@ module github.com/ipfs/boxo/examples go 1.21 require ( - github.com/ipfs/boxo v0.19.0 + github.com/ipfs/boxo v0.22.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-ipld-prime v0.21.0 - github.com/libp2p/go-libp2p v0.36.1 + github.com/libp2p/go-libp2p v0.36.2 github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/multiformats/go-multiaddr v0.13.0 github.com/multiformats/go-multicodec v0.9.0 @@ -122,7 +122,7 @@ require ( github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.8 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/ice/v2 v2.3.32 // indirect + github.com/pion/ice/v2 v2.3.34 // indirect github.com/pion/interceptor v0.1.29 // indirect github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.12 // indirect @@ -133,9 +133,9 @@ require ( github.com/pion/sdp/v3 v3.0.9 // indirect github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/transport/v2 v2.2.9 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/webrtc/v3 v3.2.50 // indirect + github.com/pion/webrtc/v3 v3.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect diff --git a/examples/go.sum b/examples/go.sum index 67b7b18ea..e247582dd 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -271,8 +271,8 @@ github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+ github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.36.1 h1:piAHesy0/8ifBEBUS8HF2m7ywR5vnktUFv00dTsVKcs= -github.com/libp2p/go-libp2p v0.36.1/go.mod h1:vHzel3CpRB+vS11fIjZSJAU4ALvieKV9VZHC9VerHj8= +github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U= +github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= @@ -372,8 +372,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI= -github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU= +github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM= +github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -399,17 +399,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/ github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= -github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE= -github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw= github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34= -github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo= +github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= +github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/files/file.go b/files/file.go index 7ac1fc98a..e2ece2862 100644 --- a/files/file.go +++ b/files/file.go @@ -4,6 +4,7 @@ import ( "errors" "io" "os" + "time" ) var ( @@ -17,6 +18,14 @@ var ( type Node interface { io.Closer + // Mode returns the mode. + // Optional, if unknown/unspecified returns zero. + Mode() os.FileMode + + // ModTime returns the last modification time. If the last + // modification time is unknown/unspecified ModTime returns zero. + ModTime() (mtime time.Time) + // Size returns size of this file (if this file is a directory, total size of // all files stored in the tree should be returned). Some implementations may // choose not to implement this diff --git a/files/file_test.go b/files/file_test.go index 3edecf107..07d9f04db 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -3,8 +3,10 @@ package files import ( "io" "mime/multipart" + "os" "strings" "testing" + "time" ) func TestSliceFiles(t *testing.T) { @@ -49,6 +51,21 @@ func TestReaderFiles(t *testing.T) { } } +func TestReaderFileStat(t *testing.T) { + reader := strings.NewReader("beep boop") + mode := os.FileMode(0754) + mtime := time.Date(2020, 11, 2, 12, 27, 35, 55555, time.UTC) + stat := &mockFileInfo{name: "test", mode: mode, mtime: mtime} + + rf := NewReaderStatFile(reader, stat) + if rf.Mode() != mode { + t.Fatalf("Expected file mode to be [%v] but got [%v]", mode, rf.Mode()) + } + if rf.ModTime() != mtime { + t.Fatalf("Expected file modified time to be [%v] but got [%v]", mtime, rf.ModTime()) + } +} + func TestMultipartFiles(t *testing.T) { data := ` --Boundary! @@ -141,3 +158,127 @@ implicit file2 }, }) } + +func TestMultipartFilesWithMode(t *testing.T) { + data := ` +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="file-0?mode=0754&mtime=1604320500&mtime-nsecs=55555"; filename="%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC" +Some-Header: beep + +beep +--Boundary! +Content-Type: application/x-directory +Content-Disposition: form-data; name="dir-0?mode=755&mtime=1604320500"; ans=42; filename="dir1" + +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="file"; filename="dir1/nested" + +some content +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="file?mode=600"; filename="dir1/nested2"; ans=42 + +some content +--Boundary! +Content-Type: application/symlink +Content-Disposition: form-data; name="file-5"; filename="dir1/simlynk" + +anotherfile +--Boundary! +Content-Type: application/symlink +Content-Disposition: form-data; name="file?mtime=1604320500"; filename="dir1/simlynk2" + +anotherfile +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="dir?mode=0644"; filename="implicit1/implicit2/deep_implicit" + +implicit file1 +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="dir?mode=755&mtime=1604320500"; filename="implicit1/shallow_implicit" + +implicit file2 +--Boundary!-- + +` + + reader := strings.NewReader(data) + mpReader := multipart.NewReader(reader, "Boundary!") + dir, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + CheckDir(t, dir, []Event{ + { + kind: TFile, + name: "£ẞǑǓÆ æ ♫♬", + value: "beep", + mode: 0754, + mtime: time.Unix(1604320500, 55555), + }, + { + kind: TDirStart, + name: "dir1", + mode: 0755, + mtime: time.Unix(1604320500, 0), + }, + { + kind: TFile, + name: "nested", + value: "some content", + }, + { + kind: TFile, + name: "nested2", + value: "some content", + mode: 0600, + }, + { + kind: TSymlink, + name: "simlynk", + value: "anotherfile", + mode: 0777, + }, + { + kind: TSymlink, + name: "simlynk2", + value: "anotherfile", + mode: 0777, + mtime: time.Unix(1604320500, 0), + }, + { + kind: TDirEnd, + }, + { + kind: TDirStart, + name: "implicit1", + }, + { + kind: TDirStart, + name: "implicit2", + }, + { + kind: TFile, + name: "deep_implicit", + value: "implicit file1", + mode: 0644, + }, + { + kind: TDirEnd, + }, + { + kind: TFile, + name: "shallow_implicit", + value: "implicit file2", + mode: 0755, + mtime: time.Unix(1604320500, 0), + }, + { + kind: TDirEnd, + }, + }) +} diff --git a/files/filter_test.go b/files/filter_test.go index 00b2e8baf..f2de61168 100644 --- a/files/filter_test.go +++ b/files/filter_test.go @@ -4,17 +4,33 @@ import ( "os" "path/filepath" "testing" + "time" ) type mockFileInfo struct { os.FileInfo - name string + name string + mode os.FileMode + mtime time.Time + size int64 } func (m *mockFileInfo) Name() string { return m.name } +func (m *mockFileInfo) Mode() os.FileMode { + return m.mode +} + +func (m *mockFileInfo) ModTime() time.Time { + return m.mtime +} + +func (m *mockFileInfo) Size() int64 { + return m.size +} + func (m *mockFileInfo) Sys() interface{} { return nil } diff --git a/files/helpers_test.go b/files/helpers_test.go index 0180b8f27..32e54544e 100644 --- a/files/helpers_test.go +++ b/files/helpers_test.go @@ -2,7 +2,9 @@ package files import ( "io" + "os" "testing" + "time" ) type Kind int @@ -18,6 +20,8 @@ type Event struct { kind Kind name string value string + mode os.FileMode + mtime time.Time } func CheckDir(t *testing.T, dir Directory, expected []Event) { @@ -50,6 +54,14 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) { t.Fatalf("[%d] expected filename to be %q", i, next.name) } + if next.mode != 0 && it.Node().Mode()&os.ModePerm != next.mode { + t.Fatalf("[%d] expected mode for '%s' to be %O, got %O", i, it.Name(), next.mode, it.Node().Mode()) + } + + if !next.mtime.IsZero() && !it.Node().ModTime().Equal(next.mtime) { + t.Fatalf("[%d] expected modification time for '%s' to be %q", i, it.Name(), next.mtime) + } + switch next.kind { case TFile: mf, ok := it.Node().(File) diff --git a/files/linkfile.go b/files/linkfile.go index 526998652..6881068f7 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -3,21 +3,41 @@ package files import ( "os" "strings" + "time" ) type Symlink struct { Target string - stat os.FileInfo + mtime time.Time reader strings.Reader } func NewLinkFile(target string, stat os.FileInfo) File { - lf := &Symlink{Target: target, stat: stat} + mtime := time.Time{} + if stat != nil { + mtime = stat.ModTime() + } + return NewSymlinkFile(target, mtime) +} + +func NewSymlinkFile(target string, mtime time.Time) File { + lf := &Symlink{ + Target: target, + mtime: mtime, + } lf.reader.Reset(lf.Target) return lf } +func (lf *Symlink) Mode() os.FileMode { + return os.ModeSymlink | os.ModePerm +} + +func (lf *Symlink) ModTime() time.Time { + return lf.mtime +} + func (lf *Symlink) Close() error { return nil } diff --git a/files/meta.go b/files/meta.go new file mode 100644 index 000000000..4ae38a991 --- /dev/null +++ b/files/meta.go @@ -0,0 +1,46 @@ +package files + +import ( + "fmt" + "os" + "time" +) + +// UpdateMeta sets the unix mode and modification time of the filesystem object +// referenced by path. +func UpdateMeta(path string, mode os.FileMode, mtime time.Time) error { + if err := UpdateModTime(path, mtime); err != nil { + return err + } + return UpdateFileMode(path, mode) +} + +// UpdateUnix sets the unix mode and modification time of the filesystem object +// referenced by path. The mode is in the form of a unix mode. +func UpdateMetaUnix(path string, mode uint32, mtime time.Time) error { + return UpdateMeta(path, UnixPermsToModePerms(mode), mtime) +} + +// UpdateFileMode sets the unix mode of the filesystem object referenced by path. +func UpdateFileMode(path string, mode os.FileMode) error { + if err := updateMode(path, mode); err != nil { + return fmt.Errorf("[%v] failed to update file mode on '%s'", err, path) + } + return nil +} + +// UpdateFileModeUnix sets the unix mode of the filesystem object referenced by +// path. It takes the mode in the form of a unix mode. +func UpdateFileModeUnix(path string, mode uint32) error { + return UpdateFileMode(path, UnixPermsToModePerms(mode)) +} + +// UpdateModTime sets the last access and modification time of the target +// filesystem object to the given time. When the given path references a +// symlink, if supported, the symlink is updated. +func UpdateModTime(path string, mtime time.Time) error { + if err := updateMtime(path, mtime); err != nil { + return fmt.Errorf("[%v] failed to update last modification time on '%s'", err, path) + } + return nil +} diff --git a/files/meta_other.go b/files/meta_other.go new file mode 100644 index 000000000..2c4645049 --- /dev/null +++ b/files/meta_other.go @@ -0,0 +1,23 @@ +//go:build !linux && !freebsd && !netbsd && !openbsd && !dragonfly && !windows +// +build !linux,!freebsd,!netbsd,!openbsd,!dragonfly,!windows + +package files + +import ( + "os" + "time" +) + +func updateMode(path string, mode os.FileMode) error { + if mode == 0 { + return nil + } + return os.Chmod(path, mode) +} + +func updateMtime(path string, mtime time.Time) error { + if mtime.IsZero() { + return nil + } + return os.Chtimes(path, mtime, mtime) +} diff --git a/files/meta_posix.go b/files/meta_posix.go new file mode 100644 index 000000000..808cbb997 --- /dev/null +++ b/files/meta_posix.go @@ -0,0 +1,41 @@ +//go:build linux || freebsd || netbsd || openbsd || dragonfly +// +build linux freebsd netbsd openbsd dragonfly + +package files + +import ( + "os" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +func updateMode(path string, mode os.FileMode) error { + if mode == 0 { + return nil + } + return os.Chmod(path, mode) +} + +func updateMtime(path string, mtime time.Time) error { + if mtime.IsZero() { + return nil + } + var AtFdCwd = -100 + pathname, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + tm := syscall.NsecToTimespec(mtime.UnixNano()) + ts := [2]syscall.Timespec{tm, tm} + _, _, e := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AtFdCwd), + uintptr(unsafe.Pointer(pathname)), uintptr(unsafe.Pointer(&ts)), + uintptr(unix.AT_SYMLINK_NOFOLLOW), 0, 0) + if e != 0 { + return error(e) + } + return nil +} diff --git a/files/meta_windows.go b/files/meta_windows.go new file mode 100644 index 000000000..e060ec02f --- /dev/null +++ b/files/meta_windows.go @@ -0,0 +1,30 @@ +package files + +import ( + "os" + "time" +) + +// os.Chmod - On Windows, only the 0200 bit (owner writable) of mode is used; It +// controls whether the file's read-only attribute is set or cleared. The other +// bits are currently unused. +// +// Use mode 0400 for a read-only file and 0600 for a readable+writable file. +func updateMode(path string, mode os.FileMode) error { + if mode == 0 { + return nil + } + // read+write if owner, group or world writeable + if mode&0222 != 0 { + return os.Chmod(path, 0600) + } + // otherwise read-only + return os.Chmod(path, 0400) +} + +func updateMtime(path string, mtime time.Time) error { + if mtime.IsZero() { + return nil + } + return os.Chtimes(path, mtime, mtime) +} diff --git a/files/multifilereader.go b/files/multifilereader.go index 1a5d4ac1a..33fde2889 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -8,6 +8,8 @@ import ( "net/textproto" "net/url" "path" + "strconv" + "strings" "sync" ) @@ -89,18 +91,12 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // handle starting a new file part if !mfr.closed { - mfr.currentFile = entry.Node() // write the boundary and headers header := make(textproto.MIMEHeader) - filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name())) - dispositionPrefix := "attachment" - if mfr.form { - dispositionPrefix = "form-data; name=\"file\"" - } - - header.Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"", dispositionPrefix, filename)) + filename := path.Join(path.Join(mfr.path...), entry.Name()) + mfr.addContentDisposition(header, filename) var contentType string @@ -119,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { return 0, ErrNotSupported } - header.Set("Content-Type", contentType) + header.Set(contentTypeHeader, contentType) if rf, ok := entry.Node().(FileInfo); ok { if mfr.rawAbsPath { // Legacy compatibility with old servers. @@ -157,6 +153,36 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { return written, nil } +func (mfr *MultiFileReader) addContentDisposition(header textproto.MIMEHeader, filename string) { + sb := &strings.Builder{} + params := url.Values{} + + if mode := mfr.currentFile.Mode(); mode != 0 { + params.Add("mode", "0"+strconv.FormatUint(uint64(mode), 8)) + } + if mtime := mfr.currentFile.ModTime(); !mtime.IsZero() { + params.Add("mtime", strconv.FormatInt(mtime.Unix(), 10)) + if n := mtime.Nanosecond(); n > 0 { + params.Add("mtime-nsecs", strconv.FormatInt(int64(n), 10)) + } + } + + sb.Grow(120) + if mfr.form { + sb.WriteString("form-data; name=\"file") + if len(params) > 0 { + fmt.Fprintf(sb, "?%s", params.Encode()) + } + sb.WriteString("\"") + } else { + sb.WriteString("attachment") + } + + fmt.Fprintf(sb, "; filename=\"%s\"", url.QueryEscape(filename)) + + header.Set(contentDispositionHeader, sb.String()) +} + // Boundary returns the boundary string to be used to separate files in the multipart data func (mfr *MultiFileReader) Boundary() string { return mfr.mpWriter.Boundary() diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index b39217037..623c5404a 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -3,8 +3,13 @@ package files import ( "bytes" "io" + "io/fs" "mime/multipart" + "net/textproto" + "path" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -12,7 +17,12 @@ import ( var text = "Some text! :)" func newBytesFileWithPath(abspath string, b []byte) File { - return &ReaderFile{abspath, bytesReaderCloser{bytes.NewReader(b)}, nil, int64(len(b))} + return &ReaderFile{ + abspath: abspath, + reader: bytesReaderCloser{bytes.NewReader(b)}, + stat: &mockFileInfo{name: path.Base(abspath), mode: 0754, mtime: time.Unix(1604320500, 55555)}, + fsize: int64(len(b)), + } } func makeMultiFileReader(t *testing.T, binaryFileName, rawAbsPath bool) (string, *MultiFileReader) { @@ -53,6 +63,9 @@ func runMultiFileReaderToMultiFileTest(t *testing.T, binaryFileName, rawAbsPath, require.True(t, it.Next()) require.Equal(t, "beep.txt", it.Name()) + n := it.Node() + require.Equal(t, fs.FileMode(0754), n.Mode(), "unexpected file mode") + require.Equal(t, time.Unix(1604320500, 55555), n.ModTime(), "unexpected last modification time") require.True(t, it.Next()) require.Equal(t, "boop", it.Name()) require.NotNil(t, DirFromEntry(it)) @@ -103,12 +116,20 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { func getTestMultiFileReader(t *testing.T) *MultiFileReader { sf := NewMapDirectory(map[string]Node{ - "file.txt": NewBytesFile([]byte(text)), + "file.txt": NewReaderStatFile( + strings.NewReader(text), + &mockFileInfo{name: "file.txt", mode: 0, mtime: time.Time{}}), "boop": NewMapDirectory(map[string]Node{ - "a.txt": NewBytesFile([]byte("bleep")), - "b.txt": NewBytesFile([]byte("bloop")), + "a.txt": NewReaderStatFile( + strings.NewReader("bleep"), + &mockFileInfo{name: "a.txt", mode: 0744, mtime: time.Time{}}), + "b.txt": NewReaderStatFile( + strings.NewReader("bloop"), + &mockFileInfo{name: "b.txt", mode: 0666, mtime: time.Unix(1604320500, 0)}), }), - "beep.txt": NewBytesFile([]byte("beep")), + "beep.txt": NewReaderStatFile( + strings.NewReader("beep"), + &mockFileInfo{name: "beep.txt", mode: 0754, mtime: time.Unix(1604320500, 55555)}), }) // testing output by reading it with the go stdlib "mime/multipart" Reader @@ -242,3 +263,25 @@ func TestCommonPrefix(t *testing.T) { }, }) } + +func TestContentDispositonEncoding(t *testing.T) { + testContentDispositionEncoding(t, false, "£ẞǑǓÆ æ ♫♬", + "attachment; filename=\"%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC\"") + testContentDispositionEncoding(t, true, "£ẞǑǓÆ æ ♫♬", + "form-data; name=\"file\"; filename=\"%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC\"") +} + +func testContentDispositionEncoding(t *testing.T, form bool, filename string, expected string) { + sf := NewMapDirectory(map[string]Node{"": NewBytesFile([]byte(""))}) + mfr := NewMultiFileReader(sf, form, false) + if _, err := mfr.Read(nil); err != nil { + t.Fatal("MultiFileReader.Read failed") + } + + header := make(textproto.MIMEHeader) + mfr.addContentDisposition(header, filename) + v := header.Get(contentDispositionHeader) + if v != expected { + t.Fatalf("content-disposition did not match:\nExpected: %s\nActual : %s", expected, v) + } +} diff --git a/files/multipartfile.go b/files/multipartfile.go index b5aab9620..3f0a5b3fa 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -5,8 +5,12 @@ import ( "mime" "mime/multipart" "net/url" + "os" "path" + "path/filepath" + "strconv" "strings" + "time" ) const ( @@ -17,17 +21,48 @@ const ( applicationSymlink = "application/symlink" applicationFile = "application/octet-stream" - contentTypeHeader = "Content-Type" + contentTypeHeader = "Content-Type" + contentDispositionHeader = "Content-Disposition" ) +// multiPartFileInfo implements the `fs.FileInfo` interface for a file or +// directory received in a `multipart.part`. +type multiPartFileInfo struct { + name string + mode os.FileMode + mtime time.Time +} + +func (fi *multiPartFileInfo) Name() string { return fi.name } +func (fi *multiPartFileInfo) Mode() os.FileMode { return fi.mode } +func (fi *multiPartFileInfo) ModTime() time.Time { return fi.mtime } +func (fi *multiPartFileInfo) IsDir() bool { return fi.mode.IsDir() } +func (fi *multiPartFileInfo) Sys() interface{} { return nil } +func (fi *multiPartFileInfo) Size() int64 { panic("size for multipart file info is not supported") } + type multipartDirectory struct { path string walker *multipartWalker + stat os.FileInfo // part is the part describing the directory. It's nil when implicit. part *multipart.Part } +func (f *multipartDirectory) Mode() os.FileMode { + if f.stat == nil { + return 0 + } + return f.stat.Mode() +} + +func (f *multipartDirectory) ModTime() time.Time { + if f.stat == nil { + return time.Time{} + } + return f.stat.ModTime() +} + type multipartWalker struct { part *multipart.Part reader *multipart.Reader @@ -85,12 +120,15 @@ func (w *multipartWalker) nextFile() (Node, error) { } } + name := fileName(part) + switch contentType { case multipartFormdataType, applicationDirectory: return &multipartDirectory{ part: part, - path: fileName(part), + path: name, walker: w, + stat: fileInfo(name, part), }, nil case applicationSymlink: out, err := io.ReadAll(part) @@ -98,7 +136,7 @@ func (w *multipartWalker) nextFile() (Node, error) { return nil, err } - return NewLinkFile(string(out), nil), nil + return NewLinkFile(string(out), fileInfo(name, part)), nil default: var absPath string if absPathEncoded := part.Header.Get("abspath-encoded"); absPathEncoded != "" { @@ -113,6 +151,7 @@ func (w *multipartWalker) nextFile() (Node, error) { return &ReaderFile{ reader: part, abspath: absPath, + stat: fileInfo(name, part), }, nil } } @@ -169,6 +208,44 @@ func (it *multipartIterator) Node() Node { return it.curFile } +// fileInfo constructs an `os.FileInfo` from a `multipart.part` serving +// a file or directory. +func fileInfo(name string, part *multipart.Part) os.FileInfo { + fi := multiPartFileInfo{name: filepath.Base(name)} + formName := part.FormName() + + i := strings.IndexByte(formName, '?') + if i == -1 { + return &fi + } + + params, err := url.ParseQuery(formName[i+1:]) + if err != nil { + return nil + } + + if v := params["mode"]; v != nil { + mode, err := strconv.ParseUint(v[0], 8, 32) + if err == nil { + fi.mode = os.FileMode(mode) + } + } + + var secs, nsecs int64 + if v := params["mtime"]; v != nil { + secs, err = strconv.ParseInt(v[0], 10, 64) + if err != nil { + return &fi + } + } + if v := params["mtime-nsecs"]; v != nil { + nsecs, _ = strconv.ParseInt(v[0], 10, 64) + } + fi.mtime = time.Unix(secs, nsecs) + + return &fi +} + func (it *multipartIterator) Next() bool { if it.f.walker.reader == nil || it.err != nil { return false @@ -206,9 +283,9 @@ func (it *multipartIterator) Next() bool { } return true } - it.curName = name // Finally, advance to the next file. + it.curName = name it.curFile, it.err = it.f.walker.nextFile() return it.err == nil diff --git a/files/readerfile.go b/files/readerfile.go index bf3fa1c9e..8b9e4069c 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "time" ) // ReaderFile is a implementation of File created from an `io.Reader`. @@ -13,8 +14,21 @@ type ReaderFile struct { abspath string reader io.ReadCloser stat os.FileInfo + fsize int64 +} - fsize int64 +func (f *ReaderFile) Mode() os.FileMode { + if f.stat == nil { + return 0 + } + return f.stat.Mode() +} + +func (f *ReaderFile) ModTime() time.Time { + if f.stat == nil { + return time.Time{} + } + return f.stat.ModTime() } func NewBytesFile(b []byte) File { @@ -32,6 +46,10 @@ func (b bytesReaderCloser) Close() error { return nil } +func NewBytesStatFile(b []byte, stat os.FileInfo) File { + return NewReaderStatFile(bytes.NewReader(b), stat) +} + func NewReaderFile(reader io.Reader) File { return NewReaderStatFile(reader, nil) } diff --git a/files/serialfile.go b/files/serialfile.go index bd25bd1bc..cf4d44be3 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "time" ) // serialFile implements Node, and reads from a path on the OS filesystem. @@ -164,6 +165,14 @@ func (f *serialFile) Size() (int64, error) { return du, err } +func (f *serialFile) Mode() os.FileMode { + return f.stat.Mode() +} + +func (f *serialFile) ModTime() time.Time { + return f.stat.ModTime() +} + var ( _ Directory = &serialFile{} _ DirIterator = &serialIterator{} diff --git a/files/slicedirectory.go b/files/slicedirectory.go index 9cf910c6a..7a444b65a 100644 --- a/files/slicedirectory.go +++ b/files/slicedirectory.go @@ -1,6 +1,11 @@ package files -import "sort" +import ( + "cmp" + "os" + "slices" + "time" +) type fileEntry struct { name string @@ -49,22 +54,51 @@ func (it *sliceIterator) Err() error { // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { files []DirEntry + stat os.FileInfo +} + +func (f *SliceFile) Mode() os.FileMode { + if f.stat != nil { + return f.stat.Mode() + } + return 0 +} + +func (f *SliceFile) ModTime() time.Time { + if f.stat != nil { + return f.stat.ModTime() + } + return time.Time{} } func NewMapDirectory(f map[string]Node) Directory { - ents := make([]DirEntry, 0, len(f)) + return NewSliceDirectory(sortDirEntries(f)) +} + +func NewMapStatDirectory(f map[string]Node, stat os.FileInfo) Directory { + return NewSliceStatDirectory(sortDirEntries(f), stat) +} + +func sortDirEntries(f map[string]Node) []DirEntry { + ents := make([]DirEntry, len(f)) + var i int for name, nd := range f { - ents = append(ents, FileEntry(name, nd)) + ents[i] = FileEntry(name, nd) + i++ } - sort.Slice(ents, func(i, j int) bool { - return ents[i].Name() < ents[j].Name() + slices.SortFunc(ents, func(a, b DirEntry) int { + return cmp.Compare(a.Name(), b.Name()) }) - return NewSliceDirectory(ents) + return ents } func NewSliceDirectory(files []DirEntry) Directory { - return &SliceFile{files} + return &SliceFile{files: files} +} + +func NewSliceStatDirectory(files []DirEntry, stat os.FileInfo) Directory { + return &SliceFile{files: files, stat: stat} } func (f *SliceFile) Entries() DirIterator { diff --git a/files/tarwriter.go b/files/tarwriter.go index e5d857116..c982abf63 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -10,23 +10,25 @@ import ( "time" ) -var ErrUnixFSPathOutsideRoot = errors.New("relative UnixFS paths outside the root are now allowed, use CAR instead") +var ErrUnixFSPathOutsideRoot = errors.New("relative UnixFS paths outside the root are not allowed, use CAR instead") type TarWriter struct { TarW *tar.Writer baseDirSet bool baseDir string + format tar.Format } // NewTarWriter wraps given io.Writer into a new tar writer func NewTarWriter(w io.Writer) (*TarWriter, error) { return &TarWriter{ - TarW: tar.NewWriter(w), + TarW: tar.NewWriter(w), + format: tar.FormatUnknown, }, nil } func (w *TarWriter) writeDir(f Directory, fpath string) error { - if err := writeDirHeader(w.TarW, fpath); err != nil { + if err := w.writeHeader(f, fpath, 0); err != nil { return err } @@ -45,7 +47,7 @@ func (w *TarWriter) writeFile(f File, fpath string) error { return err } - if err := writeFileHeader(w.TarW, fpath, uint64(size)); err != nil { + if err = w.writeHeader(f, fpath, size); err != nil { return err } @@ -76,7 +78,7 @@ func validateTarFilePath(baseDir, fpath string) bool { return true } -// WriteNode adds a node to the archive. +// WriteFile adds a node to the archive. func (w *TarWriter) WriteFile(nd Node, fpath string) error { if !w.baseDirSet { w.baseDirSet = true // Use a variable for this as baseDir may be an empty string. @@ -89,7 +91,7 @@ func (w *TarWriter) WriteFile(nd Node, fpath string) error { switch nd := nd.(type) { case *Symlink: - return writeSymlinkHeader(w.TarW, nd.Target, fpath) + return w.writeHeader(nd, fpath, 0) case File: return w.writeFile(nd, fpath) case Directory: @@ -104,32 +106,33 @@ func (w *TarWriter) Close() error { return w.TarW.Close() } -func writeDirHeader(w *tar.Writer, fpath string) error { - return w.WriteHeader(&tar.Header{ - Name: fpath, - Typeflag: tar.TypeDir, - Mode: 0o777, - ModTime: time.Now().Truncate(time.Second), - // TODO: set mode, dates, etc. when added to unixFS - }) -} +func (w *TarWriter) writeHeader(n Node, fpath string, size int64) error { + hdr := &tar.Header{ + Format: w.format, + Name: fpath, + Size: size, + Mode: int64(UnixPermsOrDefault(n)), + } + + switch nd := n.(type) { + case *Symlink: + hdr.Typeflag = tar.TypeSymlink + hdr.Linkname = nd.Target + case Directory: + hdr.Typeflag = tar.TypeDir + default: + hdr.Typeflag = tar.TypeReg + } + + if m := n.ModTime(); m.IsZero() { + hdr.ModTime = time.Now() + } else { + hdr.ModTime = m + } -func writeFileHeader(w *tar.Writer, fpath string, size uint64) error { - return w.WriteHeader(&tar.Header{ - Name: fpath, - Size: int64(size), - Typeflag: tar.TypeReg, - Mode: 0o644, - ModTime: time.Now().Truncate(time.Second), - // TODO: set mode, dates, etc. when added to unixFS - }) + return w.TarW.WriteHeader(hdr) } -func writeSymlinkHeader(w *tar.Writer, target, fpath string) error { - return w.WriteHeader(&tar.Header{ - Name: fpath, - Linkname: target, - Mode: 0o777, - Typeflag: tar.TypeSymlink, - }) +func (w *TarWriter) SetFormat(format tar.Format) { + w.format = format } diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index 0e1488e7f..559d77f3d 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -11,11 +11,13 @@ import ( func TestTarWriter(t *testing.T) { tf := NewMapDirectory(map[string]Node{ "file.txt": NewBytesFile([]byte(text)), - "boop": NewMapDirectory(map[string]Node{ + "boop": NewMapStatDirectory(map[string]Node{ "a.txt": NewBytesFile([]byte("bleep")), "b.txt": NewBytesFile([]byte("bloop")), - }), - "beep.txt": NewBytesFile([]byte("beep")), + }, &mockFileInfo{name: "", mode: 0750, mtime: time.Unix(6600000000, 0)}), + "beep.txt": NewBytesStatFile([]byte("beep"), + &mockFileInfo{name: "beep.txt", size: 4, mode: 0766, mtime: time.Unix(1604320500, 54321)}), + "boop-sl": NewSymlinkFile("boop", time.Unix(6600050000, 0)), }) pr, pw := io.Pipe() @@ -23,6 +25,7 @@ func TestTarWriter(t *testing.T) { if err != nil { t.Fatal(err) } + tw.SetFormat(tar.FormatPAX) tr := tar.NewReader(pr) go func() { @@ -33,8 +36,9 @@ func TestTarWriter(t *testing.T) { }() var cur *tar.Header + const delta = 4 * time.Second - checkHeader := func(name string, typ byte, size int64) { + checkHeader := func(name string, typ byte, size int64, mode int64, mtime time.Time) { if cur.Name != name { t.Errorf("got wrong name: %s != %s", cur.Name, name) } @@ -44,41 +48,52 @@ func TestTarWriter(t *testing.T) { if cur.Size != size { t.Errorf("got wrong size: %d != %d", cur.Size, size) } - now := time.Now() - if cur.ModTime.After(now) { - t.Errorf("wrote timestamp in the future: %s (now) < %s", now, cur.ModTime) + if cur.Mode != mode { + t.Errorf("got wrong mode: %d != %d", cur.Mode, mode) + } + if mtime.IsZero() { + interval := time.Since(cur.ModTime) + if interval < -delta || interval > delta { + t.Errorf("expected timestamp to be current: %s", cur.ModTime) + } + } else if cur.ModTime.UnixNano() != mtime.UnixNano() { + t.Errorf("got wrong timestamp: %s != %s", cur.ModTime, mtime) } } if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("", tar.TypeDir, 0) + checkHeader("", tar.TypeDir, 0, 0755, time.Time{}) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("beep.txt", tar.TypeReg, 4) + checkHeader("beep.txt", tar.TypeReg, 4, 0766, time.Unix(1604320500, 54321)) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("boop", tar.TypeDir, 0) + checkHeader("boop", tar.TypeDir, 0, 0750, time.Unix(6600000000, 0)) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("boop/a.txt", tar.TypeReg, 5) + checkHeader("boop/a.txt", tar.TypeReg, 5, 0644, time.Time{}) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("boop/b.txt", tar.TypeReg, 5) + checkHeader("boop/b.txt", tar.TypeReg, 5, 0644, time.Time{}) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("file.txt", tar.TypeReg, 13) + checkHeader("boop-sl", tar.TypeSymlink, 0, 0777, time.Unix(6600050000, 0)) + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("file.txt", tar.TypeReg, 13, 0644, time.Time{}) if cur, err = tr.Next(); err != io.EOF { t.Fatal(err) @@ -101,7 +116,7 @@ func TestTarWriterRelativePathInsideRoot(t *testing.T) { } defer tw.Close() - if err := tw.WriteFile(tf, ""); err != nil { + if err = tw.WriteFile(tf, ""); err != nil { t.Error(err) } } @@ -122,7 +137,7 @@ func TestTarWriterFailsFileOutsideRoot(t *testing.T) { } defer tw.Close() - if err := tw.WriteFile(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + if err = tw.WriteFile(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) } } @@ -143,7 +158,7 @@ func TestTarWriterFailsFileOutsideRootWithBaseDir(t *testing.T) { } defer tw.Close() - if err := tw.WriteFile(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + if err = tw.WriteFile(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) } } diff --git a/files/util.go b/files/util.go index e727e7ae6..1ac673b0e 100644 --- a/files/util.go +++ b/files/util.go @@ -1,5 +1,7 @@ package files +import "os" + // ToFile is an alias for n.(File). If the file isn't a regular file, nil value // will be returned func ToFile(n Node) File { @@ -23,3 +25,38 @@ func FileFromEntry(e DirEntry) File { func DirFromEntry(e DirEntry) Directory { return ToDir(e.Node()) } + +// UnixPermsOrDefault returns the unix style permissions stored for the given +// Node, or default unix permissions for the Node type. +func UnixPermsOrDefault(n Node) uint32 { + perms := ModePermsToUnixPerms(n.Mode()) + if perms != 0 { + return perms + } + + switch n.(type) { + case *Symlink: + return 0777 + case Directory: + return 0755 + default: + return 0644 + } +} + +// ModePermsToUnixPerms converts the permission bits of an os.FileMode to unix +// style mode permissions. +func ModePermsToUnixPerms(fileMode os.FileMode) uint32 { + return uint32((fileMode & 0xC00000 >> 12) | (fileMode & os.ModeSticky >> 11) | (fileMode & 0x1FF)) +} + +// UnixPermsToModePerms converts unix style mode permissions to os.FileMode +// permissions, as it only operates on permission bits it does not set the +// underlying type (fs.ModeDir, fs.ModeSymlink, etc.) in the returned +// os.FileMode. +func UnixPermsToModePerms(unixPerms uint32) os.FileMode { + if unixPerms == 0 { + return 0 + } + return os.FileMode((unixPerms & 0x1FF) | (unixPerms & 0xC00 << 12) | (unixPerms & 0x200 << 11)) +} diff --git a/files/util_test.go b/files/util_test.go new file mode 100644 index 000000000..396abec99 --- /dev/null +++ b/files/util_test.go @@ -0,0 +1,24 @@ +package files + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestModePermsToUnixPerms(t *testing.T) { + assert.Equal(t, uint32(0777), ModePermsToUnixPerms(os.FileMode(0777))) + assert.Equal(t, uint32(04755), ModePermsToUnixPerms(0755|os.ModeSetuid)) + assert.Equal(t, uint32(02777), ModePermsToUnixPerms(0777|os.ModeSetgid)) + assert.Equal(t, uint32(01377), ModePermsToUnixPerms(0377|os.ModeSticky)) + assert.Equal(t, uint32(05300), ModePermsToUnixPerms(0300|os.ModeSetuid|os.ModeSticky)) +} + +func TestUnixPermsToModePerms(t *testing.T) { + assert.Equal(t, os.FileMode(0777), UnixPermsToModePerms(0777)) + assert.Equal(t, 0755|os.ModeSetuid, UnixPermsToModePerms(04755)) + assert.Equal(t, 0777|os.ModeSetgid, UnixPermsToModePerms(02777)) + assert.Equal(t, 0377|os.ModeSticky, UnixPermsToModePerms(01377)) + assert.Equal(t, 0300|os.ModeSetuid|os.ModeSticky, UnixPermsToModePerms(05300)) +} diff --git a/files/webfile.go b/files/webfile.go index 4586eab63..8791fce8a 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -7,8 +7,16 @@ import ( "net/http" "net/url" "os" + "strconv" + "time" ) +// the HTTP Response header that provides the last modified timestamp +const LastModifiedHeaderName = "Last-Modified" + +// the HTTP Response header that provides the unix file mode +const FileModeHeaderName = "File-Mode" + // WebFile is an implementation of File which reads it // from a Web URL (http). A GET request will be performed // against the source when calling Read(). @@ -16,6 +24,16 @@ type WebFile struct { body io.ReadCloser url *url.URL contentLength int64 + mode os.FileMode + mtime time.Time +} + +func (wf *WebFile) Mode() os.FileMode { + return wf.mode +} + +func (wf *WebFile) ModTime() time.Time { + return wf.mtime } // NewWebFile creates a WebFile with the given URL, which @@ -38,10 +56,26 @@ func (wf *WebFile) start() error { } wf.body = resp.Body wf.contentLength = resp.ContentLength + wf.getResponseMetaData(resp) } return nil } +func (wf *WebFile) getResponseMetaData(resp *http.Response) { + ts := resp.Header.Get(LastModifiedHeaderName) + if ts != "" { + if mtime, err := time.Parse(time.RFC1123, ts); err == nil { + wf.mtime = mtime + } + } + md := resp.Header.Get(FileModeHeaderName) + if md != "" { + if mode, err := strconv.ParseInt(md, 8, 32); err == nil { + wf.mode = os.FileMode(mode) + } + } +} + // Read reads the File from it's web location. On the first // call to Read, a GET request will be performed against the // WebFile's URL, using Go's default HTTP client. Any further diff --git a/files/webfile_test.go b/files/webfile_test.go index 94cddb5d2..b2a7238ab 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -6,12 +6,19 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "testing" + "time" ) func TestWebFile(t *testing.T) { const content = "Hello world!" + const mode = 0644 + mtime := time.Unix(16043205005, 0) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add(LastModifiedHeaderName, mtime.Format(time.RFC1123)) + w.Header().Add(FileModeHeaderName, strconv.FormatUint(uint64(mode), 8)) fmt.Fprint(w, content) })) defer s.Close() @@ -28,6 +35,12 @@ func TestWebFile(t *testing.T) { if string(body) != content { t.Fatalf("expected %q but got %q", content, string(body)) } + if actual := wf.Mode(); actual != mode { + t.Fatalf("expected file mode %q but got 0%q", mode, strconv.FormatUint(uint64(actual), 8)) + } + if actual := wf.ModTime(); !actual.Equal(mtime) { + t.Fatalf("expected last modified time %q but got %q", mtime, actual) + } } func TestWebFile_notFound(t *testing.T) { diff --git a/gateway/backend_car_files.go b/gateway/backend_car_files.go index c384bbe2c..50c298a38 100644 --- a/gateway/backend_car_files.go +++ b/gateway/backend_car_files.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "io" + "os" + "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipld/unixfs" @@ -50,6 +52,14 @@ func (b *backpressuredFile) Close() error { return nil } +func (b *backpressuredFile) Mode() os.FileMode { + panic("not implemented") +} + +func (b *backpressuredFile) ModTime() time.Time { + panic("not implemented") +} + func (b *backpressuredFile) Size() (int64, error) { return b.size, nil } @@ -126,6 +136,14 @@ func (b *singleUseDirectory) Close() error { return nil } +func (b *singleUseDirectory) Mode() os.FileMode { + return 0 +} + +func (b *singleUseDirectory) ModTime() time.Time { + return time.Time{} +} + func (b *singleUseDirectory) Size() (int64, error) { //TODO implement me panic("implement me") diff --git a/go.mod b/go.mod index 3ccf62b41..a625a8b81 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/libp2p/go-buffer-pool v0.1.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.36.1 + github.com/libp2p/go-libp2p v0.36.2 github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.3 @@ -147,7 +147,7 @@ require ( github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.8 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/ice/v2 v2.3.32 // indirect + github.com/pion/ice/v2 v2.3.34 // indirect github.com/pion/interceptor v0.1.29 // indirect github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.12 // indirect @@ -158,9 +158,9 @@ require ( github.com/pion/sdp/v3 v3.0.9 // indirect github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/transport/v2 v2.2.9 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/webrtc/v3 v3.2.50 // indirect + github.com/pion/webrtc/v3 v3.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect @@ -195,5 +195,3 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect ) - -replace github.com/libp2p/go-libp2p => github.com/libp2p/go-libp2p v0.35.1-0.20240804142423-e2e0d2917f55 diff --git a/go.sum b/go.sum index 354e5378c..e539af643 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+ github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.35.1-0.20240804142423-e2e0d2917f55 h1:/iBsYYCzlVCiMMUfXWiHzgWpTFzZwes3cTlamdzXv6g= -github.com/libp2p/go-libp2p v0.35.1-0.20240804142423-e2e0d2917f55/go.mod h1:mdtNGqy0AQuiYJuO1bXPdFOyFeyMTMSVZ03OBi/XLS4= +github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U= +github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= @@ -375,8 +375,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI= -github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU= +github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM= +github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -402,17 +402,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/ github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= -github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE= -github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw= github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34= -github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo= +github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= +github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/ipld/unixfs/file/unixfile.go b/ipld/unixfs/file/unixfile.go index 5ef968d1b..0cf7616c1 100644 --- a/ipld/unixfs/file/unixfile.go +++ b/ipld/unixfs/file/unixfile.go @@ -3,6 +3,8 @@ package unixfile import ( "context" "errors" + "os" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" uio "github.com/ipfs/boxo/ipld/unixfs/io" @@ -21,6 +23,8 @@ type ufsDirectory struct { dserv ipld.DAGService dir uio.Directory size int64 + mode os.FileMode + mtime time.Time } type ufsIterator struct { @@ -118,6 +122,14 @@ func (d *ufsDirectory) Entries() files.DirIterator { } } +func (d *ufsDirectory) Mode() os.FileMode { + return d.mode +} + +func (d *ufsDirectory) ModTime() time.Time { + return d.mtime +} + func (d *ufsDirectory) Size() (int64, error) { return d.size, nil } @@ -126,6 +138,14 @@ type ufsFile struct { uio.DagReader } +func (f *ufsFile) Mode() os.FileMode { + return f.DagReader.Mode() +} + +func (f *ufsFile) ModTime() time.Time { + return f.DagReader.ModTime() +} + func (f *ufsFile) Size() (int64, error) { return int64(f.DagReader.Size()), nil } @@ -141,12 +161,19 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) return nil, err } + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + return &ufsDirectory{ ctx: ctx, dserv: dserv, - dir: dir, - size: int64(size), + dir: dir, + size: int64(size), + mode: fsn.Mode(), + mtime: fsn.ModTime(), }, nil } @@ -157,11 +184,12 @@ func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (fi if err != nil { return nil, err } + if fsn.IsDir() { return newUnixfsDir(ctx, dserv, dn) } if fsn.Type() == ft.TSymlink { - return files.NewLinkFile(string(fsn.Data()), nil), nil + return files.NewSymlinkFile(string(fsn.Data()), fsn.ModTime()), nil } case *dag.RawNode: diff --git a/ipld/unixfs/importer/balanced/balanced_test.go b/ipld/unixfs/importer/balanced/balanced_test.go index 5a5dcf9ad..4ea4cb8a9 100644 --- a/ipld/unixfs/importer/balanced/balanced_test.go +++ b/ipld/unixfs/importer/balanced/balanced_test.go @@ -7,6 +7,7 @@ import ( "io" mrand "math/rand" "testing" + "time" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" uio "github.com/ipfs/boxo/ipld/unixfs/io" @@ -26,6 +27,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, err Maxlinks: h.DefaultLinksPerBlock, } + return buildTestDagWithParams(spl, dbp) +} + +func buildTestDagWithParams(spl chunker.Splitter, dbp h.DagBuilderParams) (*dag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -335,3 +340,46 @@ func TestSeekingConsistency(t *testing.T) { t.Fatal(err) } } + +func TestMetadataNoData(t *testing.T) { + testMetadata(t, new(bytes.Buffer)) +} + +func TestMetadata(t *testing.T) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + _, err := io.CopyN(buf, random.NewRand(), nbytes) + if err != nil { + t.Fatal(err) + } + + testMetadata(t, buf) +} + +func testMetadata(t *testing.T, buf *bytes.Buffer) { + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + FileMode: 0522, + FileModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.FileModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime) + } + + if dr.Mode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode) + } +} diff --git a/ipld/unixfs/importer/balanced/builder.go b/ipld/unixfs/importer/balanced/builder.go index 0fdb0fd28..915d0a439 100644 --- a/ipld/unixfs/importer/balanced/builder.go +++ b/ipld/unixfs/importer/balanced/builder.go @@ -130,18 +130,33 @@ import ( // | Chunk 1 | | Chunk 2 | | Chunk 3 | // +=========+ +=========+ + - - - - + func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { + var root ipld.Node + var err error + if db.Done() { - // No data, return just an empty node. - root, err := db.NewLeafNode(nil, ft.TFile) - if err != nil { - return nil, err - } + // No data, just create an empty node. + root, err = db.NewLeafNode(nil, ft.TFile) // This works without Filestore support (`ProcessFileStore`). // TODO: Why? Is there a test case missing? + } else { + root, err = layoutData(db) + } + + if err != nil { + return nil, err + } - return root, db.Add(root) + if db.HasFileAttributes() { + err = db.SetFileAttributes(root) + if err != nil { + return nil, err + } } + return root, db.Add(root) +} + +func layoutData(db *h.DagBuilderHelper) (ipld.Node, error) { // The first `root` will be a single leaf node with data // (corner case), after that subsequent `root` nodes will // always be internal nodes (with a depth > 0) that can @@ -172,7 +187,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { } } - return root, db.Add(root) + return root, nil } // fillNodeRec will "fill" the given internal (non-leaf) `node` with data by diff --git a/ipld/unixfs/importer/helpers/dagbuilder.go b/ipld/unixfs/importer/helpers/dagbuilder.go index 25514d795..aefffad15 100644 --- a/ipld/unixfs/importer/helpers/dagbuilder.go +++ b/ipld/unixfs/importer/helpers/dagbuilder.go @@ -5,6 +5,7 @@ import ( "errors" "io" "os" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" @@ -23,13 +24,15 @@ var ErrMissingFsRef = errors.New("missing file path or URL, can't create filesto // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { - dserv ipld.DAGService - spl chunker.Splitter - recvdErr error - rawLeaves bool - nextData []byte // the next item to return. - maxlinks int - cidBuilder cid.Builder + dserv ipld.DAGService + spl chunker.Splitter + recvdErr error + rawLeaves bool + nextData []byte // the next item to return. + maxlinks int + cidBuilder cid.Builder + fileMode os.FileMode + fileModTime time.Time // Filestore support variables. // ---------------------------- @@ -62,6 +65,12 @@ type DagBuilderParams struct { // DAGService to write blocks to (required) Dagserv ipld.DAGService + // The unixfs file mode + FileMode os.FileMode + + // The unixfs last modified time + FileModTime time.Time + // NoCopy signals to the chunker that it should track fileinfo for // filestore adds NoCopy bool @@ -71,11 +80,13 @@ type DagBuilderParams struct { // chunker.Splitter as data source. func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) { db := &DagBuilderHelper{ - dserv: dbp.Dagserv, - spl: spl, - rawLeaves: dbp.RawLeaves, - cidBuilder: dbp.CidBuilder, - maxlinks: dbp.Maxlinks, + dserv: dbp.Dagserv, + spl: spl, + rawLeaves: dbp.RawLeaves, + cidBuilder: dbp.CidBuilder, + maxlinks: dbp.Maxlinks, + fileMode: dbp.FileMode, + fileModTime: dbp.FileModTime, } if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { db.fullPath = fi.AbsPath() @@ -138,9 +149,9 @@ func (db *DagBuilderHelper) GetCidBuilder() cid.Builder { return db.cidBuilder } -// NewLeafNode creates a leaf node filled with data. If rawLeaves is -// defined then a raw leaf will be returned. Otherwise, it will create -// and return `FSNodeOverDag` with `fsNodeType`. +// NewLeafNode creates a leaf node filled with data. If rawLeaves is defined +// then a raw leaf will be returned. Otherwise, it will create and return +// `FSNodeOverDag` with `fsNodeType`. func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) { if len(data) > BlockSizeLimit { return nil, ErrSizeLimitExceeded @@ -161,6 +172,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType // Encapsulate the data in UnixFS node (instead of a raw node). fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType) fsNodeOverDag.SetFileData(data) + node, err := fsNodeOverDag.Commit() if err != nil { return nil, err @@ -172,9 +184,10 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType return node, nil } -// FillNodeLayer will add datanodes as children to the give node until +// FillNodeLayer will add data-nodes as children to the given node until // it is full in this layer or no more data. -// NOTE: This function creates raw data nodes so it only works +// +// NOTE: This function creates raw data nodes, so it only works // for the `trickle.Layout`. func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error { // while we have room AND we're not done @@ -265,6 +278,34 @@ func (db *DagBuilderHelper) Maxlinks() int { return db.maxlinks } +// HasFileAttributes will return false if Filestore is being used, +// otherwise returns true if a file mode or last modification time is set. +func (db *DagBuilderHelper) HasFileAttributes() bool { + return db.fullPath == "" && (db.fileMode != 0 || !db.fileModTime.IsZero()) +} + +// SetFileAttributes stores file attributes present in the `DagBuilderHelper` +// into the associated `ft.FSNode`. +func (db *DagBuilderHelper) SetFileAttributes(n ipld.Node) error { + if pn, ok := n.(*dag.ProtoNode); ok { + fsn, err := ft.FSNodeFromBytes(pn.Data()) + if err != nil { + return err + } + fsn.SetModTime(db.fileModTime) + fsn.SetMode(db.fileMode) + + d, err := fsn.GetBytes() + if err != nil { + return err + } + + pn.SetData(d) + } + + return nil +} + // FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a // `dag.ProtoNode`. Instead of just having a single `ipld.Node` that // would need to be constantly (un)packed to access and modify its @@ -288,7 +329,7 @@ type FSNodeOverDag struct { } // NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode` -// decoupled from one onther (and will continue in that way until +// decoupled from one anonther (and will continue in that way until // `Commit` is called), with `fsNodeType` specifying the type of // the UnixFS layer node (either `File` or `Raw`). func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag { @@ -374,6 +415,26 @@ func (n *FSNodeOverDag) SetFileData(fileData []byte) { n.file.SetData(fileData) } +// SetMode sets the file mode of the associated `ft.FSNode`. +func (n *FSNodeOverDag) SetMode(mode os.FileMode) { + n.file.SetMode(mode) +} + +// SetModTime sets the file modification time of the associated `ft.FSNode`. +func (n *FSNodeOverDag) SetModTime(ts time.Time) { + n.file.SetModTime(ts) +} + +// Mode returns the file mode of the associated `ft.FSNode` +func (n *FSNodeOverDag) Mode() os.FileMode { + return n.file.Mode() +} + +// ModTime returns the last modification time of the associated `ft.FSNode` +func (n *FSNodeOverDag) ModTime() time.Time { + return n.file.ModTime() +} + // GetDagNode fills out the proper formatting for the FSNodeOverDag node // inside of a DAG node and returns the dag node. // TODO: Check if we have committed (passed the UnixFS information diff --git a/ipld/unixfs/importer/trickle/trickle_test.go b/ipld/unixfs/importer/trickle/trickle_test.go index 9078fdc02..d495fd208 100644 --- a/ipld/unixfs/importer/trickle/trickle_test.go +++ b/ipld/unixfs/importer/trickle/trickle_test.go @@ -6,7 +6,9 @@ import ( "fmt" "io" mrand "math/rand" + "runtime" "testing" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -40,6 +42,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav RawLeaves: bool(rawLeaves), } + return buildTestDagWithParams(ds, spl, dbp) +} + +func buildTestDagWithParams(ds ipld.DAGService, spl chunker.Splitter, dbp h.DagBuilderParams) (*merkledag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -59,7 +65,7 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav Getter: ds, Direct: dbp.Maxlinks, LayerRepeat: depthRepeat, - RawLeaves: bool(rawLeaves), + RawLeaves: dbp.RawLeaves, }) } @@ -668,3 +674,111 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { t.Fatal(err) } } + +func TestAppendWithModTime(t *testing.T) { + const nbytes = 128 * 1024 + + timestamp := time.Now() + buf := random.Bytes(nbytes) + + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBDataWithStat(buf[:nbytes/2], nbytes/2, 0, timestamp)) + + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + + dbp := &h.DagBuilderParams{ + Dagserv: mdtest.Mock(), + Maxlinks: h.DefaultLinksPerBlock, + } + + r := bytes.NewReader(buf[nbytes/2:]) + db, err := dbp.New(chunker.NewSizeSplitter(r, 500)) + if err != nil { + t.Fatal(err) + } + + nd2, err := Append(context.Background(), nd, db) + if err != nil { + t.Fatal(err) + } + + fsn, _ := ft.ExtractFSNode(nd2) + + if !fsn.ModTime().After(timestamp) { + t.Errorf("expected modification time to be updated") + } + +} + +func TestAppendToEmptyWithModTime(t *testing.T) { + timestamp := time.Now() + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBDataWithStat(nil, 0, 0, timestamp)) + + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + + dbp := &h.DagBuilderParams{ + Dagserv: mdtest.Mock(), + Maxlinks: h.DefaultLinksPerBlock, + } + + db, err := dbp.New(chunker.DefaultSplitter(bytes.NewReader([]byte("test")))) + if err != nil { + t.Fatal(err) + } + + nd2, err := Append(context.Background(), nd, db) + if err != nil { + t.Fatal(err) + } + + fsn, _ := ft.ExtractFSNode(nd2) + + if !fsn.ModTime().After(timestamp) { + t.Errorf("expected modification time to be updated") + } +} + +func TestMetadata(t *testing.T) { + runBothSubtests(t, testMetadata) +} + +func testMetadata(t *testing.T, rawLeaves UseRawLeaves) { + const nbytes = 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + _, err := io.CopyN(buf, random.NewRand(), nbytes) + if err != nil { + t.Fatal(err) + } + + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + FileMode: 0522, + FileModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.FileModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime) + } + + if dr.Mode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode) + } +} diff --git a/ipld/unixfs/importer/trickle/trickledag.go b/ipld/unixfs/importer/trickle/trickledag.go index 09a8b8672..2b9d31dfa 100644 --- a/ipld/unixfs/importer/trickle/trickledag.go +++ b/ipld/unixfs/importer/trickle/trickledag.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -43,6 +44,13 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { return nil, err } + if db.HasFileAttributes() { + err = db.SetFileAttributes(root) + } + + if err != nil { + return nil, err + } return root, db.Add(root) } @@ -94,7 +102,6 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } // Convert to unixfs node for working with easily - fsn, err := h.NewFSNFromDag(base) if err != nil { return nil, err @@ -109,9 +116,10 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } if db.Done() { - // TODO: If `FillNodeLayer` stop `Commit`ing this should be - // the place (besides the function end) to call it. - return fsn.GetDagNode() + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } + return fsn.Commit() } // If continuing, our depth has increased by one @@ -142,11 +150,7 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } } } - _, err = fsn.Commit() - if err != nil { - return nil, err - } - return fsn.GetDagNode() + return fsn.Commit() } func appendFillLastChild(ctx context.Context, fsn *h.FSNodeOverDag, depth int, repeatNumber int, db *h.DagBuilderHelper) error { diff --git a/ipld/unixfs/io/dagreader.go b/ipld/unixfs/io/dagreader.go index 77dc8d921..bb1c83800 100644 --- a/ipld/unixfs/io/dagreader.go +++ b/ipld/unixfs/io/dagreader.go @@ -5,6 +5,8 @@ import ( "context" "errors" "io" + "os" + "time" mdag "github.com/ipfs/boxo/ipld/merkledag" unixfs "github.com/ipfs/boxo/ipld/unixfs" @@ -29,6 +31,8 @@ var ( type DagReader interface { ReadSeekCloser Size() uint64 + Mode() os.FileMode + ModTime() time.Time CtxReadFull(context.Context, []byte) (int, error) } @@ -44,6 +48,8 @@ type ReadSeekCloser interface { // the given node, using the passed in DAGService for data retrieval. func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { var size uint64 + var mode os.FileMode + var modTime time.Time switch n := n.(type) { case *mdag.RawNode: @@ -55,6 +61,9 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe return nil, err } + mode = fsNode.Mode() + modTime = fsNode.ModTime() + switch fsNode.Type() { case unixfs.TFile, unixfs.TRaw: size = fsNode.FileSize() @@ -93,6 +102,8 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe cancel: cancel, serv: serv, size: size, + mode: mode, + modTime: modTime, rootNode: n, dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)), }, nil @@ -129,7 +140,19 @@ type dagReader struct { // Passed to the `dagWalker` that will use it to request nodes. // TODO: Revisit name. - serv ipld.NodeGetter + serv ipld.NodeGetter + mode os.FileMode + modTime time.Time +} + +// Mode returns the UnixFS file mode or 0 if not set. +func (dr *dagReader) Mode() os.FileMode { + return dr.mode +} + +// ModTime returns the UnixFS file last modification time if set. +func (dr *dagReader) ModTime() time.Time { + return dr.modTime } // Size returns the total size of the data from the DAG structured file. diff --git a/ipld/unixfs/mod/dagmodifier.go b/ipld/unixfs/mod/dagmodifier.go index f662a0a71..c075523f8 100644 --- a/ipld/unixfs/mod/dagmodifier.go +++ b/ipld/unixfs/mod/dagmodifier.go @@ -7,6 +7,7 @@ import ( "context" "errors" "io" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" help "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -258,6 +259,9 @@ func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) { } // Update newly written node.. + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } b, err := fsn.GetBytes() if err != nil { return cid.Cid{}, err @@ -527,8 +531,17 @@ func (dm *DagModifier) dagTruncate(ctx context.Context, n ipld.Node, size uint64 if err != nil { return nil, err } - nd.SetData(ft.WrapData(fsn.Data()[:size])) - return nd, nil + + fsn.SetData(fsn.Data()[:size]) + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } + data, err := fsn.GetBytes() + if err != nil { + return nil, err + } + + return mdag.NodeWithData(data), nil case *mdag.RawNode: return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix()) } diff --git a/ipld/unixfs/pb/unixfs.pb.go b/ipld/unixfs/pb/unixfs.pb.go index 805c11289..d02e110f2 100644 --- a/ipld/unixfs/pb/unixfs.pb.go +++ b/ipld/unixfs/pb/unixfs.pb.go @@ -82,6 +82,8 @@ type Data struct { Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"` Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"` + Mode *uint32 `protobuf:"varint,7,opt,name=mode" json:"mode,omitempty"` + Mtime *IPFSTimestamp `protobuf:"bytes,8,opt,name=mtime" json:"mtime,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -158,6 +160,20 @@ func (m *Data) GetFanout() uint64 { return 0 } +func (m *Data) GetMode() uint32 { + if m != nil && m.Mode != nil { + return *m.Mode + } + return 0 +} + +func (m *Data) GetMtime() *IPFSTimestamp { + if m != nil { + return m.Mtime + } + return nil +} + type Metadata struct { MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -201,31 +217,91 @@ func (m *Metadata) GetMimeType() string { return "" } +// mostly copied from proto 3 - with int32 nanos changed to fixed32 for js-ipfs compatibility +// https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto +type IPFSTimestamp struct { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + Seconds *int64 `protobuf:"varint,1,req,name=seconds" json:"seconds,omitempty"` + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + Nanos *uint32 `protobuf:"fixed32,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IPFSTimestamp) Reset() { *m = IPFSTimestamp{} } +func (m *IPFSTimestamp) String() string { return proto.CompactTextString(m) } +func (*IPFSTimestamp) ProtoMessage() {} +func (*IPFSTimestamp) Descriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{2} +} +func (m *IPFSTimestamp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IPFSTimestamp.Unmarshal(m, b) +} +func (m *IPFSTimestamp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IPFSTimestamp.Marshal(b, m, deterministic) +} +func (m *IPFSTimestamp) XXX_Merge(src proto.Message) { + xxx_messageInfo_IPFSTimestamp.Merge(m, src) +} +func (m *IPFSTimestamp) XXX_Size() int { + return xxx_messageInfo_IPFSTimestamp.Size(m) +} +func (m *IPFSTimestamp) XXX_DiscardUnknown() { + xxx_messageInfo_IPFSTimestamp.DiscardUnknown(m) +} + +var xxx_messageInfo_IPFSTimestamp proto.InternalMessageInfo + +func (m *IPFSTimestamp) GetSeconds() int64 { + if m != nil && m.Seconds != nil { + return *m.Seconds + } + return 0 +} + +func (m *IPFSTimestamp) GetNanos() uint32 { + if m != nil && m.Nanos != nil { + return *m.Nanos + } + return 0 +} + func init() { proto.RegisterEnum("unixfs.v1.pb.Data_DataType", Data_DataType_name, Data_DataType_value) proto.RegisterType((*Data)(nil), "unixfs.v1.pb.Data") proto.RegisterType((*Metadata)(nil), "unixfs.v1.pb.Metadata") + proto.RegisterType((*IPFSTimestamp)(nil), "unixfs.pb.IPFSTimestamp") } func init() { proto.RegisterFile("unixfs.proto", fileDescriptor_e2fd76cc44dfc7c3) } var fileDescriptor_e2fd76cc44dfc7c3 = []byte{ - // 267 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0x41, 0x4f, 0x83, 0x30, - 0x18, 0x86, 0x05, 0xba, 0x0d, 0xbe, 0xa1, 0x69, 0xbe, 0x83, 0x21, 0x9a, 0x18, 0xc2, 0xc1, 0x70, - 0xc2, 0xe8, 0x3f, 0xd0, 0x2c, 0xc6, 0x0b, 0x97, 0x6e, 0xf1, 0xe0, 0xc5, 0x94, 0xad, 0x84, 0x66, - 0x8c, 0x12, 0xe8, 0x54, 0xfc, 0x1b, 0xfe, 0x61, 0x53, 0x18, 0xdb, 0x2e, 0x4d, 0x9e, 0xf6, 0x79, - 0x9b, 0x37, 0x2f, 0xf8, 0xfb, 0x4a, 0xfe, 0xe4, 0x6d, 0x52, 0x37, 0x4a, 0x2b, 0x1c, 0xe9, 0xeb, - 0x31, 0xa9, 0xb3, 0xe8, 0xcf, 0x06, 0xb2, 0xe0, 0x9a, 0xe3, 0x03, 0x90, 0x55, 0x57, 0x8b, 0xc0, - 0x0a, 0xed, 0xf8, 0xea, 0xe9, 0x36, 0x39, 0xb7, 0x12, 0x63, 0xf4, 0x87, 0x51, 0x58, 0x2f, 0x22, - 0x0e, 0xc1, 0xc0, 0x0e, 0xad, 0xd8, 0x67, 0xc3, 0x27, 0x37, 0xe0, 0xe6, 0xb2, 0x14, 0xad, 0xfc, - 0x15, 0x81, 0x13, 0x5a, 0x31, 0x61, 0x47, 0xc6, 0x3b, 0x80, 0xac, 0x54, 0xeb, 0xad, 0x81, 0x36, - 0x20, 0xa1, 0x13, 0x13, 0x76, 0x76, 0x63, 0xb2, 0x05, 0x6f, 0x8b, 0xbe, 0xc4, 0x64, 0xc8, 0x8e, - 0x8c, 0xd7, 0x30, 0xcd, 0x79, 0xa5, 0xf6, 0x3a, 0x98, 0xf6, 0x2f, 0x07, 0x8a, 0xde, 0xc1, 0x1d, - 0x5b, 0xe1, 0x0c, 0x1c, 0xc6, 0xbf, 0xe9, 0x05, 0x5e, 0x82, 0xb7, 0x90, 0x8d, 0x58, 0x6b, 0xd5, - 0x74, 0xd4, 0x42, 0x17, 0xc8, 0xab, 0x2c, 0x05, 0xb5, 0xd1, 0x07, 0x37, 0x15, 0x9a, 0x6f, 0xb8, - 0xe6, 0xd4, 0xc1, 0x39, 0xcc, 0x96, 0xdd, 0xae, 0x94, 0xd5, 0x96, 0x12, 0x93, 0x79, 0x7b, 0x4e, - 0x57, 0xcb, 0x82, 0x37, 0x1b, 0x3a, 0x89, 0xee, 0x4f, 0xa6, 0xe9, 0x95, 0xca, 0x9d, 0x38, 0x8c, - 0x63, 0xc5, 0x1e, 0x3b, 0xf2, 0xcb, 0xfc, 0xc3, 0x1b, 0x76, 0xfa, 0xac, 0xb3, 0xff, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xbd, 0x16, 0xf8, 0x45, 0x67, 0x01, 0x00, 0x00, + // 322 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x5f, 0x4b, 0xc3, 0x30, + 0x14, 0xc5, 0xed, 0xbf, 0xb5, 0xbb, 0xdb, 0xa4, 0x5c, 0x44, 0x82, 0x0f, 0x52, 0xfa, 0x20, 0x7d, + 0x90, 0x3e, 0xf8, 0x05, 0x44, 0x18, 0x43, 0x1f, 0x06, 0x92, 0x0d, 0xdf, 0xb3, 0x35, 0x63, 0x61, + 0x4d, 0x33, 0x9a, 0x0c, 0x9d, 0x9f, 0xd3, 0x0f, 0x24, 0x49, 0xd7, 0xe9, 0x5e, 0x4a, 0x7f, 0xb9, + 0xe7, 0x84, 0x73, 0x6e, 0x60, 0x7c, 0x68, 0xc4, 0xd7, 0x46, 0x97, 0xfb, 0x56, 0x19, 0x85, 0xc3, + 0x9e, 0x56, 0xf9, 0x8f, 0x0f, 0xe1, 0x94, 0x19, 0x86, 0x8f, 0x10, 0x2e, 0x8f, 0x7b, 0x4e, 0xbc, + 0xcc, 0x2f, 0xae, 0x9f, 0x48, 0x79, 0x96, 0x94, 0x76, 0xec, 0x3e, 0x76, 0x4e, 0x9d, 0x0a, 0xb1, + 0x73, 0x11, 0x3f, 0xf3, 0x8a, 0x31, 0xed, 0x6e, 0xb8, 0x83, 0x64, 0x23, 0x6a, 0xae, 0xc5, 0x37, + 0x27, 0x41, 0xe6, 0x15, 0x21, 0x3d, 0x33, 0xde, 0x03, 0xac, 0x6a, 0xb5, 0xde, 0x59, 0xd0, 0x24, + 0xcc, 0x82, 0x22, 0xa4, 0xff, 0x4e, 0xac, 0x77, 0xcb, 0xf4, 0xd6, 0x25, 0x88, 0x3a, 0x6f, 0xcf, + 0x78, 0x0b, 0x83, 0x0d, 0x6b, 0xd4, 0xc1, 0x90, 0x81, 0x9b, 0x9c, 0xc8, 0x66, 0x90, 0xaa, 0xe2, + 0x24, 0xce, 0xbc, 0x62, 0x42, 0xdd, 0x3f, 0x96, 0x10, 0x49, 0x23, 0x24, 0x27, 0x49, 0xe6, 0x15, + 0xa3, 0x8b, 0x1a, 0x6f, 0xef, 0xb3, 0xc5, 0x52, 0x48, 0xae, 0x0d, 0x93, 0x7b, 0xda, 0xc9, 0xf2, + 0x0f, 0x48, 0xfa, 0x66, 0x18, 0x43, 0x40, 0xd9, 0x67, 0x7a, 0x85, 0x13, 0x18, 0x4e, 0x45, 0xcb, + 0xd7, 0x46, 0xb5, 0xc7, 0xd4, 0xc3, 0x04, 0xc2, 0x99, 0xa8, 0x79, 0xea, 0xe3, 0x18, 0x92, 0x39, + 0x37, 0xac, 0x62, 0x86, 0xa5, 0x01, 0x8e, 0x20, 0x5e, 0x1c, 0x65, 0x2d, 0x9a, 0x5d, 0x1a, 0x5a, + 0xcf, 0xeb, 0xcb, 0x7c, 0xb9, 0xd8, 0xb2, 0xb6, 0x4a, 0xa3, 0xfc, 0xe1, 0x4f, 0x69, 0xbb, 0xcd, + 0x85, 0xe4, 0xa7, 0xed, 0x7a, 0xc5, 0x90, 0x9e, 0x39, 0x7f, 0x86, 0xc9, 0x45, 0x2e, 0x24, 0x10, + 0x6b, 0xbe, 0x56, 0x4d, 0xa5, 0xdd, 0x4b, 0x04, 0xb4, 0x47, 0xbc, 0x81, 0xa8, 0x61, 0x8d, 0xd2, + 0x6e, 0xe7, 0x31, 0xed, 0xe0, 0x37, 0x00, 0x00, 0xff, 0xff, 0x36, 0xaf, 0xfa, 0x7c, 0xd9, 0x01, + 0x00, 0x00, } diff --git a/ipld/unixfs/pb/unixfs.proto b/ipld/unixfs/pb/unixfs.proto index f65673f54..bd02aa410 100644 --- a/ipld/unixfs/pb/unixfs.proto +++ b/ipld/unixfs/pb/unixfs.proto @@ -21,8 +21,25 @@ message Data { optional uint64 hashType = 5; optional uint64 fanout = 6; + optional uint32 mode = 7; + optional IPFSTimestamp mtime = 8; } message Metadata { optional string MimeType = 1; } + +// mostly copied from proto 3 - with int32 nanos changed to fixed32 for js-ipfs compatibility +// https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto +message IPFSTimestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + required int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + optional fixed32 nanos = 2; +} diff --git a/ipld/unixfs/unixfs.go b/ipld/unixfs/unixfs.go index 4131df837..fb2c9bbf2 100644 --- a/ipld/unixfs/unixfs.go +++ b/ipld/unixfs/unixfs.go @@ -6,10 +6,12 @@ package unixfs import ( "errors" "fmt" + "os" + "time" proto "github.com/gogo/protobuf/proto" + files "github.com/ipfs/boxo/files" dag "github.com/ipfs/boxo/ipld/merkledag" - pb "github.com/ipfs/boxo/ipld/unixfs/pb" ipld "github.com/ipfs/go-ipld-format" ) @@ -34,6 +36,7 @@ const ( // Common errors var ( ErrMalformedFileFormat = errors.New("malformed data in file format") + ErrNotProtoNode = errors.New("expected a ProtoNode as internal node") ErrUnrecognizedType = errors.New("unrecognized node type") ) @@ -69,6 +72,22 @@ func FilePBData(data []byte, totalsize uint64) []byte { return data } +func FilePBDataWithStat(data []byte, totalsize uint64, mode os.FileMode, mtime time.Time) []byte { + pbfile := new(pb.Data) + typ := pb.Data_File + pbfile.Type = &typ + pbfile.Data = data + pbfile.Filesize = proto.Uint64(totalsize) + + pbDataAddStat(pbfile, mode, mtime) + + data, err := proto.Marshal(pbfile) + if err != nil { + panic(err) + } + return data +} + // FolderPBData returns Bytes that represent a Directory. func FolderPBData() []byte { pbfile := new(pb.Data) @@ -83,6 +102,36 @@ func FolderPBData() []byte { return data } +func FolderPBDataWithStat(mode os.FileMode, mtime time.Time) []byte { + pbfile := new(pb.Data) + typ := pb.Data_Directory + pbfile.Type = &typ + + pbDataAddStat(pbfile, mode, mtime) + + data, err := proto.Marshal(pbfile) + if err != nil { + //this really shouldnt happen, i promise + panic(err) + } + return data +} + +func pbDataAddStat(data *pb.Data, mode os.FileMode, mtime time.Time) { + if mode != 0 { + data.Mode = proto.Uint32(files.ModePermsToUnixPerms(mode)) + } + if !mtime.IsZero() { + data.Mtime = &pb.IPFSTimestamp{ + Seconds: proto.Int64(mtime.Unix()), + } + + if nanos := uint32(mtime.Nanosecond()); nanos > 0 { + data.Mtime.Nanos = &nanos + } + } +} + // WrapData marshals raw bytes into a `Data_Raw` type protobuf message. func WrapData(b []byte) []byte { pbdata := new(pb.Data) @@ -303,6 +352,93 @@ func (n *FSNode) IsDir() bool { } } +// Mode returns the optionally stored file permissions +func (n *FSNode) Mode() (m os.FileMode) { + perms := n.format.GetMode() & 0xFFF + if perms != 0 { + m = files.UnixPermsToModePerms(perms) + switch n.Type() { + case pb.Data_Directory, pb.Data_HAMTShard: + m |= os.ModeDir + case pb.Data_Symlink: + m |= os.ModeSymlink + } + } + return m +} + +// SetMode stores the given mode permissions, or nullifies stored permissions +// if none were provided and there are no extended bits set. +func (n *FSNode) SetMode(m os.FileMode) { + n.SetModeFromUnixPermissions(files.ModePermsToUnixPerms(m)) +} + +// SetModeFromUnixPermissions stores the given unix permissions, or nullifies stored permissions +// if none were provided and there are no extended bits set. +func (n *FSNode) SetModeFromUnixPermissions(unixPerms uint32) { + // preserve existing most significant 20 bits + newMode := (n.format.GetMode() & 0xFFFFF000) | (unixPerms & 0xFFF) + + if unixPerms == 0 { + if newMode&0xFFFFF000 == 0 { + n.format.Mode = nil + return + } + } + n.format.Mode = &newMode +} + +// ExtendedMode returns the 20 bits of extended file mode +func (n *FSNode) ExtendedMode() uint32 { + return (n.format.GetMode() & 0xFFFFF000) >> 12 +} + +// SetExtendedMode stores the 20 bits of extended file mode, only the first +// 20 bits of the `mode` argument are used, the remaining 12 bits are ignored. +func (n *FSNode) SetExtendedMode(mode uint32) { + newMode := (mode << 12) | (0xFFF & n.format.GetMode()) + if newMode == 0 { + n.format.Mode = nil + } else { + n.format.Mode = &newMode + } +} + +// ModTime returns the stored last modified timestamp if available. +func (n *FSNode) ModTime() time.Time { + ts := n.format.GetMtime() + if ts == nil || ts.Seconds == nil { + return time.Time{} + } + if ts.Nanos == nil { + return time.Unix(*ts.Seconds, 0) + } + if *ts.Nanos < 1 || *ts.Nanos > 999999999 { + return time.Time{} + } + + return time.Unix(*ts.Seconds, int64(*ts.Nanos)) +} + +// SetModTime stores the given last modified timestamp, otherwise nullifies stored timestamp. +func (n *FSNode) SetModTime(ts time.Time) { + if ts.IsZero() { + n.format.Mtime = nil + return + } + + if n.format.Mtime == nil { + n.format.Mtime = &pb.IPFSTimestamp{} + } + + n.format.Mtime.Seconds = proto.Int64(ts.Unix()) + if ts.Nanosecond() > 0 { + n.format.Mtime.Nanos = proto.Uint32(uint32(ts.Nanosecond())) + } else { + n.format.Mtime.Nanos = nil + } +} + // Metadata is used to store additional FSNode information. type Metadata struct { MimeType string @@ -360,6 +496,10 @@ func EmptyDirNode() *dag.ProtoNode { return dag.NodeWithData(FolderPBData()) } +func EmptyDirNodeWithStat(mode os.FileMode, mtime time.Time) *dag.ProtoNode { + return dag.NodeWithData(FolderPBDataWithStat(mode, mtime)) +} + // EmptyFileNode creates an empty file Protonode. func EmptyFileNode() *dag.ProtoNode { return dag.NodeWithData(FilePBData(nil, 0)) @@ -405,7 +545,7 @@ func ReadUnixFSNodeData(node ipld.Node) (data []byte, err error) { func ExtractFSNode(node ipld.Node) (*FSNode, error) { protoNode, ok := node.(*dag.ProtoNode) if !ok { - return nil, errors.New("expected a ProtoNode as internal node") + return nil, ErrNotProtoNode } fsNode, err := FSNodeFromBytes(protoNode.Data()) diff --git a/ipld/unixfs/unixfs_test.go b/ipld/unixfs/unixfs_test.go index b785be8ad..4cbc22ca8 100644 --- a/ipld/unixfs/unixfs_test.go +++ b/ipld/unixfs/unixfs_test.go @@ -2,7 +2,9 @@ package unixfs import ( "bytes" + "os" "testing" + "time" proto "github.com/gogo/protobuf/proto" @@ -183,3 +185,251 @@ func TestIsDir(t *testing.T) { } } } + +func (n *FSNode) getPbData(t *testing.T) *pb.Data { + b, err := n.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + return pbn +} + +func TestMode(t *testing.T) { + fsn := NewFSNode(TDirectory) + fsn.SetMode(1) + if !fsn.Mode().IsDir() { + t.Fatal("expected mode for directory") + } + + fsn = NewFSNode(TSymlink) + fsn.SetMode(1) + if fsn.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Fatal("expected mode for symlink") + } + + fsn = NewFSNode(TFile) + + // not stored + if fsn.Mode() != 0 { + t.Fatal("expected mode not to be set") + } + + fileMode := os.FileMode(0640) + fsn.SetMode(fileMode) + if !fsn.Mode().IsRegular() { + t.Fatal("expected a regular file mode") + } + mode := fsn.Mode() + + if mode&os.ModePerm != fileMode { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&0xFFFFF000 != 0 { + t.Fatalf("expected high-order 20 bits of mode to be clear but got %b", (mode&0xFFFFF000)>>12) + } + + fsn.SetMode(fileMode | os.ModeSticky) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky == 0 { + t.Fatal("expected permissions to have sticky bit set") + } + if mode&os.ModeSetuid != 0 { + t.Fatal("expected permissions to have setuid bit unset") + } + if mode&os.ModeSetgid != 0 { + t.Fatal("expected permissions to have setgid bit unset") + } + + fsn.SetMode(fileMode | os.ModeSticky | os.ModeSetuid) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky == 0 { + t.Fatal("expected permissions to have sticky bit set") + } + if mode&os.ModeSetuid == 0 { + t.Fatal("expected permissions to have setuid bit set") + } + if mode&os.ModeSetgid != 0 { + t.Fatal("expected permissions to have setgid bit unset") + } + + fsn.SetMode(fileMode | os.ModeSetuid | os.ModeSetgid) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky != 0 { + t.Fatal("expected permissions to have sticky bit unset") + } + if mode&os.ModeSetuid == 0 { + t.Fatal("expected permissions to have setuid bit set") + } + if mode&os.ModeSetgid == 0 { + t.Fatal("expected permissions to have setgid bit set") + } + + // check the internal format (unix permissions) + fsn.SetMode(fileMode | os.ModeSetuid | os.ModeSticky) + pbn := fsn.getPbData(t) + // unix perms setuid and sticky bits should also be set + expected := uint32(05000 | (fileMode & os.ModePerm)) + if *pbn.Mode != expected { + t.Fatalf("expected stored permissions to be %O but got %O", expected, *pbn.Mode) + } + + fsn.SetMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode != nil { + t.Fatal("expected file mode to be unset") + } + + fsn.SetExtendedMode(1) + fsn.SetMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode == nil { + t.Fatal("expected extended mode to be preserved") + } +} + +func TestExtendedMode(t *testing.T) { + fsn := NewFSNode(TFile) + fsn.SetMode(os.ModePerm | os.ModeSetuid | os.ModeSticky) + const expectedUnixMode = uint32(05777) + + expectedExtMode := uint32(0xAAAAA) + fsn.SetExtendedMode(expectedExtMode) + extMode := fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn := fsn.getPbData(t) + expectedPbMode := (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected stored mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + expectedExtMode = uint32(0x55555) + fsn.SetExtendedMode(expectedExtMode) + extMode = fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn = fsn.getPbData(t) + expectedPbMode = (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected stored mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + // ignore bits 21..32 + expectedExtMode = uint32(0xFFFFF) + fsn.SetExtendedMode(0xAAAFFFFF) + extMode = fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn = fsn.getPbData(t) + expectedPbMode = (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected raw mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + fsn.SetMode(0) + fsn.SetExtendedMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode != nil { + t.Fatal("expected file mode to be unset") + } +} + +func (n *FSNode) setPbModTime(seconds *int64, nanos *uint32) { + if n.format.Mtime == nil { + n.format.Mtime = &pb.IPFSTimestamp{} + } + + n.format.Mtime.Seconds = seconds + n.format.Mtime.Nanos = nanos +} + +func TestModTime(t *testing.T) { + tm := time.Now() + expectedUnix := tm.Unix() + n := NewFSNode(TFile) + + // not stored + mt := n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + // valid timestamps + n.SetModTime(tm) + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Unix(expectedUnix, 0) + n.SetModTime(tm) + pbn := n.getPbData(t) + if pbn.Mtime.Nanos != nil { + t.Fatal("expected nanoseconds to be nil") + } + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Unix(expectedUnix, 3489753) + n.SetModTime(tm) + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Time{} + n.SetModTime(tm) + pbn = n.getPbData(t) + if pbn.Mtime != nil { + t.Fatal("expected modification time to be unset") + } + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + n.setPbModTime(&expectedUnix, nil) + mt = n.ModTime() + if !mt.Equal(time.Unix(expectedUnix, 0)) { + t.Fatalf("expected modification time to be %v but got %v", time.Unix(expectedUnix, 0), mt) + } + + // invalid timestamps + n.setPbModTime(nil, proto.Uint32(1000)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + n.setPbModTime(&expectedUnix, proto.Uint32(0)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + n.setPbModTime(&expectedUnix, proto.Uint32(1000000000)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } +} diff --git a/mfs/dir.go b/mfs/dir.go index 86c85d1c5..38302ac39 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -41,8 +41,6 @@ type Directory struct { // UnixFS directory implementation used for creating, // reading and editing directories. unixfsDir uio.Directory - - modTime time.Time } // NewDirectory constructs a new MFS directory. @@ -64,7 +62,6 @@ func NewDirectory(ctx context.Context, name string, node ipld.Node, parent paren ctx: ctx, unixfsDir: db, entriesCache: make(map[string]FSNode), - modTime: time.Now(), }, nil } @@ -135,8 +132,6 @@ func (d *Directory) updateChild(c child) error { return err } - d.modTime = time.Now() - return nil } @@ -292,6 +287,10 @@ func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) } func (d *Directory) Mkdir(name string) (*Directory, error) { + return d.MkdirWithOpts(name, MkdirOpts{}) +} + +func (d *Directory) MkdirWithOpts(name string, opts MkdirOpts) (*Directory, error) { d.lock.Lock() defer d.lock.Unlock() @@ -307,7 +306,7 @@ func (d *Directory) Mkdir(name string) (*Directory, error) { } } - ndir := ft.EmptyDirNode() + ndir := ft.EmptyDirNodeWithStat(opts.Mode, opts.ModTime) ndir.SetCidBuilder(d.GetCidBuilder()) err = d.dagService.Add(d.ctx, ndir) @@ -367,7 +366,6 @@ func (d *Directory) AddChild(name string, nd ipld.Node) error { return err } - d.modTime = time.Now() return nil } @@ -427,3 +425,68 @@ func (d *Directory) GetNode() (ipld.Node, error) { return nd.Copy(), err } + +func (d *Directory) SetMode(mode os.FileMode) error { + nd, err := d.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetMode(mode) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return d.setNodeData(data, nd.Links()) +} + +func (d *Directory) SetModTime(ts time.Time) error { + nd, err := d.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetModTime(ts) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return d.setNodeData(data, nd.Links()) +} + +func (d *Directory) setNodeData(data []byte, links []*ipld.Link) error { + nd := dag.NodeWithData(data) + nd.SetLinks(links) + + err := d.dagService.Add(d.ctx, nd) + if err != nil { + return err + } + + err = d.parent.updateChildEntry(child{d.name, nd}) + if err != nil { + return err + } + + d.lock.Lock() + defer d.lock.Unlock() + db, err := uio.NewDirectoryFromNode(d.dagService, nd) + if err != nil { + return err + } + d.unixfsDir = db + + return nil +} diff --git a/mfs/file.go b/mfs/file.go index 56c2b0046..aff025db6 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -3,7 +3,9 @@ package mfs import ( "context" "errors" + "os" "sync" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" @@ -177,3 +179,101 @@ func (fi *File) Sync() error { func (fi *File) Type() NodeType { return TFile } + +func (fi *File) Mode() (os.FileMode, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + + nd, err := fi.GetNode() + if err != nil { + return 0, err + } + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return 0, err + } + return fsn.Mode() & 0xFFF, nil +} + +func (fi *File) SetMode(mode os.FileMode) error { + nd, err := fi.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + if errors.Is(err, ft.ErrNotProtoNode) { + // Wrap raw node in protonode. + data := nd.RawData() + return fi.setNodeData(ft.FilePBDataWithStat(data, uint64(len(data)), mode, time.Time{})) + } + return err + } + + fsn.SetMode(mode) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return fi.setNodeData(data) +} + +// ModTime returns the files' last modification time +func (fi *File) ModTime() (time.Time, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + + nd, err := fi.GetNode() + if err != nil { + return time.Time{}, err + } + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return time.Time{}, err + } + return fsn.ModTime(), nil +} + +// SetModTime sets the files' last modification time +func (fi *File) SetModTime(ts time.Time) error { + nd, err := fi.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + if errors.Is(err, ft.ErrNotProtoNode) { + // Wrap raw node in protonode. + data := nd.RawData() + return fi.setNodeData(ft.FilePBDataWithStat(data, uint64(len(data)), 0, ts)) + } + return err + } + + fsn.SetModTime(ts) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return fi.setNodeData(data) +} + +func (fi *File) setNodeData(data []byte) error { + nd := dag.NodeWithData(data) + err := fi.inode.dagService.Add(context.TODO(), nd) + if err != nil { + return err + } + + fi.nodeLock.Lock() + defer fi.nodeLock.Unlock() + fi.node = nd + parent := fi.inode.parent + name := fi.inode.name + + return parent.updateChildEntry(child{name, fi.node}) +} diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index eb5585a64..ee2726160 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -11,6 +11,7 @@ import ( "math/rand" "os" gopath "path" + "runtime" "sort" "strings" "sync" @@ -511,7 +512,19 @@ func TestMfsFile(t *testing.T) { fi := fsn.(*File) if fi.Type() != TFile { - t.Fatal("some is seriously wrong here") + t.Fatal("something is seriously wrong here") + } + + if m, err := fi.Mode(); err != nil { + t.Fatal("failed to get file mode: ", err) + } else if m != 0 { + t.Fatal("mode should not be set on a new file") + } + + if ts, err := fi.ModTime(); err != nil { + t.Fatal("failed to get file mtime: ", err) + } else if !ts.IsZero() { + t.Fatal("modification time should not be set on a new file") } wfd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) @@ -615,6 +628,12 @@ func TestMfsFile(t *testing.T) { t.Fatal(err) } + if ts, err := fi.ModTime(); err != nil { + t.Fatal("failed to get file mtime: ", err) + } else if !ts.IsZero() { + t.Fatal("file with unset modification time should not update modification time") + } + // make sure we can get node. TODO: verify it later _, err = fi.GetNode() if err != nil { @@ -622,6 +641,233 @@ func TestMfsFile(t *testing.T) { } } +func TestMfsModeAndModTime(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ds, rt := setupRoot(ctx, t) + rootdir := rt.GetDirectory() + nd := getRandFile(t, ds, 1000) + + err := rootdir.AddChild("file", nd) + if err != nil { + t.Fatal(err) + } + + fsn, err := rootdir.Child("file") + if err != nil { + t.Fatal(err) + } + + fi := fsn.(*File) + + if fi.Type() != TFile { + t.Fatal("something is seriously wrong here") + } + + var mode os.FileMode + ts, _ := time.Now(), time.Time{} + + // can set mode + if err = fi.SetMode(0644); err == nil { + if mode, err = fi.Mode(); mode != 0644 { + t.Fatal("failed to get correct mode of file") + } + } + if err != nil { + t.Fatal("failed to check file mode: ", err) + } + + // can set last modification time + if err = fi.SetModTime(ts); err == nil { + ts2, err := fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.Equal(ts) { + t.Fatal("failed to get correct modification time of file") + } + } + if err != nil { + t.Fatal("failed to check file modification time: ", err) + } + + // test modification time update after write (on closing file) + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err := fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + _, err = wfd.Write([]byte("test")) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err := fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file write") + } + + // writeAt + ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + _, err = wfd.WriteAt([]byte("test"), 42) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file writeAt") + } + + // truncate (shrink) + ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + err = wfd.Truncate(100) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file truncate (shrink)") + } + + // truncate (expand) + ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + err = wfd.Truncate(1500) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file truncate (expand)") + } +} + +func TestMfsRawNodeSetModeAndMtime(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, rt := setupRoot(ctx, t) + rootdir := rt.GetDirectory() + + // Create raw-node file. + nd := dag.NewRawNode(random.Bytes(256)) + _, err := ft.ExtractFSNode(nd) + if !errors.Is(err, ft.ErrNotProtoNode) { + t.Fatal("Expected non-proto node") + } + + err = rootdir.AddChild("file", nd) + if err != nil { + t.Fatal(err) + } + + fsn, err := rootdir.Child("file") + if err != nil { + t.Fatal(err) + } + + fi := fsn.(*File) + if fi.Type() != TFile { + t.Fatal("something is seriously wrong here") + } + + // Check for expected error when getting mode and mtime. + _, err = fi.Mode() + if !errors.Is(err, ft.ErrNotProtoNode) { + t.Fatal("Expected non-proto node") + } + _, err = fi.ModTime() + if !errors.Is(err, ft.ErrNotProtoNode) { + t.Fatal("Expected non-proto node") + } + + // Set and check mode. + err = fi.SetMode(0644) + if err != nil { + t.Fatalf("failed to set file mode: %s", err) + } + mode, err := fi.Mode() + if err != nil { + t.Fatalf("failed to check file mode: %s", err) + } + if mode != 0644 { + t.Fatal("failed to get correct mode of file, got", mode.String()) + } + + // Mtime should still be unset. + mtime, err := fi.ModTime() + if err != nil { + t.Fatalf("failed to get file modification time: %s", err) + } + if !mtime.IsZero() { + t.Fatalf("expected mtime to be unset") + } + + // Set and check mtime. + now := time.Now() + err = fi.SetModTime(now) + if err != nil { + t.Fatalf("failed to set file modification time: %s", err) + } + mtime, err = fi.ModTime() + if err != nil { + t.Fatalf("failed to get file modification time: %s", err) + } + if !mtime.Equal(now) { + t.Fatal("failed to get correct modification time of file") + } +} + func TestMfsDirListNames(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/mfs/ops.go b/mfs/ops.go index 693264704..09dbab00f 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,6 +7,7 @@ import ( "os" gopath "path" "strings" + "time" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -122,6 +123,8 @@ type MkdirOpts struct { Mkparents bool Flush bool CidBuilder cid.Builder + Mode os.FileMode + ModTime time.Time } // Mkdir creates a directory at 'path' under the directory 'd', creating @@ -171,7 +174,7 @@ func Mkdir(r *Root, pth string, opts MkdirOpts) error { cur = next } - final, err := cur.Mkdir(parts[len(parts)-1]) + final, err := cur.MkdirWithOpts(parts[len(parts)-1], opts) if err != nil { if !opts.Mkparents || err != os.ErrExist || final == nil { return err @@ -243,3 +246,21 @@ func FlushPath(ctx context.Context, rt *Root, pth string) (ipld.Node, error) { rt.repub.WaitPub(ctx) return nd.GetNode() } + +func Chmod(rt *Root, pth string, mode os.FileMode) error { + nd, err := Lookup(rt, pth) + if err != nil { + return err + } + + return nd.SetMode(mode) +} + +func Touch(rt *Root, pth string, ts time.Time) error { + nd, err := Lookup(rt, pth) + if err != nil { + return err + } + + return nd.SetModTime(ts) +} diff --git a/mfs/root.go b/mfs/root.go index c08d2d053..5a7cb7ed1 100644 --- a/mfs/root.go +++ b/mfs/root.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "os" "time" dag "github.com/ipfs/boxo/ipld/merkledag" @@ -73,6 +74,8 @@ type FSNode interface { Flush() error Type() NodeType + SetModTime(ts time.Time) error + SetMode(mode os.FileMode) error } // IsDir checks whether the FSNode is dir type diff --git a/out b/out deleted file mode 100644 index e69de29bb..000000000 diff --git a/tar/extractor.go b/tar/extractor.go index 8b9dfce6d..550fb895c 100644 --- a/tar/extractor.go +++ b/tar/extractor.go @@ -6,8 +6,12 @@ import ( "fmt" "io" "os" - fp "path/filepath" + "path/filepath" + "runtime" "strings" + "time" + + "github.com/ipfs/boxo/files" ) var ( @@ -18,23 +22,31 @@ var ( // Extractor is used for extracting tar files to a filesystem. // -// The Extractor can only extract tar files containing files, directories and symlinks. Additionally, the tar files must -// either have a single file, or symlink in them, or must have all of its objects inside of a single root directory -// object. +// The Extractor can only extract tar files containing files, directories and +// symlinks. Additionally, the tar files must either have a single file, or +// symlink in them, or must have all of its objects inside of a single root +// directory object. +// +// If the tar file contains a single file/symlink then it will try and extract +// it with semantics similar to Linux's `cp`. In particular, the name of the +// extracted file/symlink will match the extraction path. If the extraction +// path is a directory then it will extract into the directory using its +// original name. // -// If the tar file contains a single file/symlink then it will try and extract it with semantics similar to Linux's -// `cp`. In particular, the name of the extracted file/symlink will match the extraction path. If the extraction path -// is a directory then it will extract into the directory using its original name. +// If an associated mode and last modification time was stored in the archive +// it is restored. // -// Overwriting: Extraction of files and symlinks will result in overwriting the existing objects with the same name -// when possible (i.e. other files, symlinks, and empty directories). +// Overwriting: Extraction of files and symlinks will result in overwriting the +// existing objects with the same name when possible (i.e. other files, +// symlinks, and empty directories). type Extractor struct { - Path string - Progress func(int64) int64 + Path string + Progress func(int64) int64 + deferredUpdates []deferredUpdate } -// Extract extracts a tar file to the file system. See the Extractor for more information on the limitations on the -// tar files that can be extracted. +// Extract extracts a tar file to the file system. See the Extractor for more +// information on the limitations on the tar files that can be extracted. func (te *Extractor) Extract(reader io.Reader) error { if isNullDevice(te.Path) { return nil @@ -42,8 +54,6 @@ func (te *Extractor) Extract(reader io.Reader) error { tarReader := tar.NewReader(reader) - var firstObjectWasDir bool - header, err := tarReader.Next() if err != nil && err != io.EOF { return err @@ -52,10 +62,25 @@ func (te *Extractor) Extract(reader io.Reader) error { return errors.New("empty tar file") } - // Specially handle the first entry assuming it is a single root object (e.g. root directory, single file, - // or single symlink) + te.deferredUpdates = make([]deferredUpdate, 0, 80) + doUpdates := func() error { + for i := len(te.deferredUpdates) - 1; i >= 0; i-- { + m := te.deferredUpdates[i] + err := files.UpdateMetaUnix(m.path, uint32(m.mode), m.mtime) + if err != nil { + return err + } + } + te.deferredUpdates = nil + return nil + } + defer func() { err = doUpdates() }() + + // Specially handle the first entry assuming it is a single root object + // (e.g. root directory, single file, or single symlink). - // track what the root tar path is so we can ensure that all other entries are below the root + // track what the root tar path is so we can ensure that all other entries + // are below the root. if strings.Contains(header.Name, "/") { return fmt.Errorf("root name contains multiple components : %q : %w", header.Name, errInvalidRoot) } @@ -65,30 +90,39 @@ func (te *Extractor) Extract(reader io.Reader) error { } rootName := header.Name - // Get the platform-specific output path - rootOutputPath := fp.Clean(te.Path) + // Get the platform-specific output path. + rootOutputPath := filepath.Clean(te.Path) if err := validatePlatformPath(rootOutputPath); err != nil { return err } - // If the last element in the rootOutputPath (which is passed by the user) is a symlink do not follow it - // this makes it easier for users to reason about where files are getting extracted to even when the tar is not - // from a trusted source + var firstObjectWasDir bool + + // If the last element in the rootOutputPath (which is passed by the user) + // is a symlink do not follow it this makes it easier for users to reason + // about where files are getting extracted to even when the tar is not from + // a trusted source // - // For example, if the user extracts a mutable link to a tar file (http://sometimesbad.tld/t.tar) and situationally - // it contains a folder, file, or symlink the outputs could hop around the user's file system. This is especially - // annoying since we allow symlinks to point anywhere a user might want them to. + // For example, if the user extracts a mutable link to a tar file + // (http://sometimesbad.tld/t.tar) and situationally it contains a folder, + // file, or symlink the outputs could hop around the user's file system. + // This is especially annoying since we allow symlinks to point anywhere a + // user might want them to. switch header.Typeflag { case tar.TypeDir: - // if this is the root directory, use it as the output path for remaining files + // if this is the root directory, use it as the output path for + // remaining files. firstObjectWasDir = true if err := te.extractDir(rootOutputPath); err != nil { return err } + if err := te.deferUpdate(rootOutputPath, header); err != nil { + return err + } case tar.TypeReg, tar.TypeSymlink: - // Check if the output path already exists, so we know whether we should - // create our output with that name, or if we should put the output inside - // a preexisting directory + // Check if the output path already exists, so we know whether we + // should create our output with that name, or if we should put the + // output inside a preexisting directory. rootIsExistingDirectory := false // We do not follow links here @@ -101,30 +135,35 @@ func (te *Extractor) Extract(reader io.Reader) error { } outputPath := rootOutputPath - // If the root is a directory which already exists then put the file/symlink in the directory + // If the root is a directory which already exists then put the + // file/symlink in the directory. if rootIsExistingDirectory { - // make sure the root has a valid name + // make sure the root has a valid name. if err := validatePathComponent(rootName); err != nil { return err } - // If the output path directory exists then put the file/symlink into the directory. - outputPath = fp.Join(rootOutputPath, rootName) + // If the output path directory exists then put the file/symlink + // into the directory. + outputPath = filepath.Join(rootOutputPath, rootName) } - // If an object with the target name already exists overwrite it + // If an object with the target name already exists overwrite it. if header.Typeflag == tar.TypeReg { if err := te.extractFile(outputPath, tarReader); err != nil { return err } - } else if err := te.extractSymlink(outputPath, header); err != nil { + if err := files.UpdateMetaUnix(outputPath, uint32(header.Mode), header.ModTime); err != nil { + return err + } + } else if err := te.extractSymlink(outputPath, rootOutputPath, header); err != nil { return err } default: return fmt.Errorf("unrecognized tar header type: %d", header.Typeflag) } - // files come recursively in order + // files come recursively in order. for { header, err := tarReader.Next() if err != nil && err != io.EOF { @@ -134,12 +173,13 @@ func (te *Extractor) Extract(reader io.Reader) error { break } - // Make sure that we only have a single root element + // Make sure that we only have a single root element. if !firstObjectWasDir { return fmt.Errorf("the root was not a directory and the tar has multiple entries: %w", errInvalidRoot) } - // validate the path to remove paths we refuse to work with and make it easier to reason about + // validate the path in order to remove paths we refuse to work with + // and make it easier to reason about. if err := validateTarPath(header.Name); err != nil { return err } @@ -155,14 +195,15 @@ func (te *Extractor) Extract(reader io.Reader) error { return err } - // This check should already be covered by previous validation, but may catch bugs that slip through. - // Checks if the relative path matches or exceeds the root - // We check for matching because the outputPath function strips the original root - rel, err := fp.Rel(rootOutputPath, outputPath) + // This check should already be covered by previous validation, but may + // catch bugs that slip through. Checks if the relative path matches or + // exceeds the root We check for matching because the outputPath + // function strips the original root + rel, err := filepath.Rel(rootOutputPath, outputPath) if err != nil || rel == "." { return errInvalidRootMultipleRoots } - for _, e := range strings.Split(fp.ToSlash(rel), "/") { + for _, e := range strings.Split(filepath.ToSlash(rel), "/") { if e == ".." { return errors.New("relative path contains '..'") } @@ -173,12 +214,18 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := te.extractDir(outputPath); err != nil { return err } + if err := te.deferUpdate(outputPath, header); err != nil { + return err + } case tar.TypeReg: if err := te.extractFile(outputPath, tarReader); err != nil { return err } + if err := files.UpdateMetaUnix(outputPath, uint32(header.Mode), header.ModTime); err != nil { + return err + } case tar.TypeSymlink: - if err := te.extractSymlink(outputPath, header); err != nil { + if err := te.extractSymlink(outputPath, rootOutputPath, header); err != nil { return err } default: @@ -188,7 +235,7 @@ func (te *Extractor) Extract(reader io.Reader) error { return nil } -// validateTarPath returns an error if the path has problematic characters +// validateTarPath returns an error if the path has problematic characters. func validateTarPath(tarPath string) error { if len(tarPath) == 0 { return errors.New("path is empty") @@ -208,8 +255,9 @@ func validateTarPath(tarPath string) error { return nil } -// getRelativePath returns the relative path between rootTarPath and tarPath. Assumes both paths have been cleaned. -// Will error if the tarPath is not below the rootTarPath. +// getRelativePath returns the relative path between rootTarPath and tarPath. +// Assumes both paths have been cleaned. Will error if the tarPath is not below +// the rootTarPath. func getRelativePath(rootName, tarPath string) (string, error) { if !strings.HasPrefix(tarPath, rootName+"/") { return "", errInvalidRootMultipleRoots @@ -217,7 +265,8 @@ func getRelativePath(rootName, tarPath string) (string, error) { return tarPath[len(rootName)+1:], nil } -// outputPath returns the directory path at which to place the file relativeTarPath. Assumes relativeTarPath is cleaned. +// outputPath returns the directory path at which to place the file +// relativeTarPath. Assumes relativeTarPath is cleaned. func (te *Extractor) outputPath(basePlatformPath, relativeTarPath string) (string, error) { elems := strings.Split(relativeTarPath, "/") @@ -226,10 +275,10 @@ func (te *Extractor) outputPath(basePlatformPath, relativeTarPath string) (strin if err := validatePathComponent(e); err != nil { return "", err } - platformPath = fp.Join(platformPath, e) + platformPath = filepath.Join(platformPath, e) - // Last element is not checked since it will be removed (if it exists) by any of the extraction functions. - // For more details see: + // Last element is not checked since it will be removed (if it exists) + // by any of the extraction functions. For more details see: // https://github.com/libarchive/libarchive/blob/0fd2ed25d78e9f4505de5dcb6208c6c0ff8d2edb/libarchive/archive_write_disk_posix.c#L2810 if i == len(elems)-1 { break @@ -259,46 +308,69 @@ func (te *Extractor) extractDir(path string) error { return err } - if stat, err := os.Lstat(path); err != nil { + stat, err := os.Lstat(path) + if err != nil { return err - } else if !stat.IsDir() { + } + if !stat.IsDir() { return errExtractedDirToSymlink } return nil } -func (te *Extractor) extractSymlink(path string, h *tar.Header) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { +func (te *Extractor) extractSymlink(path, rootPath string, h *tar.Header) error { + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + // Before extracting a file or other symlink, the old path is removed to + // prevent a simlink being created that causes a subsequent extraction to + // escape the root. + // + // Each element of the path of the symlink being extracted is evaluated to + // ensure that there is not a symlink at any point in the path. This is + // done in outputPath. + err = os.Symlink(h.Linkname, path) + if err != nil { return err } - return os.Symlink(h.Linkname, path) + switch runtime.GOOS { + case "linux", "freebsd", "netbsd", "openbsd", "dragonfly": + return files.UpdateModTime(path, h.ModTime) + default: + return nil + } } func (te *Extractor) extractFile(path string, r *tar.Reader) error { - // Attempt removing the target so we can overwrite files, symlinks and empty directories - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + // Attempt removing the target so we can overwrite files, symlinks and + // empty directories. + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { return err } - // Create a temporary file in the target directory and then rename the temporary file to the target to better deal - // with races on the file system. - base := fp.Dir(path) + // Create a temporary file in the target directory and then rename the + // temporary file to the target to better deal with races on the file + // system. + base := filepath.Dir(path) tmpfile, err := os.CreateTemp(base, "") if err != nil { return err } - if err := copyWithProgress(tmpfile, r, te.Progress); err != nil { + if err = copyWithProgress(tmpfile, r, te.Progress); err != nil { _ = tmpfile.Close() _ = os.Remove(tmpfile.Name()) return err } - if err := tmpfile.Close(); err != nil { + if err = tmpfile.Close(); err != nil { _ = os.Remove(tmpfile.Name()) return err } - if err := os.Rename(tmpfile.Name(), path); err != nil { + if err = os.Rename(tmpfile.Name(), path); err != nil { _ = os.Remove(tmpfile.Name()) return err } @@ -327,3 +399,45 @@ func copyWithProgress(to io.Writer, from io.Reader, cb func(int64) int64) error } } } + +type deferredUpdate struct { + path string + mode int64 + mtime time.Time +} + +func (te *Extractor) deferUpdate(path string, header *tar.Header) error { + if header.Mode == 0 && header.ModTime.IsZero() { + return nil + } + + prefix := func() string { + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '/' { + return path[:i] + } + } + return path + } + + n := len(te.deferredUpdates) + if n > 0 && len(path) < len(te.deferredUpdates[n-1].path) { + // if possible, apply the previous deferral. + m := te.deferredUpdates[n-1] + if strings.HasPrefix(m.path, prefix()) { + err := files.UpdateMetaUnix(m.path, uint32(m.mode), m.mtime) + if err != nil { + return err + } + te.deferredUpdates = te.deferredUpdates[:n-1] + } + } + + te.deferredUpdates = append(te.deferredUpdates, deferredUpdate{ + path: path, + mode: header.Mode, + mtime: header.ModTime, + }) + + return nil +} diff --git a/tar/extractor_test.go b/tar/extractor_test.go index d2b4e00fc..7e31fbea5 100644 --- a/tar/extractor_test.go +++ b/tar/extractor_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/ipfs/boxo/files" "github.com/stretchr/testify/assert" ) @@ -50,7 +51,7 @@ func TestSingleFile(t *testing.T) { fileData := "file data" testTarExtraction(t, nil, []tarEntry{ - &fileTarEntry{fileName, []byte(fileData)}, + &fileTarEntry{path: fileName, buf: []byte(fileData)}, }, func(t *testing.T, extractDir string) { f, err := os.Open(fp.Join(extractDir, fileName)) @@ -64,11 +65,34 @@ func TestSingleFile(t *testing.T) { ) } +func TestSingleFileWithMeta(t *testing.T) { + fileName := "file2..ext" + fileData := "file2 data" + mode := 0654 + mtime := time.Now().Round(time.Second) + + testTarExtraction(t, nil, []tarEntry{ + &fileTarEntry{path: fileName, buf: []byte(fileData), mode: mode, mtime: mtime}, + }, + func(t *testing.T, extractDir string) { + path := fp.Join(extractDir, fileName) + testMeta(t, path, mode, mtime) + f, err := os.Open(path) + assert.NoError(t, err) + data, err := io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, fileData, string(data)) + assert.NoError(t, f.Close()) + }, + nil, + ) +} + func TestSingleDirectory(t *testing.T) { dirName := "dir..sfx" testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{dirName}, + &dirTarEntry{path: dirName}, }, func(t *testing.T, extractDir string) { f, err := os.Open(extractDir) @@ -85,13 +109,37 @@ func TestSingleDirectory(t *testing.T) { ) } +func TestSingleDirectoryWithMeta(t *testing.T) { + dirName := "dir2..sfx" + mode := 0765 + mtime := time.Now().Round(time.Second) + + testTarExtraction(t, nil, []tarEntry{ + &dirTarEntry{path: dirName, mode: mode, mtime: mtime}, + }, + func(t *testing.T, extractDir string) { + testMeta(t, extractDir, mode, mtime) + f, err := os.Open(extractDir) + if err != nil { + t.Fatal(err) + } + objs, err := f.Readdir(1) + if err == io.EOF && len(objs) == 0 { + return + } + t.Fatalf("expected an empty directory") + }, + nil, + ) +} + func TestDirectoryFollowSymlinkToNothing(t *testing.T) { dirName := "dir" childName := "child" entries := []tarEntry{ - &dirTarEntry{dirName}, - &dirTarEntry{dirName + "/" + childName}, + &dirTarEntry{path: dirName}, + &dirTarEntry{path: dirName + "/" + childName}, } testTarExtraction(t, func(t *testing.T, rootDir string) { @@ -109,8 +157,8 @@ func TestDirectoryFollowSymlinkToFile(t *testing.T) { childName := "child" entries := []tarEntry{ - &dirTarEntry{dirName}, - &dirTarEntry{dirName + "/" + childName}, + &dirTarEntry{path: dirName}, + &dirTarEntry{path: dirName + "/" + childName}, } testTarExtraction(t, func(t *testing.T, rootDir string) { @@ -132,8 +180,8 @@ func TestDirectoryFollowSymlinkToDirectory(t *testing.T) { childName := "child" entries := []tarEntry{ - &dirTarEntry{dirName}, - &dirTarEntry{dirName + "/" + childName}, + &dirTarEntry{path: dirName}, + &dirTarEntry{path: dirName + "/" + childName}, } testTarExtraction(t, func(t *testing.T, rootDir string) { @@ -159,7 +207,7 @@ func TestSingleSymlink(t *testing.T) { symlinkName := "symlink" testTarExtraction(t, nil, []tarEntry{ - &symlinkTarEntry{targetName, symlinkName}, + &symlinkTarEntry{target: targetName, path: symlinkName}, }, func(t *testing.T, extractDir string) { symlinkPath := fp.Join(extractDir, symlinkName) fi, err := os.Lstat(symlinkPath) @@ -177,37 +225,37 @@ func TestSingleSymlink(t *testing.T) { func TestMultipleRoots(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root"}, - &dirTarEntry{"sibling"}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "sibling"}, }, nil, errInvalidRoot) } func TestMultipleRootsNested(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child1"}, - &dirTarEntry{"root/child2"}, + &dirTarEntry{path: "root/child1"}, + &dirTarEntry{path: "root/child2"}, }, nil, errInvalidRoot) } func TestOutOfOrderRoot(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child"}, - &dirTarEntry{"root"}, + &dirTarEntry{path: "root/child"}, + &dirTarEntry{path: "root"}, }, nil, errInvalidRoot) } func TestOutOfOrder(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child/grandchild"}, - &dirTarEntry{"root/child"}, + &dirTarEntry{path: "root/child/grandchild"}, + &dirTarEntry{path: "root/child"}, }, nil, errInvalidRoot) } func TestNestedDirectories(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root"}, - &dirTarEntry{"root/child"}, - &dirTarEntry{"root/child/grandchild"}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "root/child"}, + &dirTarEntry{path: "root/child/grandchild"}, }, func(t *testing.T, extractDir string) { walkIndex := 0 err := fp.Walk(extractDir, @@ -234,19 +282,167 @@ func TestNestedDirectories(t *testing.T) { func TestRootDirectoryHasSubpath(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child"}, - &dirTarEntry{"root/child/grandchild"}, + &dirTarEntry{path: "root/child"}, + &dirTarEntry{path: "root/child/grandchild"}, }, nil, errInvalidRoot) } func TestFilesAndFolders(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root"}, - &dirTarEntry{"root/childdir"}, - &fileTarEntry{"root/childdir/file1", []byte("some data")}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "root/childdir"}, + &fileTarEntry{path: "root/childdir/file1", buf: []byte("some data")}, }, nil, nil) } +func TestFilesAndFoldersWithMetadata(t *testing.T) { + tm := time.Unix(660000000, 0) + + entries := []tarEntry{ + &dirTarEntry{path: "root", mtime: tm.Add(5 * time.Second)}, + &dirTarEntry{path: "root/childdir", mode: 03775}, + &fileTarEntry{path: "root/childdir/file1", buf: []byte("some data"), mode: 04744, + mtime: tm.Add(10 * time.Second)}, + &fileTarEntry{path: "root/childdir/file2", buf: []byte("some data"), mode: 0560, + mtime: tm.Add(10 * time.Second)}, + &fileTarEntry{path: "root/childdir/file3", buf: []byte("some data"), mode: 06540, + mtime: tm.Add(10 * time.Second)}, + } + + testTarExtraction(t, nil, entries, func(t *testing.T, extractDir string) { + walkIndex := 0 + err := fp.Walk(extractDir, + func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + switch walkIndex { + case 0: // root + assert.Equal(t, tm.Add(5*time.Second), fi.ModTime()) + case 1: // childdir + if runtime.GOOS != "windows" { + assert.Equal(t, 0775, int(fi.Mode()&0xFFF)) + assert.Equal(t, os.ModeSetgid, fi.Mode()&os.ModeSetgid) + assert.Equal(t, os.ModeSticky, fi.Mode()&os.ModeSticky) + } else { + assert.Equal(t, 0777, int(fi.Mode()&0xFFF)) + } + case 2: // file1 + assert.Equal(t, tm.Add(10*time.Second), fi.ModTime()) + if runtime.GOOS != "windows" { + assert.Equal(t, 0744, int(fi.Mode()&0xFFF)) + assert.Equal(t, os.ModeSetuid, fi.Mode()&os.ModeSetuid) + } else { + assert.Equal(t, 0666, int(fi.Mode()&0xFFF)) + } + case 3: // file2 + assert.Equal(t, tm.Add(10*time.Second), fi.ModTime()) + if runtime.GOOS != "windows" { + assert.Equal(t, 0560, int(fi.Mode()&0xFFF)) + assert.Equal(t, 0, int(fi.Mode()&os.ModeSetuid)) + } else { + assert.Equal(t, 0666, int(fi.Mode()&0xFFF)) + } + case 4: // file3 + assert.Equal(t, tm.Add(10*time.Second), fi.ModTime()) + if runtime.GOOS != "windows" { + assert.Equal(t, 0540, int(fi.Mode()&0xFFF)) + assert.Equal(t, os.ModeSetgid, fi.Mode()&os.ModeSetgid) + assert.Equal(t, os.ModeSetuid, fi.Mode()&os.ModeSetuid) + } else { + assert.Equal(t, 0444, int(fi.Mode()&0xFFF)) + } + default: + assert.Fail(t, "has more than 5 entries", path) + } + walkIndex++ + return nil + }) + assert.NoError(t, err) + }, + nil) +} + +func TestSymlinkWithModTime(t *testing.T) { + if !symlinksEnabled { + t.Skip("symlinks disabled on this platform", symlinksEnabledErr) + } + if runtime.GOOS == "darwin" { + t.Skip("changing symlink modification time is not currently supported on darwin") + } + tm := time.Unix(660000000, 0) + add5 := func() time.Time { + tm = tm.Add(5 * time.Second) + return tm + } + + entries := []tarEntry{ + &dirTarEntry{path: "root"}, + &symlinkTarEntry{target: "child", path: "root/a", mtime: add5()}, + &dirTarEntry{path: "root/child"}, + &fileTarEntry{path: "root/child/file1", buf: []byte("data")}, + &symlinkTarEntry{target: "child/file1", path: "root/file1-sl", mtime: add5()}, + } + + testTarExtraction(t, nil, entries, func(t *testing.T, extractDir string) { + tm = time.Unix(660000000, 0) + + fi, err := os.Lstat(fp.Join(extractDir, "a")) + assert.NoError(t, err) + add5() + if runtime.GOOS != "windows" { + assert.Equal(t, tm, fi.ModTime()) + } + + fi, err = os.Lstat(fp.Join(extractDir, "file1-sl")) + assert.NoError(t, err) + add5() + if runtime.GOOS != "windows" { + assert.Equal(t, tm, fi.ModTime()) + } + }, + nil) +} + +func TestDeferredUpdate(t *testing.T) { + tm := time.Unix(660000000, 0) + add5 := func() time.Time { + tm = tm.Add(5 * time.Second) + return tm + } + + // must be in lexical order + entries := []tarEntry{ + &dirTarEntry{path: "root", mtime: add5()}, + &dirTarEntry{path: "root/a", mtime: add5()}, + &dirTarEntry{path: "root/a/beta", mtime: add5(), mode: 0500}, + &dirTarEntry{path: "root/a/beta/centauri", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/lima", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/papa", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/xanadu", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/z", mtime: add5()}, + &dirTarEntry{path: "root/a/delta", mtime: add5()}, + &dirTarEntry{path: "root/iota", mtime: add5()}, + &dirTarEntry{path: "root/q", mtime: add5()}, + } + + testTarExtraction(t, nil, entries, func(t *testing.T, extractDir string) { + tm = time.Unix(660000000, 0) + err := fp.Walk(extractDir, + func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + assert.Equal(t, add5(), fi.ModTime()) + return nil + }) + assert.NoError(t, err) + }, + nil) + +} + func TestInternalSymlinkTraverse(t *testing.T) { if !symlinksEnabled { t.Skip("symlinks disabled on this platform", symlinksEnabledErr) @@ -254,10 +450,10 @@ func TestInternalSymlinkTraverse(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ // FIXME: We are ignoring the first element in the path check so // we add a directory at the start to bypass this. - &dirTarEntry{"root"}, - &dirTarEntry{"root/child"}, - &symlinkTarEntry{"child", "root/symlink-dir"}, - &fileTarEntry{"root/symlink-dir/file", []byte("file")}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "root/child"}, + &symlinkTarEntry{target: "child", path: "root/symlink-dir"}, + &fileTarEntry{path: "root/symlink-dir/file", buf: []byte("file")}, }, nil, errTraverseSymlink, @@ -271,9 +467,9 @@ func TestExternalSymlinkTraverse(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ // FIXME: We are ignoring the first element in the path check so // we add a directory at the start to bypass this. - &dirTarEntry{"inner"}, - &symlinkTarEntry{"..", "inner/symlink-dir"}, - &fileTarEntry{"inner/symlink-dir/file", []byte("overwrite content")}, + &dirTarEntry{path: "inner"}, + &symlinkTarEntry{target: "..", path: "inner/symlink-dir"}, + &fileTarEntry{path: "inner/symlink-dir/file", buf: []byte("overwrite content")}, }, nil, errTraverseSymlink, @@ -295,13 +491,14 @@ func TestLastElementOverwrite(t *testing.T) { assert.Equal(t, len(originalData), n) }, []tarEntry{ - &dirTarEntry{"root"}, - &symlinkTarEntry{"../outside-ref", "root/symlink"}, - &fileTarEntry{"root/symlink", []byte("overwrite content")}, + &dirTarEntry{path: "root"}, + &symlinkTarEntry{target: "../outside-ref", path: "root/symlink"}, + &fileTarEntry{path: "root/symlink", buf: []byte("overwrite content")}, }, func(t *testing.T, extractDir string) { - // Check that outside-ref still exists but has not been - // overwritten or truncated (still size the same). + // Check that outside-ref still exists but has not been overwritten + // or truncated (still size the same). The symlink itself have been + // overwritten by the extracted file. info, err := os.Stat(fp.Join(extractDir, "..", "outside-ref")) assert.NoError(t, err) @@ -325,18 +522,12 @@ func testTarExtraction(t *testing.T, setup func(t *testing.T, rootDir string), t err = os.MkdirAll(extractDir, 0o755) assert.NoError(t, err) - // Generated TAR file. - tarFilename := fp.Join(rootDir, "generated.tar") - tarFile, err := os.Create(tarFilename) - assert.NoError(t, err) - defer tarFile.Close() - tw := tar.NewWriter(tarFile) - defer tw.Close() - if setup != nil { setup(t, rootDir) } + // Generated TAR file. + tarFilename := fp.Join(rootDir, "generated.tar") writeTarFile(t, tarFilename, tarEntries) testExtract(t, tarFilename, extractDir, extractError) @@ -358,6 +549,23 @@ func testExtract(t *testing.T, tarFile string, extractDir string, expectedError assert.ErrorIs(t, err, expectedError) } +func testMeta(t *testing.T, path string, mode int, now time.Time) { + fi, err := os.Lstat(path) + assert.NoError(t, err) + m := files.ModePermsToUnixPerms(fi.Mode()) + if runtime.GOOS == "windows" { + if fi.IsDir() { + mode = 0777 + } else if mode&0220 != 0 { + mode = 0666 + } else if mode&0440 != 0 { + mode = 0444 + } + } + assert.Equal(t, mode, int(m)) + assert.Equal(t, now.Unix(), fi.ModTime().Unix()) +} + // Based on the `writeXXXHeader` family of functions in // github.com/ipfs/go-ipfs-files@v0.0.8/tarwriter.go. func writeTarFile(t *testing.T, path string, entries []tarEntry) { @@ -385,16 +593,19 @@ var ( ) type fileTarEntry struct { - path string - buf []byte + path string + buf []byte + mode int + mtime time.Time } func (e *fileTarEntry) write(tw *tar.Writer) error { - if err := writeFileHeader(tw, e.path, uint64(len(e.buf))); err != nil { + err := writeFileHeader(tw, e.path, uint64(len(e.buf)), e.mode, e.mtime) + if err != nil { return err } - if _, err := io.Copy(tw, bytes.NewReader(e.buf)); err != nil { + if _, err = io.Copy(tw, bytes.NewReader(e.buf)); err != nil { return err } @@ -402,34 +613,35 @@ func (e *fileTarEntry) write(tw *tar.Writer) error { return nil } -func writeFileHeader(w *tar.Writer, fpath string, size uint64) error { +func writeFileHeader(w *tar.Writer, fpath string, size uint64, mode int, mtime time.Time) error { return w.WriteHeader(&tar.Header{ Name: fpath, Size: int64(size), Typeflag: tar.TypeReg, - Mode: 0o644, - ModTime: time.Now(), - // TODO: set mode, dates, etc. when added to unixFS + Mode: int64(mode), + ModTime: mtime, }) } type dirTarEntry struct { - path string + path string + mode int + mtime time.Time } func (e *dirTarEntry) write(tw *tar.Writer) error { return tw.WriteHeader(&tar.Header{ Name: e.path, Typeflag: tar.TypeDir, - Mode: 0o777, - ModTime: time.Now(), - // TODO: set mode, dates, etc. when added to unixFS + Mode: int64(e.mode), + ModTime: e.mtime, }) } type symlinkTarEntry struct { target string path string + mtime time.Time } func (e *symlinkTarEntry) write(w *tar.Writer) error { @@ -437,6 +649,7 @@ func (e *symlinkTarEntry) write(w *tar.Writer) error { Name: e.path, Linkname: e.target, Mode: 0o777, + ModTime: e.mtime, Typeflag: tar.TypeSymlink, }) }