Skip to content

Commit

Permalink
feat: support UnixFS 1.5 file mode and modification times (#653)
Browse files Browse the repository at this point in the history
  • Loading branch information
gammazero authored Aug 20, 2024
1 parent 88beadf commit aa27cd2
Show file tree
Hide file tree
Showing 47 changed files with 2,533 additions and 297 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ The following emojis are used to highlight certain changes:

### Added

- `files`, `ipld/unixfs`, `mfs` and `tar` now support optional UnixFS 1.5 mode and modification time metadata

### Changed

### Removed
Expand Down
10 changes: 5 additions & 5 deletions examples/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ module github.com/ipfs/boxo/examples
go 1.21

require (
github.com/ipfs/boxo v0.19.0
github.com/ipfs/boxo v0.22.0
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipld/go-car/v2 v2.13.1
github.com/ipld/go-ipld-prime v0.21.0
github.com/libp2p/go-libp2p v0.36.1
github.com/libp2p/go-libp2p v0.36.2
github.com/libp2p/go-libp2p-routing-helpers v0.7.3
github.com/multiformats/go-multiaddr v0.13.0
github.com/multiformats/go-multicodec v0.9.0
Expand Down Expand Up @@ -122,7 +122,7 @@ require (
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pion/datachannel v1.5.8 // indirect
github.com/pion/dtls/v2 v2.2.12 // indirect
github.com/pion/ice/v2 v2.3.32 // indirect
github.com/pion/ice/v2 v2.3.34 // indirect
github.com/pion/interceptor v0.1.29 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.12 // indirect
Expand All @@ -133,9 +133,9 @@ require (
github.com/pion/sdp/v3 v3.0.9 // indirect
github.com/pion/srtp/v2 v2.0.20 // indirect
github.com/pion/stun v0.6.1 // indirect
github.com/pion/transport/v2 v2.2.9 // indirect
github.com/pion/transport/v2 v2.2.10 // indirect
github.com/pion/turn/v2 v2.1.6 // indirect
github.com/pion/webrtc/v3 v3.2.50 // indirect
github.com/pion/webrtc/v3 v3.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
Expand Down
17 changes: 8 additions & 9 deletions examples/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -271,8 +271,8 @@ github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+
github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg=
github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
github.com/libp2p/go-libp2p v0.36.1 h1:piAHesy0/8ifBEBUS8HF2m7ywR5vnktUFv00dTsVKcs=
github.com/libp2p/go-libp2p v0.36.1/go.mod h1:vHzel3CpRB+vS11fIjZSJAU4ALvieKV9VZHC9VerHj8=
github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U=
github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ=
Expand Down Expand Up @@ -372,8 +372,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI=
github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU=
github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM=
github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M=
github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
Expand All @@ -399,17 +399,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/
github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE=
github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw=
github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s=
github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34=
github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo=
github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down
9 changes: 9 additions & 0 deletions files/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"
"io"
"os"
"time"
)

var (
Expand All @@ -17,6 +18,14 @@ var (
type Node interface {
io.Closer

// Mode returns the mode.
// Optional, if unknown/unspecified returns zero.
Mode() os.FileMode

// ModTime returns the last modification time. If the last
// modification time is unknown/unspecified ModTime returns zero.
ModTime() (mtime time.Time)

// Size returns size of this file (if this file is a directory, total size of
// all files stored in the tree should be returned). Some implementations may
// choose not to implement this
Expand Down
141 changes: 141 additions & 0 deletions files/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package files
import (
"io"
"mime/multipart"
"os"
"strings"
"testing"
"time"
)

func TestSliceFiles(t *testing.T) {
Expand Down Expand Up @@ -49,6 +51,21 @@ func TestReaderFiles(t *testing.T) {
}
}

func TestReaderFileStat(t *testing.T) {
reader := strings.NewReader("beep boop")
mode := os.FileMode(0754)
mtime := time.Date(2020, 11, 2, 12, 27, 35, 55555, time.UTC)
stat := &mockFileInfo{name: "test", mode: mode, mtime: mtime}

rf := NewReaderStatFile(reader, stat)
if rf.Mode() != mode {
t.Fatalf("Expected file mode to be [%v] but got [%v]", mode, rf.Mode())
}
if rf.ModTime() != mtime {
t.Fatalf("Expected file modified time to be [%v] but got [%v]", mtime, rf.ModTime())
}
}

func TestMultipartFiles(t *testing.T) {
data := `
--Boundary!
Expand Down Expand Up @@ -141,3 +158,127 @@ implicit file2
},
})
}

func TestMultipartFilesWithMode(t *testing.T) {
data := `
--Boundary!
Content-Type: text/plain
Content-Disposition: form-data; name="file-0?mode=0754&mtime=1604320500&mtime-nsecs=55555"; filename="%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC"
Some-Header: beep
beep
--Boundary!
Content-Type: application/x-directory
Content-Disposition: form-data; name="dir-0?mode=755&mtime=1604320500"; ans=42; filename="dir1"
--Boundary!
Content-Type: text/plain
Content-Disposition: form-data; name="file"; filename="dir1/nested"
some content
--Boundary!
Content-Type: text/plain
Content-Disposition: form-data; name="file?mode=600"; filename="dir1/nested2"; ans=42
some content
--Boundary!
Content-Type: application/symlink
Content-Disposition: form-data; name="file-5"; filename="dir1/simlynk"
anotherfile
--Boundary!
Content-Type: application/symlink
Content-Disposition: form-data; name="file?mtime=1604320500"; filename="dir1/simlynk2"
anotherfile
--Boundary!
Content-Type: text/plain
Content-Disposition: form-data; name="dir?mode=0644"; filename="implicit1/implicit2/deep_implicit"
implicit file1
--Boundary!
Content-Type: text/plain
Content-Disposition: form-data; name="dir?mode=755&mtime=1604320500"; filename="implicit1/shallow_implicit"
implicit file2
--Boundary!--
`

reader := strings.NewReader(data)
mpReader := multipart.NewReader(reader, "Boundary!")
dir, err := NewFileFromPartReader(mpReader, multipartFormdataType)
if err != nil {
t.Fatal(err)
}

CheckDir(t, dir, []Event{
{
kind: TFile,
name: "£ẞǑǓÆ æ ♫♬",
value: "beep",
mode: 0754,
mtime: time.Unix(1604320500, 55555),
},
{
kind: TDirStart,
name: "dir1",
mode: 0755,
mtime: time.Unix(1604320500, 0),
},
{
kind: TFile,
name: "nested",
value: "some content",
},
{
kind: TFile,
name: "nested2",
value: "some content",
mode: 0600,
},
{
kind: TSymlink,
name: "simlynk",
value: "anotherfile",
mode: 0777,
},
{
kind: TSymlink,
name: "simlynk2",
value: "anotherfile",
mode: 0777,
mtime: time.Unix(1604320500, 0),
},
{
kind: TDirEnd,
},
{
kind: TDirStart,
name: "implicit1",
},
{
kind: TDirStart,
name: "implicit2",
},
{
kind: TFile,
name: "deep_implicit",
value: "implicit file1",
mode: 0644,
},
{
kind: TDirEnd,
},
{
kind: TFile,
name: "shallow_implicit",
value: "implicit file2",
mode: 0755,
mtime: time.Unix(1604320500, 0),
},
{
kind: TDirEnd,
},
})
}
18 changes: 17 additions & 1 deletion files/filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,33 @@ import (
"os"
"path/filepath"
"testing"
"time"
)

type mockFileInfo struct {
os.FileInfo
name string
name string
mode os.FileMode
mtime time.Time
size int64
}

func (m *mockFileInfo) Name() string {
return m.name
}

func (m *mockFileInfo) Mode() os.FileMode {
return m.mode
}

func (m *mockFileInfo) ModTime() time.Time {
return m.mtime
}

func (m *mockFileInfo) Size() int64 {
return m.size
}

func (m *mockFileInfo) Sys() interface{} {
return nil
}
Expand Down
12 changes: 12 additions & 0 deletions files/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package files

import (
"io"
"os"
"testing"
"time"
)

type Kind int
Expand All @@ -18,6 +20,8 @@ type Event struct {
kind Kind
name string
value string
mode os.FileMode
mtime time.Time
}

func CheckDir(t *testing.T, dir Directory, expected []Event) {
Expand Down Expand Up @@ -50,6 +54,14 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) {
t.Fatalf("[%d] expected filename to be %q", i, next.name)
}

if next.mode != 0 && it.Node().Mode()&os.ModePerm != next.mode {
t.Fatalf("[%d] expected mode for '%s' to be %O, got %O", i, it.Name(), next.mode, it.Node().Mode())
}

if !next.mtime.IsZero() && !it.Node().ModTime().Equal(next.mtime) {
t.Fatalf("[%d] expected modification time for '%s' to be %q", i, it.Name(), next.mtime)
}

switch next.kind {
case TFile:
mf, ok := it.Node().(File)
Expand Down
24 changes: 22 additions & 2 deletions files/linkfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,41 @@ package files
import (
"os"
"strings"
"time"
)

type Symlink struct {
Target string

stat os.FileInfo
mtime time.Time
reader strings.Reader
}

func NewLinkFile(target string, stat os.FileInfo) File {
lf := &Symlink{Target: target, stat: stat}
mtime := time.Time{}
if stat != nil {
mtime = stat.ModTime()
}
return NewSymlinkFile(target, mtime)
}

func NewSymlinkFile(target string, mtime time.Time) File {
lf := &Symlink{
Target: target,
mtime: mtime,
}
lf.reader.Reset(lf.Target)
return lf
}

func (lf *Symlink) Mode() os.FileMode {
return os.ModeSymlink | os.ModePerm
}

func (lf *Symlink) ModTime() time.Time {
return lf.mtime
}

func (lf *Symlink) Close() error {
return nil
}
Expand Down
Loading

0 comments on commit aa27cd2

Please sign in to comment.