Skip to content

Commit

Permalink
feat: now supports single file dag creation
Browse files Browse the repository at this point in the history
  • Loading branch information
f7f376a1fcd0d0e11a10ed1b6577c9 committed Feb 11, 2024
1 parent 5cbce8b commit 9eab9f7
Showing 1 changed file with 130 additions and 60 deletions.
190 changes: 130 additions & 60 deletions dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,34 @@ import (
"github.com/multiformats/go-multibase"
)

type fileInfoDirEntry struct {
fileInfo os.FileInfo
}

func (e fileInfoDirEntry) Name() string {
return e.fileInfo.Name()
}

func (e fileInfoDirEntry) IsDir() bool {
return e.fileInfo.IsDir()
}

func (e fileInfoDirEntry) Type() fs.FileMode {
return e.fileInfo.Mode().Type()
}

func (e fileInfoDirEntry) Info() (fs.FileInfo, error) {
return e.fileInfo, nil
}

func newDirEntry(path string) (fs.DirEntry, error) {
fileInfo, err := os.Stat(path)
if err != nil {
return nil, err
}
return fileInfoDirEntry{fileInfo: fileInfo}, nil
}

func CreateDag(path string, encoding ...multibase.Encoding) (*Dag, error) {
var e multibase.Encoding
if len(encoding) > 0 {
Expand All @@ -24,30 +52,25 @@ func CreateDag(path string, encoding ...multibase.Encoding) (*Dag, error) {

dag := CreateDagBuilder()

relPath := filepath.Base(path)
builder := CreateDagLeafBuilder(relPath)
builder.SetType(DirectoryLeafType)
fileInfo, err := os.Stat(path)
if err != nil {
return nil, err
}

entries, err := ioutil.ReadDir(path)
dirEntry, err := newDirEntry(path)
if err != nil {
return nil, err
}

for _, entry := range entries {
if entry.Name() != ".meta" {
leaf, err := processEntry(entry, &path, dag, encoder)
if err != nil {
return nil, err
}
parentPath := filepath.Dir(path)

label := dag.GetNextAvailableLabel()
builder.AddLink(label, leaf.Hash)
leaf.SetLabel(label)
dag.AddLeaf(leaf, encoder, nil)
}
}
var leaf *DagLeaf

leaf, err := builder.BuildRootLeaf(dag, encoder)
if fileInfo.IsDir() {
leaf, err = processDirectory(dirEntry, &parentPath, dag, encoder, true)
} else {
leaf, err = processFile(dirEntry, &parentPath, dag, encoder, true)
}

if err != nil {
return nil, err
Expand All @@ -59,7 +82,24 @@ func CreateDag(path string, encoding ...multibase.Encoding) (*Dag, error) {
return dag.BuildDag(rootHash), nil
}

func processEntry(entry fs.FileInfo, path *string, dag *DagBuilder, encoder multibase.Encoder) (*DagLeaf, error) {
func processEntry(entry fs.DirEntry, path *string, dag *DagBuilder, encoder multibase.Encoder) (*DagLeaf, error) {
var result *DagLeaf
var err error

if entry.IsDir() {
result, err = processDirectory(entry, path, dag, encoder, false)
} else {
result, err = processFile(entry, path, dag, encoder, false)
}

if err != nil {
return nil, err
}

return result, nil
}

func processDirectory(entry fs.DirEntry, path *string, dag *DagBuilder, encoder multibase.Encoder, isRoot bool) (*DagLeaf, error) {
entryPath := filepath.Join(*path, entry.Name())

relPath, err := filepath.Rel(*path, entryPath)
Expand All @@ -69,61 +109,91 @@ func processEntry(entry fs.FileInfo, path *string, dag *DagBuilder, encoder mult

builder := CreateDagLeafBuilder(relPath)

if entry.IsDir() {
builder.SetType(DirectoryLeafType)
builder.SetType(DirectoryLeafType)

entries, err := os.ReadDir(entryPath)
if err != nil {
return nil, err
}

var result *DagLeaf

entries, err := ioutil.ReadDir(entryPath)
for _, entry := range entries {
leaf, err := processEntry(entry, &entryPath, dag, encoder)
if err != nil {
return nil, err
}

for _, entry := range entries {
if entry.Name() != ".meta" {
leaf, err := processEntry(entry, &entryPath, dag, encoder)
if err != nil {
return nil, err
}

label := dag.GetNextAvailableLabel()
builder.AddLink(label, leaf.Hash)
leaf.SetLabel(label)
dag.AddLeaf(leaf, encoder, nil)
}
}
label := dag.GetNextAvailableLabel()
builder.AddLink(label, leaf.Hash)
leaf.SetLabel(label)
dag.AddLeaf(leaf, encoder, nil)
}

if isRoot {
result, err = builder.BuildRootLeaf(dag, encoder)
} else {
fileData, err := ioutil.ReadFile(entryPath)
if err != nil {
return nil, err
}
result, err = builder.BuildLeaf(encoder)
}

builder.SetType(FileLeafType)
if err != nil {
return nil, err
}

fileChunks := chunkFile(fileData, ChunkSize)
return result, nil
}

if len(fileChunks) == 1 {
builder.SetData(fileChunks[0])
} else {
for i, chunk := range fileChunks {
chunkEntryPath := filepath.Join(relPath, strconv.Itoa(i))
chunkBuilder := CreateDagLeafBuilder(chunkEntryPath)

chunkBuilder.SetType(ChunkLeafType)
chunkBuilder.SetData(chunk)

chunkLeaf, err := chunkBuilder.BuildLeaf(encoder)
if err != nil {
return nil, err
}

label := dag.GetNextAvailableLabel()
builder.AddLink(label, chunkLeaf.Hash)
chunkLeaf.SetLabel(label)
dag.AddLeaf(chunkLeaf, encoder, nil)
func processFile(entry fs.DirEntry, path *string, dag *DagBuilder, encoder multibase.Encoder, isRoot bool) (*DagLeaf, error) {
entryPath := filepath.Join(*path, entry.Name())

relPath, err := filepath.Rel(*path, entryPath)
if err != nil {
return nil, err
}

var result *DagLeaf

builder := CreateDagLeafBuilder(relPath)

builder.SetType(FileLeafType)

fileData, err := os.ReadFile(entryPath)
if err != nil {
return nil, err
}

builder.SetType(FileLeafType)

fileChunks := chunkFile(fileData, ChunkSize)

if len(fileChunks) == 1 {
builder.SetData(fileChunks[0])
} else {
for i, chunk := range fileChunks {
chunkEntryPath := filepath.Join(relPath, strconv.Itoa(i))
chunkBuilder := CreateDagLeafBuilder(chunkEntryPath)

chunkBuilder.SetType(ChunkLeafType)
chunkBuilder.SetData(chunk)

chunkLeaf, err := chunkBuilder.BuildLeaf(encoder)
if err != nil {
return nil, err
}

label := dag.GetNextAvailableLabel()
builder.AddLink(label, chunkLeaf.Hash)
chunkLeaf.SetLabel(label)
dag.AddLeaf(chunkLeaf, encoder, nil)
}
}

result, err := builder.BuildLeaf(encoder)
if isRoot {
result, err = builder.BuildRootLeaf(dag, encoder)
} else {
result, err = builder.BuildLeaf(encoder)
}

if err != nil {
return nil, err
}
Expand Down

0 comments on commit 9eab9f7

Please sign in to comment.