From a549705fdac7520be22f715caae3b87f7730877f Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 19 Aug 2023 20:24:19 +0100 Subject: [PATCH] Added differences for root leaves --- dag/dag.go | 208 ++++++++++++++++++++++++++++++++++------- dag/dag_test.go | 27 +++--- dag/leaves.go | 240 +++++++++++++++++++++++++++++++++++------------- dag/testing.go | 13 ++- dag/types.go | 13 ++- 5 files changed, 384 insertions(+), 117 deletions(-) diff --git a/dag/dag.go b/dag/dag.go index 6ac9f60..9b88dfd 100644 --- a/dag/dag.go +++ b/dag/dag.go @@ -80,15 +80,17 @@ func CreateDag(path string, encoding ...multibase.Encoding) (*Dag, error) { return nil, err } - leaf, err := CheckMetaFile(path, dag, encoder) + metaLeaf, err := CheckMetaFile(path, dag, encoder) if err != nil { log.Println("Failed to check meta file") } else { - builder.AddLink("0", leaf.Hash) - leaf.SetLabel("0") - dag.AddLeaf(leaf, encoder, nil) + builder.AddLink("0", metaLeaf.Hash) + metaLeaf.SetLabel("0") + dag.AddLeaf(metaLeaf, encoder, nil) } + dag.AddLeaf(metaLeaf, encoder, nil) + for _, entry := range entries { if entry.Name() != ".meta" { leaf, err := processEntry(entry, &path, dag, encoder) @@ -96,23 +98,25 @@ func CreateDag(path string, encoding ...multibase.Encoding) (*Dag, error) { return nil, err } - label := builder.GetNextAvailableLabel() + label := dag.GetNextAvailableLabel() builder.AddLink(label, leaf.Hash) leaf.SetLabel(label) dag.AddLeaf(leaf, encoder, nil) } } - leaf, err = builder.BuildLeaf(encoder) - + leaf, err := builder.BuildRootLeaf(encoder, len(dag.Leafs)) if err != nil { return nil, err } dag.AddLeaf(leaf, encoder, nil) - rootHash := leaf.Hash - return dag.BuildDag(rootHash), nil + builtDag := dag.BuildDag(leaf.Hash) + + builtDag.GenerateLabelMap() + + return builtDag, nil } func processMetaFile(path *string, basePath *string, dag *DagBuilder, encoder multibase.Encoder) (*DagLeaf, error) { @@ -147,7 +151,7 @@ func processMetaFile(path *string, basePath *string, dag *DagBuilder, encoder mu return nil, err } - label := builder.GetNextAvailableLabel() + label := dag.GetNextAvailableLabel() builder.AddLink(label, chunkLeaf.Hash) chunkLeaf.SetLabel(label) dag.AddLeaf(chunkLeaf, encoder, nil) @@ -180,15 +184,6 @@ func processEntry(entry fs.FileInfo, path *string, dag *DagBuilder, encoder mult return nil, err } - leaf, err := CheckMetaFile(entryPath, dag, encoder) - if err != nil { - log.Println("Failed to check meta file") - } else { - builder.AddLink("0", leaf.Hash) - leaf.SetLabel("0") - dag.AddLeaf(leaf, encoder, nil) - } - for _, entry := range entries { if entry.Name() != ".meta" { leaf, err := processEntry(entry, &entryPath, dag, encoder) @@ -196,7 +191,7 @@ func processEntry(entry fs.FileInfo, path *string, dag *DagBuilder, encoder mult return nil, err } - label := builder.GetNextAvailableLabel() + label := dag.GetNextAvailableLabel() builder.AddLink(label, leaf.Hash) leaf.SetLabel(label) dag.AddLeaf(leaf, encoder, nil) @@ -227,7 +222,7 @@ func processEntry(entry fs.FileInfo, path *string, dag *DagBuilder, encoder mult return nil, err } - label := builder.GetNextAvailableLabel() + label := dag.GetNextAvailableLabel() builder.AddLink(label, chunkLeaf.Hash) chunkLeaf.SetLabel(label) dag.AddLeaf(chunkLeaf, encoder, nil) @@ -260,7 +255,8 @@ func chunkFile(fileData []byte, chunkSize int) [][]byte { func CreateDagBuilder() *DagBuilder { return &DagBuilder{ - Leafs: map[string]*DagLeaf{}, + Labels: map[string]string{}, + Leafs: map[string]*DagLeaf{}, } } @@ -271,6 +267,8 @@ func (b *DagBuilder) AddLeaf(leaf *DagLeaf, encoder multibase.Encoder, parentLea if !exists { parentLeaf.AddLink(leaf.Hash) } + + b.Labels[label] = leaf.Hash } b.Leafs[leaf.Hash] = leaf @@ -280,22 +278,34 @@ func (b *DagBuilder) AddLeaf(leaf *DagLeaf, encoder multibase.Encoder, parentLea func (b *DagBuilder) BuildDag(root string) *Dag { return &Dag{ - Leafs: b.Leafs, - Root: root, + Leafs: b.Leafs, + Labels: b.Labels, + Root: root, } } func (dag *Dag) Verify(encoder multibase.Encoder) (bool, error) { result := true - for _, leaf := range dag.Leafs { - leafResult, err := leaf.VerifyLeaf(encoder) - if err != nil { - return false, err - } + for hash, leaf := range dag.Leafs { + if hash == dag.Root { + leafResult, err := leaf.VerifyRootLeaf(encoder) + if err != nil { + return false, err + } + + if !leafResult { + return false, nil + } + } else { + leafResult, err := leaf.VerifyLeaf(encoder) + if err != nil { + return false, err + } - if !leafResult { - result = false + if !leafResult { + return false, nil + } } } @@ -332,7 +342,8 @@ func (dag *Dag) CreateDirectory(path string, encoder multibase.Encoder) error { /* if runtime.GOOS == "windows" { - p, err := syscall.UTF16PtrFromString(fileName) + p, err := syscal + l.UTF16PtrFromString(fileName) if err != nil { log.Fatal(err) } @@ -363,9 +374,40 @@ func ReadDag(path string) (*Dag, error) { return nil, fmt.Errorf("could not decode Dag: %w", err) } + result.GenerateLabelMap() + return &result, nil } +func (dag *Dag) GenerateLabelMap() { + dag.Labels = map[string]string{} + + for hash, _ := range dag.Leafs { + label := GetLabel(hash) + + dag.Labels[label] = hash + } +} + +func (dag *Dag) FindLeafByLabel(label string) *DagLeaf { + hash, exists := dag.Labels[label] + + fmt.Println("Find leaf: " + label) + + if exists { + fmt.Println("Found leaf: " + hash) + + leaf, result := dag.Leafs[hash] + if result { + return leaf + } else { + return nil + } + } else { + return nil + } +} + func (dag *Dag) FindLeafByHash(hash string) *DagLeaf { leaf, result := dag.Leafs[hash] if result { @@ -398,7 +440,13 @@ func (dag *Dag) DeleteLeaf(leaf *DagLeaf, encoder multibase.Encoder) error { return fmt.Errorf("Parent leaf does not exist") } - err := parentLeaf.RemoveLink(GetLabel(leaf.Hash)) + label := GetLabel(leaf.Hash) + + if label == "0" { + return fmt.Errorf("Unable to delete leaf 0") + } + + err := parentLeaf.RemoveLink(label) if err != nil { return err } @@ -407,11 +455,18 @@ func (dag *Dag) DeleteLeaf(leaf *DagLeaf, encoder multibase.Encoder) error { delete(dag.Leafs, leaf.Hash) + err = dag.WriteToMetaLeaf(label, encoder) + if err != nil { + return err + } + root, err := dag.RegenerateDag(parentLeaf, encoder) if err != nil { return err } + dag.GenerateLabelMap() + dag.Root = *root return nil @@ -503,7 +558,92 @@ func (dag *Dag) RemoveChildren(leaf *DagLeaf, encoder multibase.Encoder) { } } -// There must be a better way, but this is all I could think of doing at the time +func (dag *Dag) AddLeaf(leaf *DagLeaf) { + dag.Leafs[leaf.Hash] = leaf +} + +func (dag *Dag) GetDataFromLeaf(leaf *DagLeaf) ([]byte, error) { + if len(leaf.Data) <= 0 { + return []byte{}, nil + } + + var content []byte + + if len(leaf.Links) > 0 { + for _, link := range leaf.Links { + childLeaf := dag.Leafs[link] + if childLeaf == nil { + return nil, fmt.Errorf("invalid link: %s", link) + } + + content = append(content, childLeaf.Data...) + } + } else { + content = leaf.Data + } + + return content, nil +} + +func (dag *Dag) WriteToMetaLeaf(label string, encoder multibase.Encoder) error { + leaf := dag.FindLeafByLabel("0") + + if leaf == nil { + return fmt.Errorf("Meta leaf does not exist") + } + + content, err := dag.GetDataFromLeaf(leaf) + if err != nil { + return err + } + + builder := CreateDagLeafBuilder(".meta") + + builder.SetType(FileLeafType) + + fileChunks := chunkFile(content, ChunkSize) + + if len(fileChunks) == 1 { + builder.SetData(fileChunks[0]) + } else { + for i, chunk := range fileChunks { + chunkEntryPath := filepath.Join(".meta", strconv.Itoa(i)) + chunkBuilder := CreateDagLeafBuilder(chunkEntryPath) + + chunkBuilder.SetType(ChunkLeafType) + chunkBuilder.SetData(chunk) + + chunkLeaf, err := chunkBuilder.BuildLeaf(encoder) + if err != nil { + return err + } + + label := dag.GetNextAvailableLabel() + builder.AddLink(label, chunkLeaf.Hash) + chunkLeaf.SetLabel(label) + dag.AddLeaf(chunkLeaf) + } + } + + metaLeaf, err := builder.BuildLeaf(encoder) + if err != nil { + return err + } + + rootLeaf := dag.Leafs[dag.Root] + + if err != nil { + log.Println("Failed to update meta leaf") + } else { + metaLeaf.SetLabel("0") + rootLeaf.AddLink(metaLeaf.Hash) + + dag.AddLeaf(metaLeaf) + } + + return nil +} + func (dag *Dag) DoesExistMoreThanOnce(hash string) bool { count := 0 diff --git a/dag/dag_test.go b/dag/dag_test.go index 2ab3d1c..4ca03aa 100644 --- a/dag/dag_test.go +++ b/dag/dag_test.go @@ -87,13 +87,13 @@ func TestPartial(t *testing.T) { parentLeaf.Links = map[string]string{} // Verify the root leaf - result, err := parentLeaf.VerifyLeaf(encoder) + result, err := parentLeaf.VerifyRootLeaf(encoder) if err != nil { t.Fatal("Failed to verify branch for random leaf") } if !result { - t.Fatal("Root leaf verified correctly") + t.Fatal("Root leaf failed to verify") } // Create a new dag builder and add the root leaf @@ -108,7 +108,7 @@ func TestPartial(t *testing.T) { } // Now retrieve a random child of the parent leaf from the original dag to simulate branch verification - randomLeaf := FindRandomChild(originalParentLeaf, dag.Leafs, encoder) + randomLeaf := originalParentLeaf.FindRandomChild(dag.Leafs, encoder) randomLeaf = randomLeaf.Clone() // Remove the links as the leaf probably wouldn't have them @@ -127,17 +127,20 @@ func TestPartial(t *testing.T) { // Retrieve the branch for random child branch, err := originalParentLeaf.GetBranch(GetLabel(randomLeaf.Hash)) //index if err != nil { - t.Fatal("Failed to retrieve root leaf branch") + t.Errorf("Failed to retrieve root leaf branch: %v", err) + //t.Fatal("Failed to retrieve root leaf branch") } - // Verify the branch before adding the leaf to the dag - result, err = parentLeaf.VerifyBranch(branch) - if err != nil { - t.Fatal("Failed to verify branch for random leaf") - } + if branch != nil { + // Verify the branch before adding the leaf to the dag + result, err = parentLeaf.VerifyBranch(branch) + if err != nil { + t.Fatal("Failed to verify branch for random leaf") + } - if !result { - t.Fatal("Branch verified correctly") + if !result { + t.Fatal("Branch verified correctly") + } } // Add the leaf to the dag builder @@ -212,7 +215,7 @@ func TestDelete(t *testing.T) { } rootLeaf := dag.Leafs[dag.Root] - randomLeaf := FindRandomChild(rootLeaf, dag.Leafs, encoder) + randomLeaf := rootLeaf.FindRandomChild(dag.Leafs, encoder) err = dag.DeleteLeaf(randomLeaf, encoder) if err != nil { diff --git a/dag/leaves.go b/dag/leaves.go index 44893e5..788f70a 100644 --- a/dag/leaves.go +++ b/dag/leaves.go @@ -65,24 +65,57 @@ func (b *DagLeafBuilder) GetLatestLabel() string { return result } -func (b *DagLeafBuilder) GetNextAvailableLabel() string { - latestLabel := b.GetLatestLabel() +func (b *DagLeafBuilder) BuildLeaf(encoder multibase.Encoder) (*DagLeaf, error) { + if b.LeafType == "" { + err := fmt.Errorf("Leaf must have a type defined") + return nil, err + } - number, err := strconv.ParseInt(latestLabel, 10, 64) + merkleRoot := []byte{} + + if len(b.Links) > 1 { + builder := tree.CreateTree() + for _, link := range b.Links { + builder.AddLeaf(GetLabel(link), link) + } + + merkleTree, _, err := builder.Build() + if err != nil { + return nil, err + } + + merkleRoot = merkleTree.Root + } + + leafData := &DagLeaf{ + Name: b.Name, + Type: b.LeafType, + MerkleRoot: merkleRoot, + CurrentLinkCount: len(b.Links), + Data: b.Data, + } + + serializedLeafData, err := cbor.Marshal(leafData) if err != nil { - fmt.Println("Failed to parse label") + return nil, err } - nextLabel := strconv.FormatInt(number+1, 10) + hash := sha256.Sum256(serializedLeafData) + result := &DagLeaf{ + Hash: encoder.Encode(hash[:]), + Name: b.Name, + Type: b.LeafType, + MerkleRoot: merkleRoot, + CurrentLinkCount: len(b.Links), + Data: b.Data, + Links: b.Links, + } - return nextLabel + return result, nil } -func (b *DagLeafBuilder) BuildLeaf(encoder multibase.Encoder) (*DagLeaf, error) { - if b.LeafType == "" { - err := fmt.Errorf("Leaf must have a type defined") - return nil, err - } +func (b *DagLeafBuilder) BuildRootLeaf(encoder multibase.Encoder, leafCount int) (*DagLeaf, error) { + b.LeafType = DirectoryLeafType merkleRoot := []byte{} @@ -102,20 +135,14 @@ func (b *DagLeafBuilder) BuildLeaf(encoder multibase.Encoder) (*DagLeaf, error) latestLabel := b.GetLatestLabel() - leafData := struct { - Name string - Type LeafType - MerkleRoot []byte - CurrentLinkCount int - LatestLabel string - Data []byte - }{ + leafData := &DagLeaf{ Name: b.Name, Type: b.LeafType, MerkleRoot: merkleRoot, CurrentLinkCount: len(b.Links), - LatestLabel: latestLabel, Data: b.Data, + LeafCount: leafCount, + LatestLabel: latestLabel, } serializedLeafData, err := cbor.Marshal(leafData) @@ -130,9 +157,10 @@ func (b *DagLeafBuilder) BuildLeaf(encoder multibase.Encoder) (*DagLeaf, error) Type: b.LeafType, MerkleRoot: merkleRoot, CurrentLinkCount: len(b.Links), - LatestLabel: latestLabel, Data: b.Data, Links: b.Links, + LeafCount: leafCount, + LatestLabel: latestLabel, } return result, nil @@ -145,31 +173,35 @@ type kv struct { } func (leaf *DagLeaf) GetBranch(key string) (*ClassicTreeBranch, error) { - t := merkle_tree.CreateTree() + if len(leaf.Links) > 1 { + t := merkle_tree.CreateTree() - for k, v := range leaf.Links { - t.AddLeaf(k, v) - } + for k, v := range leaf.Links { + t.AddLeaf(k, v) + } - merkleTree, leafs, err := t.Build() - if err != nil { - log.Println("Failed to build merkle tree") - return nil, err - } + merkleTree, leafs, err := t.Build() + if err != nil { + log.Println("Failed to build merkle tree") + return nil, err + } - index, result := merkleTree.GetIndexForKey(key) - if !result { - return nil, fmt.Errorf("Unable to find index for given key") - } + index, result := merkleTree.GetIndexForKey(key) + if !result { + return nil, fmt.Errorf("Unable to find index for given key") + } - branchLeaf := leafs[key] + branchLeaf := leafs[key] - branch := &ClassicTreeBranch{ - Leaf: &branchLeaf, - Proof: merkleTree.Proofs[index], + branch := &ClassicTreeBranch{ + Leaf: &branchLeaf, + Proof: merkleTree.Proofs[index], + } + + return branch, nil } - return branch, nil + return nil, nil } func (leaf *DagLeaf) VerifyBranch(branch *ClassicTreeBranch) (bool, error) { @@ -182,19 +214,11 @@ func (leaf *DagLeaf) VerifyBranch(branch *ClassicTreeBranch) (bool, error) { } func (leaf *DagLeaf) VerifyLeaf(encoder multibase.Encoder) (bool, error) { - leafData := struct { - Name string - Type LeafType - MerkleRoot []byte - CurrentLinkCount int - LatestLabel string - Data []byte - }{ + leafData := &DagLeaf{ Name: leaf.Name, Type: leaf.Type, MerkleRoot: leaf.MerkleRoot, CurrentLinkCount: leaf.CurrentLinkCount, - LatestLabel: leaf.LatestLabel, Data: leaf.Data, } @@ -234,6 +258,53 @@ func (leaf *DagLeaf) VerifyLeaf(encoder multibase.Encoder) (bool, error) { return result, nil } +func (leaf *DagLeaf) VerifyRootLeaf(encoder multibase.Encoder) (bool, error) { + leafData := &DagLeaf{ + Name: leaf.Name, + Type: leaf.Type, + MerkleRoot: leaf.MerkleRoot, + CurrentLinkCount: leaf.CurrentLinkCount, + Data: leaf.Data, + LatestLabel: leaf.LatestLabel, + LeafCount: leaf.LeafCount, + } + + serializedLeafData, err := cbor.Marshal(leafData) + if err != nil { + return false, err + } + + hash := sha256.Sum256(serializedLeafData) + + var result bool = false + if HasLabel(leaf.Hash) { + result = encoder.Encode(hash[:]) == GetHash(leaf.Hash) + } else { + result = encoder.Encode(hash[:]) == leaf.Hash + } + + /* + // Should this be here? + if leaf.MerkleRoot != "" { + for i := 0; i < len(leaf.Links); i++ { + branch, err := leaf.GetBranch(i) + if err != nil { + log.Println("Failed to get leaf branch at index: ", i) + return false, err + } + + branchResult, err := leaf.VerifyBranch(branch) + + if !branchResult { + result = false + } + } + } + */ + + return result, nil +} + func (leaf *DagLeaf) CreateDirectoryLeaf(path string, dag *Dag, encoder multibase.Encoder) error { switch leaf.Type { case DirectoryLeafType: @@ -323,37 +394,82 @@ func (leaf *DagLeaf) Clone() *DagLeaf { Data: leaf.Data, MerkleRoot: leaf.MerkleRoot, CurrentLinkCount: leaf.CurrentLinkCount, + LeafCount: leaf.LeafCount, LatestLabel: leaf.LatestLabel, Links: leaf.Links, + ParentHash: leaf.ParentHash, } } -func (leaf *DagLeaf) GetLatestLabel() string { +func (builder *DagBuilder) GetLatestLabel() string { var result string = "0" var latestLabel int64 = 0 - for _, hash := range leaf.Links { - label := GetLabel(hash) - if label == "" { - fmt.Println("Failed to find label in hash") - } + for _, leaf := range builder.Leafs { + for _, hash := range leaf.Links { + label := GetLabel(hash) - parsed, err := strconv.ParseInt(label, 10, 64) - if err != nil { - fmt.Println("Failed to parse label") + if label == "" { + fmt.Println("Failed to find label in hash") + } + + parsed, err := strconv.ParseInt(label, 10, 64) + if err != nil { + fmt.Println("Failed to parse label") + } + + if parsed > latestLabel { + latestLabel = parsed + result = label + } } + } - if parsed > latestLabel { - latestLabel = parsed - result = label + return result +} + +func (dag *Dag) GetLatestLabel() string { + var result string = "0" + var latestLabel int64 = 0 + + for _, leaf := range dag.Leafs { + for _, hash := range leaf.Links { + label := GetLabel(hash) + + if label == "" { + fmt.Println("Failed to find label in hash") + } + + parsed, err := strconv.ParseInt(label, 10, 64) + if err != nil { + fmt.Println("Failed to parse label") + } + + if parsed > latestLabel { + latestLabel = parsed + result = label + } } } return result } -func (leaf *DagLeaf) GetNextAvailableLabel() string { - latestLabel := leaf.GetLatestLabel() +func (dag *DagBuilder) GetNextAvailableLabel() string { + latestLabel := dag.GetLatestLabel() + + number, err := strconv.ParseInt(latestLabel, 10, 64) + if err != nil { + fmt.Println("Failed to parse label") + } + + nextLabel := strconv.FormatInt(number+1, 10) + + return nextLabel +} + +func (dag *Dag) GetNextAvailableLabel() string { + latestLabel := dag.GetLatestLabel() number, err := strconv.ParseInt(latestLabel, 10, 64) if err != nil { diff --git a/dag/testing.go b/dag/testing.go index 73ba653..c3d20b3 100644 --- a/dag/testing.go +++ b/dag/testing.go @@ -52,7 +52,7 @@ func createRandomDirsAndFiles(path string, depth int, maxItems int) error { return nil } -func FindRandomChild(leaf *DagLeaf, leafs map[string]*DagLeaf, encoder multibase.Encoder) *DagLeaf { +func (leaf *DagLeaf) FindRandomChild(leafs map[string]*DagLeaf, encoder multibase.Encoder) *DagLeaf { if leaf.Type == DirectoryLeafType { rand.Seed(time.Now().UnixNano()) index := rand.Intn(len(leaf.Links)) @@ -62,15 +62,24 @@ func FindRandomChild(leaf *DagLeaf, leafs map[string]*DagLeaf, encoder multibase curIndex := 1 for _, link := range leaf.Links { if curIndex >= index { - newLeaf = leafs[link] + new := leafs[link] + + if GetLabel(new.Hash) != "0" { + newLeaf = new + break + } } curIndex++ } + fmt.Println("Returning new leaf") + return newLeaf } + fmt.Println("Returning original leaf") + return leaf } diff --git a/dag/types.go b/dag/types.go index af04b61..ed011cc 100644 --- a/dag/types.go +++ b/dag/types.go @@ -15,12 +15,14 @@ const ( ) type Dag struct { - Root string - Leafs map[string]*DagLeaf + Root string + Labels map[string]string + Leafs map[string]*DagLeaf } type DagBuilder struct { - Leafs map[string]*DagLeaf + Labels map[string]string + Leafs map[string]*DagLeaf } type DagLeaf struct { @@ -30,6 +32,7 @@ type DagLeaf struct { Data []byte MerkleRoot []byte CurrentLinkCount int + LeafCount int LatestLabel string Links map[string]string ParentHash string @@ -48,10 +51,6 @@ type ClassicTreeBranch struct { Proof *merkletree.Proof } -type MetaData struct { - Deleted []string -} - func SetChunkSize(size int) { ChunkSize = size }