From 03e0e182812bbe1053e4fb9ce73b63f2638248d5 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Tue, 25 Jul 2017 23:08:25 +0200 Subject: [PATCH 01/19] README: fix the example --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index db36a1c..fb044c8 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ package main import ( "crypto/sha256" - "log" "os" "github.com/NebulousLabs/merkletree" @@ -38,7 +37,7 @@ func main() { file.Seek(0, 0) // Offset needs to be set back to 0. proofIndex := uint64(7) merkleRoot, proof, numLeaves, _ := merkletree.BuildReaderProof(file, sha256.New(), segmentSize, proofIndex) - verified := VerifyProof(sha256.New(), merkleRoot, proof, proofIndex, numLeaves) + verified := merkletree.VerifyProof(sha256.New(), merkleRoot, proof, proofIndex, numLeaves) // Example 3: Using a Tree to build a merkle tree and get a proof for a // specific index for non-file objects. @@ -48,7 +47,7 @@ func main() { tree.Push([]byte("another object")) // The merkle root could be obtained by calling tree.Root(), but will also // be provided by tree.Prove() - merkleRoot, proof, proofIndex, numLeaves := tree.Prove() + merkleRoot, proof, proofIndex, numLeaves = tree.Prove() //////////////////////////////////////////////// /// Remaining examples deal with cached trees // @@ -96,6 +95,11 @@ func main() { // Now we can create the full proof for the cached tree, without having to // rehash any of the elements from subtree1. _, fullProof, _, _ := cachedTree.Prove(subtreeProof) + + _ = verified + _ = collectiveRoot + _ = revisedRoot + _ = fullProof } ``` From db36ccbf12f6ebc867c74d11391536b7b7f6cb5b Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Wed, 26 Jul 2017 23:08:25 +0200 Subject: [PATCH 02/19] fix mistakes in comments in tree_test --- tree_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tree_test.go b/tree_test.go index dab1320..9ad63b9 100644 --- a/tree_test.go +++ b/tree_test.go @@ -18,7 +18,7 @@ type MerkleTester struct { // leaves is the hashes of the data, and should be the same length. leaves [][]byte - // roots contains the root hashes of Merkle trees of various heights using + // roots contains the root hashes of Merkle trees of various lengths using // the data for input. roots map[int][]byte @@ -424,7 +424,7 @@ func TestCompatibility(t *testing.T) { // Check that proofs on larger trees are consistent. for i := 0; i < 25; i++ { - // Determine a random size for the tree up to 64M elements. + // Determine a random size for the tree up to 256k elements. sizeI, err := rand.Int(rand.Reader, big.NewInt(256e3)) if err != nil { t.Fatal(err) From 7d102da611e2ac53bea85cd86415a9e957dbcc55 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Thu, 27 Jul 2017 23:08:25 +0200 Subject: [PATCH 03/19] tree_test: add large manual example --- tree_test.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/tree_test.go b/tree_test.go index 9ad63b9..4f3cc19 100644 --- a/tree_test.go +++ b/tree_test.go @@ -47,7 +47,7 @@ func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { mt.T = t // Fill out the data and leaves values. - size := 16 + size := 100 for i := 0; i < size; i++ { mt.data = append(mt.data, []byte{byte(i)}) } @@ -110,6 +110,155 @@ func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { ), ) + mt.roots[100] = mt.join( + mt.join( + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + mt.join( + mt.join(mt.leaves[4], mt.leaves[5]), + mt.join(mt.leaves[6], mt.leaves[7]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[8], mt.leaves[9]), + mt.join(mt.leaves[10], mt.leaves[11]), + ), + mt.join( + mt.join(mt.leaves[12], mt.leaves[13]), + mt.join(mt.leaves[14], mt.leaves[15]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[16], mt.leaves[17]), + mt.join(mt.leaves[18], mt.leaves[19]), + ), + mt.join( + mt.join(mt.leaves[20], mt.leaves[21]), + mt.join(mt.leaves[22], mt.leaves[23]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[24], mt.leaves[25]), + mt.join(mt.leaves[26], mt.leaves[27]), + ), + mt.join( + mt.join(mt.leaves[28], mt.leaves[29]), + mt.join(mt.leaves[30], mt.leaves[31]), + ), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[32], mt.leaves[33]), + mt.join(mt.leaves[34], mt.leaves[35]), + ), + mt.join( + mt.join(mt.leaves[36], mt.leaves[37]), + mt.join(mt.leaves[38], mt.leaves[39]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[40], mt.leaves[41]), + mt.join(mt.leaves[42], mt.leaves[43]), + ), + mt.join( + mt.join(mt.leaves[44], mt.leaves[45]), + mt.join(mt.leaves[46], mt.leaves[47]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[48], mt.leaves[49]), + mt.join(mt.leaves[50], mt.leaves[51]), + ), + mt.join( + mt.join(mt.leaves[52], mt.leaves[53]), + mt.join(mt.leaves[54], mt.leaves[55]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[56], mt.leaves[57]), + mt.join(mt.leaves[58], mt.leaves[59]), + ), + mt.join( + mt.join(mt.leaves[60], mt.leaves[61]), + mt.join(mt.leaves[62], mt.leaves[63]), + ), + ), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[64], mt.leaves[65]), + mt.join(mt.leaves[66], mt.leaves[67]), + ), + mt.join( + mt.join(mt.leaves[68], mt.leaves[69]), + mt.join(mt.leaves[70], mt.leaves[71]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[72], mt.leaves[73]), + mt.join(mt.leaves[74], mt.leaves[75]), + ), + mt.join( + mt.join(mt.leaves[76], mt.leaves[77]), + mt.join(mt.leaves[78], mt.leaves[79]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[80], mt.leaves[81]), + mt.join(mt.leaves[82], mt.leaves[83]), + ), + mt.join( + mt.join(mt.leaves[84], mt.leaves[85]), + mt.join(mt.leaves[86], mt.leaves[87]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[88], mt.leaves[89]), + mt.join(mt.leaves[90], mt.leaves[91]), + ), + mt.join( + mt.join(mt.leaves[92], mt.leaves[93]), + mt.join(mt.leaves[94], mt.leaves[95]), + ), + ), + ), + ), + mt.join( + mt.join(mt.leaves[96], mt.leaves[97]), + mt.join(mt.leaves[98], mt.leaves[99]), + ), + ), + ) + // Manually build out some proof sets that should should match what the // Tree creates for the same values. mt.proofSets[1] = make(map[int][][]byte) From e7dac31f6c780c83796e5ed46de25b30a5f8a4cd Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 28 Jul 2017 23:08:25 +0200 Subject: [PATCH 04/19] README: change format of sections from --- to # With # it is easier to make subsections. --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fb044c8..0c27984 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -merkletree ----------- +# merkletree merkletree is a Go package for working with [Merkle trees](http://en.wikipedia.org/wiki/Merkle_tree). Specifically, this package is @@ -12,8 +11,7 @@ piece is part of the full file. When sha256 is used as the hashing algorithm, the implementation matches the merkle tree described in RFC 6962, 'Certificate Transparency'. -Usage ------ +## Usage ```go package main @@ -106,8 +104,7 @@ func main() { For more extensive documentation, refer to the [godoc](http://godoc.org/github.com/NebulousLabs/merkletree). -Notes ------ +## Notes This implementation does not retain the entire Merkle tree in memory. Rather, as each new leaf is added to the tree, is it pushed onto a stack as a "subtree From f00f999e74c6aedecd96896d0b7e105c81ac8e29 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sat, 29 Jul 2017 23:08:25 +0200 Subject: [PATCH 05/19] README: add section "Format of proof" This text was written with slices of leaves in mind https://github.com/NebulousLabs/merkletree/issues/15 --- README.md | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/README.md b/README.md index 0c27984..806a2a9 100644 --- a/README.md +++ b/README.md @@ -128,3 +128,87 @@ hashed multiple times. When using the Reader functions (ReaderRoot and BuildReaderProof), the last segment will not be padded if there are not 'segmentSize' bytes remaining. + +## Format of proof + +Note: the description below mentiones proofs of slices of leaves. +Currently only slices of one leave are supportes. + +### What is included to the proof + +A proof is a slice of slices of bytes. It begins with the leave data, +then hashes of subtrees follow. Combining all leaves which are covered in +these two groups (as leaves from the beginning of the proof or as leaves +from the subtrees whose hashes constitute the second part of the proof) +we get all leaves of the tree and each leave presents once. + +Example. Proof built in a tree of 5 leaves for element at index 2: + +``` + ┌───┴──* + *──┴──┐ │ +┌─┴─┐ ┌─┴─* │ +0 1 2 3 4 + * +``` + +Parts of the proof are marked with asterisks (*). + +If we build a proof for a slice, the rule is the same: first include all +leaves from the target slice, then add hashes of all subtrees so that +together with the target slice they cover all leaves, once. + +Example. Proof built in a tree of 7 leaves for the slice [2, 5). + +``` + ┌─────┴─────┐ + *──┴──┐ ┌──┴──* +┌─┴─┐ ┌─┴─┐ ┌─┴─* │ +0 1 2 3 4 5 6 + * * * +``` + +Example. Proof built in a tree of 7 leaves for the slice [3, 5). + +``` + ┌─────┴─────┐ + *──┴──┐ ┌──┴──* +┌─┴─┐ *─┴─┐ ┌─┴─* │ +0 1 2 3 4 5 6 + * * +``` + +### The order of stuff in the proof + +The proof starts with the data items. For a proof of one element +it is the element itself (one item in the main proof slice). +In case of slice the data is represented as multiple items in the main +proof slice, in the order of occurrence in the source data. + +Hashes of subtrees (constituting the second half of a proof) are sorted +by height (ascending), then by occurrence in the source data. The height +of an orphan subtree is equal to the height of its parent minus one. + +Some examples of how parts of proofs are ordered. A number corresponds +to the place of this leave or subtree hash in the proof. + +``` + ┌────┴───┐ + 5──┴──┐ │ +┌─┴─┐ 3─┴─┐ ┌─┴─4 + 1 2 +``` + +``` + ┌────┴───4 + ┌──┴──┐ │ +3─┴─┐ ┌─┴─┐ ┌─┴─┐ + 1 2 3 +``` + +``` + ┌────┴───┐ + 5──┴──┐ │ +┌─┴─┐ ┌─┴─┐ ┌─┴─┐ + 1 2 3 4 +``` From 8ac3a67211cf3ab2e7206fd334f5851763ae37d5 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sun, 30 Jul 2017 23:08:25 +0200 Subject: [PATCH 06/19] change implementation of verify to support slices --- README.md | 24 ++++++--- verify.go | 158 ++++++++++++++++++++++++------------------------------ 2 files changed, 87 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 806a2a9..3fe7deb 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,9 @@ func main() { // be provided by tree.Prove() merkleRoot, proof, proofIndex, numLeaves = tree.Prove() - //////////////////////////////////////////////// - /// Remaining examples deal with cached trees // - //////////////////////////////////////////////// + //////////////////////////////////////////////////// + /// Next group of examples deal with cached trees // + //////////////////////////////////////////////////// // Example 4: Creating a cached set of Merkle roots and then using them in // a cached tree. The cached tree is height 1, meaning that all elements of @@ -94,6 +94,21 @@ func main() { // rehash any of the elements from subtree1. _, fullProof, _, _ := cachedTree.Prove(subtreeProof) + //////////////////////////////////////////////////////// + /// Next group of examples deal with proofs of slices // + //////////////////////////////////////////////////////// + + // Example 7: Using a Tree to build a merkle tree and get a proof for a + // specific slice for non-file objects. + tree = merkletree.New(sha256.New()) + tree.SetSlice(1, 3) // Objects 1 and 2. + tree.Push([]byte("an object - the tree will hash the data after it is pushed")) + tree.Push([]byte("the first part of the slice")) + tree.Push([]byte("the second part of the slice")) + tree.Push([]byte("another object")) + merkleRoot, proof, _, numLeaves = tree.Prove() + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, proof, 1, 3, numLeaves) + _ = verified _ = collectiveRoot _ = revisedRoot @@ -131,9 +146,6 @@ segment will not be padded if there are not 'segmentSize' bytes remaining. ## Format of proof -Note: the description below mentiones proofs of slices of leaves. -Currently only slices of one leave are supportes. - ### What is included to the proof A proof is a slice of slices of bytes. It begins with the leave data, diff --git a/verify.go b/verify.go index e3f5aa6..ecb4117 100644 --- a/verify.go +++ b/verify.go @@ -10,111 +10,91 @@ import ( // root. False is returned if the proof set or Merkle root is nil, and if // 'numLeaves' equals 0. func VerifyProof(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) bool { - // Return false for nonsense input. A switch statement is used so that the - // cover tool will reveal if a case is not covered by the test suite. This - // would not be possible using a single if statement due to the limitations - // of the cover tool. + return VerifyProofOfSlice(h, merkleRoot, proofSet, proofIndex, proofIndex+1, numLeaves) +} + +// VerifyProofOfSlice takes a Merkle root, a proofSet, and the slice and returns +// true if the first proofEnd-proofBegin elements of the proof set are leaves +// of data in the Merkle root. False is returned if the proof set or Merkle +// root is nil, and if 'numLeaves' equals 0. Can be used with proofs returned +// by Tree.Prove and CachedTree.Prove. +func VerifyProofOfSlice(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofBegin, proofEnd, numLeaves uint64) bool { + // Return false for nonsense input. if merkleRoot == nil { return false } - if proofIndex >= numLeaves { + if proofBegin >= proofEnd { return false } - - // In a Merkle tree, every node except the root node has a sibling. - // Combining the two siblings in the correct order will create the parent - // node. Each of the remaining hashes in the proof set is a sibling to a - // node that can be built from all of the previous elements of the proof - // set. The next node is built by taking: - // - // H(0x01 || sibling A || sibling B) - // - // The difficulty of the algorithm lies in determining whether the supplied - // hash is sibling A or sibling B. This information can be determined by - // using the proof index and the total number of leaves in the tree. - // - // A pair of two siblings forms a subtree. The subtree is complete if it - // has 1 << height total leaves. When the subtree is complete, the position - // of the proof index within the subtree can be determined by looking at - // the bounds of the subtree and determining if the proof index is in the - // first or second half of the subtree. - // - // When the subtree is not complete, either 1 or 0 of the remaining hashes - // will be sibling B. All remaining hashes after that will be sibling A. - // This is true because of the way that orphans are merged into the Merkle - // tree - an orphan at height n is elevated to height n + 1, and only - // hashed when it is no longer an orphan. Each subtree will therefore merge - // with at most 1 orphan to the right before becoming an orphan itself. - // Orphan nodes are always merged with larger subtrees to the left. - // - // One vulnerability with the proof verification is that the proofSet may - // not be long enough. Before looking at an element of proofSet, a check - // needs to be made that the element exists. - - // The first element of the set is the original data. A sibling at height 1 - // is created by getting the leafSum of the original data. - height := 0 - if len(proofSet) <= height { + if proofEnd > numLeaves { return false } - sum := leafSum(h, proofSet[height]) - height++ - // While the current subtree (of height 'height') is complete, determine - // the position of the next sibling using the complete subtree algorithm. - // 'stableEnd' tells us the ending index of the last full subtree. It gets - // initialized to 'proofIndex' because the first full subtree was the - // subtree of height 1, created above (and had an ending index of - // 'proofIndex'). - stableEnd := proofIndex - for { - // Determine if the subtree is complete. This is accomplished by - // rounding down the proofIndex to the nearest 1 << 'height', adding 1 - // << 'height', and comparing the result to the number of leaves in the - // Merkle tree. - subTreeStartIndex := (proofIndex / (1 << uint(height))) * (1 << uint(height)) // round down to the nearest 1 << height - subTreeEndIndex := subTreeStartIndex + (1 << (uint(height))) - 1 // subtract 1 because the start index is inclusive - if subTreeEndIndex >= numLeaves { - // If the Merkle tree does not have a leaf at index - // 'subTreeEndIndex', then the subtree of the current height is not - // a complete subtree. - break - } - stableEnd = subTreeEndIndex - - // Determine if the proofIndex is in the first or the second half of - // the subtree. - if len(proofSet) <= height { + // Create the list of hashes on the level of leaves. + var sums [][]byte + for i := proofBegin; i < proofEnd; i++ { + if len(proofSet) == 0 { return false } - if proofIndex-subTreeStartIndex < 1< 1. + + for numLeaves > 1 { + if proofBegin%2 == 1 { + // Example: addition of % on level <- + // ┌───┴──┐ + // %──┴──* │ <- + // ┌─┴─┐ ┌─┴─┐ │ + // * * + if len(proofSet) == 0 { + return false + } + left := proofSet[0] + proofSet = proofSet[1:] + sums = append([][]byte{left}, sums...) + proofBegin -= 1 + } + if len(sums)%2 == 1 && proofEnd < numLeaves { + // Example: addition of % on level <- + // ┌───┴──┐ + // *──┴──% │ <- + // ┌─┴─┐ ┌─┴─┐ │ + // * * + if len(proofSet) == 0 { + return false + } + right := proofSet[0] + proofSet = proofSet[1:] + sums = append(sums, right) + proofEnd += 1 + } + var sums2 [][]byte + for len(sums) >= 2 { + left, right := sums[0], sums[1] + sums = sums[2:] + sums2 = append(sums2, nodeSum(h, left, right)) } - sum = nodeSum(h, sum, proofSet[height]) - height++ + if len(sums) == 1 { + sums2 = append(sums2, sums[0]) + } + sums = sums2 + proofBegin /= 2 + // proofEnd and numLeaves need +1 because they are not inclusive. + proofEnd = (proofEnd + 1) / 2 + numLeaves = (numLeaves + 1) / 2 } - // All remaining elements in the proof set will belong to a left sibling. - for height < len(proofSet) { - sum = nodeSum(h, proofSet[height], sum) - height++ + if len(proofSet) != 0 { + return false } // Compare our calculated Merkle root to the desired Merkle root. - if bytes.Compare(sum, merkleRoot) == 0 { - return true - } - return false + return bytes.Compare(sums[0], merkleRoot) == 0 } From 5be5e10fd4b53f68579534a7c0ef269ef9088b4b Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Mon, 31 Jul 2017 23:08:25 +0200 Subject: [PATCH 07/19] tree: implement SetSlice --- tree.go | 216 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 125 insertions(+), 91 deletions(-) diff --git a/tree.go b/tree.go index 8e19115..3ff2f84 100644 --- a/tree.go +++ b/tree.go @@ -7,9 +7,10 @@ import ( // A Tree takes data as leaves and returns the Merkle root. Each call to 'Push' // adds one leaf to the Merkle tree. Calling 'Root' returns the Merkle root. -// The Tree also constructs proof that a single leaf is a part of the tree. The -// leaf can be chosen with 'SetIndex'. The memory footprint of Tree grows in -// O(log(n)) in the number of leaves. +// The Tree also constructs proof that a single leaf or a slice of leaves is +// a part of the tree. The leaf can be chosen with 'SetIndex'. The slice can +// be chosen with 'SetSlice'. The memory footprint of Tree grows in O(log(n)) +// in the number of leaves. type Tree struct { // The Tree is stored as a stack of subtrees. Each subtree has a height, // and is the Merkle root of 2^height leaves. A Tree with 11 nodes is @@ -21,13 +22,15 @@ type Tree struct { head *subTree hash hash.Hash - // Helper variables used to construct proofs that the data at 'proofIndex' + // Helper variables used to construct proofs that the data at slice from + // 'proofBegin' to 'proofEnd' (a single leaf if a slice of length 1) // is in the Merkle tree. The proofSet is constructed as elements are being - // added to the tree. The first element of the proof set is the original - // data used to create the leaf at index 'proofIndex'. - currentIndex uint64 - proofIndex uint64 - proofSet [][]byte + // added to the tree. The first elements of the proof set are the original + // data used to create the leaf at indices from 'proofBegin' to 'proofEnd'. + currentIndex uint64 + proofBegin, proofEnd uint64 + lader [][][]byte + bases [][]byte // The cachedTree flag indicates that the tree is cached, meaning that // different code is used in 'Push' for creating a new head subtree. Adding @@ -40,9 +43,15 @@ type Tree struct { // of the Tree. 'sum' is the Merkle root of the subTree. If 'next' is not nil, // it will be a tree with a higher height. type subTree struct { - next *subTree - height int // Int is okay because a height over 300 is physically unachievable. - sum []byte + next *subTree + height int // Int is okay because a height over 300 is physically unachievable. + begin, end uint64 + sum []byte +} + +func (s *subTree) contains(t *Tree) bool { + return (s.begin <= t.proofBegin && t.proofBegin < s.end) || + (t.proofBegin <= s.begin && s.begin < t.proofEnd) } // sum returns the hash of the input data using the specified algorithm. @@ -78,84 +87,114 @@ func joinSubTrees(h hash.Hash, a, b *subTree) *subTree { if a.height < b.height { panic("invalid subtree presented - height mismatch") } + if a.end != b.begin { + panic("the subtrees are not adjacent") + } } return &subTree{ next: a.next, height: a.height + 1, + begin: a.begin, + end: b.end, sum: nodeSum(h, a.sum, b.sum), } } +// proofLader stores hashes that are proof parts. +// proofLader[height] is the list of hashes of subtrees +// of height 'height' in the correct order. +type proofLader [][][]byte + +// addToLader returns new proofLader with one new element. +func addToLader(lader proofLader, height int, proof []byte) proofLader { + for len(lader) <= height { + lader = append(lader, nil) + } + lader[height] = append(lader[height], proof) + return lader +} + +// cloneLader returns deep copy of proofLader. +func cloneLader(lader proofLader) proofLader { + lader2 := make(proofLader, len(lader)) + for i, step := range lader { + step2 := make([][]byte, len(step)) + copy(step2, step) + lader2[i] = step2 + } + return lader2 +} + +// foldLader converts proofLader to the tail of the proof. +// To generate a valid proof, concatenate data from the target slice +// (one element of the target slice = one element in the proof) with +// the output of foldLader. +func foldLader(lader proofLader) [][]byte { + var proofs [][]byte + for _, step := range lader { + if len(step) > 2 { + panic("More than 2 proofs of same height in proofSet") + } + for _, proof := range step { + proofs = append(proofs, proof) + } + } + return proofs +} + // New creates a new Tree. The provided hash will be used for all hashing // operations within the Tree. func New(h hash.Hash) *Tree { return &Tree{ - hash: h, + hash: h, + proofBegin: 0, + proofEnd: 1, } } // Prove creates a proof that the leaf at the established index (established by -// SetIndex) is an element of the Merkle tree. Prove will return a nil proof -// set if used incorrectly. Prove does not modify the Tree. -func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { - // Return nil if the Tree is empty, or if the proofIndex hasn't yet been +// SetIndex) or the slice of leaves (established by SetSlice) belongs to the +// Merkle tree. Prove will return a nil proof set if used incorrectly. +// Prove does not modify the Tree. +func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofBegin uint64, numLeaves uint64) { + // Return nil if the Tree is empty, or if the proofEnd hasn't yet been // reached. - if t.head == nil || len(t.proofSet) == 0 { - return t.Root(), nil, t.proofIndex, t.currentIndex + if t.head == nil || t.currentIndex < t.proofEnd { + return t.Root(), nil, t.proofBegin, t.currentIndex } - proofSet = t.proofSet + proofSet = make([][]byte, len(t.bases)) + copy(proofSet, t.bases) - // The set of subtrees must now be collapsed into a single root. The proof - // set already contains all of the elements that are members of a complete - // subtree. Of what remains, there will be at most 1 element provided from - // a sibling on the right, and all of the other proofs will be provided - // from a sibling on the left. This results from the way orphans are - // treated. All subtrees smaller than the subtree containing the proofIndex - // will be combined into a single subtree that gets combined with the - // proofIndex subtree as a single right sibling. All subtrees larger than - // the subtree containing the proofIndex will be combined with the subtree - // containing the proof index as left siblings. - - // Start at the smallest subtree and combine it with larger subtrees until - // it would be combining with the subtree that contains the proof index. We - // can recognize the subtree containing the proof index because the height - // of that subtree will be one less than the current length of the proof - // set. + // The set of subtrees must now be collapsed into a single root. + // Unlike Push, now we ignore previous height of the right subtree, + // because we have to collapse all leaves anyway. + // All needed hashes are added to the ladder. current := t.head - for current.next != nil && current.next.height < len(proofSet)-1 { + lader := cloneLader(t.lader) + for current.next != nil { + // The left subtree is higher or equal to the right subtree. + // If the right subtree is incomplete, its height is considered + // to be equal to its left sibling. + height := current.next.height + if current.contains(t) && !current.next.contains(t) { + lader = addToLader(lader, height, current.next.sum) + } else if !current.contains(t) && current.next.contains(t) { + lader = addToLader(lader, height, current.sum) + } + current = joinSubTrees(t.hash, current.next, current) } + proofSet = append(proofSet, foldLader(lader)...) // Sanity check - check that either 'current' or 'current.next' is the - // subtree containing the proof index. + // subtree containing the proof slice. if DEBUG { - if current.height != len(t.proofSet)-1 && (current.next != nil && current.next.height != len(t.proofSet)-1) { - panic("could not find the subtree containing the proof index") + if !current.contains(t) && !current.next.contains(t) { + panic("could not find the subtree containing the proof slice") } } - - // If the current subtree is not the subtree containing the proof index, - // then it must be an aggregate subtree that is to the right of the subtree - // containing the proof index, and the next subtree is the subtree - // containing the proof index. - if current.next != nil && current.next.height == len(proofSet)-1 { - proofSet = append(proofSet, current.sum) - current = current.next - } - - // The current subtree must be the subtree containing the proof index. This - // subtree does not need an entry, as the entry was created during the - // construction of the Tree. Instead, skip to the next subtree. - current = current.next - - // All remaining subtrees will be added to the proof set as a left sibling, - // completing the proof set. - for current != nil { - proofSet = append(proofSet, current.sum) - current = current.next - } - return t.Root(), proofSet, t.proofIndex, t.currentIndex + return t.Root(), proofSet, t.proofBegin, t.currentIndex } // Push will add data to the set, building out the Merkle tree and Root. The @@ -166,8 +205,8 @@ func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, func (t *Tree) Push(data []byte) { // The first element of a proof is the data at the proof index. If this // data is being inserted at the proof index, it is added to the proof set. - if t.currentIndex == t.proofIndex { - t.proofSet = append(t.proofSet, data) + if t.proofBegin <= t.currentIndex && t.currentIndex < t.proofEnd { + t.bases = append(t.bases, data) } // Hash the data to create a subtree of height 0. The sum of the new node @@ -177,6 +216,8 @@ func (t *Tree) Push(data []byte) { t.head = &subTree{ next: t.head, height: 0, + begin: t.currentIndex, + end: t.currentIndex + 1, } if t.cachedTree { t.head.sum = data @@ -189,32 +230,18 @@ func (t *Tree) Push(data []byte) { // be combined into a single subTree of height n+1. for t.head.next != nil && t.head.height == t.head.next.height { // Before combining subtrees, check whether one of the subtree hashes - // needs to be added to the proof set. This is going to be true IFF the - // subtrees being combined are one height higher than the previous - // subtree added to the proof set. The height of the previous subtree - // added to the proof set is equal to len(t.proofSet) - 1. - if t.head.height == len(t.proofSet)-1 { - // One of the subtrees needs to be added to the proof set. The - // subtree that needs to be added is the subtree that does not - // contain the proofIndex. Because the subtrees being compared are - // the smallest and rightmost trees in the Tree, this can be - // determined by rounding the currentIndex down to the number of - // nodes in the subtree and comparing that index to the proofIndex. - leaves := uint64(1 << uint(t.head.height)) - mid := (t.currentIndex / leaves) * leaves - if t.proofIndex < mid { - t.proofSet = append(t.proofSet, t.head.sum) - } else { - t.proofSet = append(t.proofSet, t.head.next.sum) - } + // needs to be added to the proof set. This is going to be true IFF + // one of the subtrees being combined overlaps with the target slice + // and another does not. + var proof []byte + if t.head.contains(t) && !t.head.next.contains(t) { + proof = t.head.next.sum + } else if !t.head.contains(t) && t.head.next.contains(t) { + proof = t.head.sum + } - // Sanity check - the proofIndex should never be less than the - // midpoint minus the number of leaves in each subtree. - if DEBUG { - if t.proofIndex < mid-leaves { - panic("proof being added with weird values") - } - } + if proof != nil { + t.lader = addToLader(t.lader, t.head.height, proof) } // Join the two subTrees into one subTree with a greater height. Then @@ -258,9 +285,16 @@ func (t *Tree) Root() []byte { // SetIndex will tell the Tree to create a storage proof for the leaf at the // input index. SetIndex must be called on an empty tree. func (t *Tree) SetIndex(i uint64) error { + return t.SetSlice(i, i+1) +} + +// SetSlice will tell the Tree to create a storage proof for the leaves +// within the slice [begin, end). SetSlice must be called on an empty tree. +func (t *Tree) SetSlice(begin, end uint64) error { if t.head != nil { - return errors.New("cannot call SetIndex on Tree if Tree has not been reset") + return errors.New("cannot call SetIndex or SetSlice on Tree if Tree has not been reset") } - t.proofIndex = i + t.proofBegin = begin + t.proofEnd = end return nil } From e62bf2bb1417284cae993d7ae85aec92c0b04e16 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Tue, 1 Aug 2017 23:08:25 +0200 Subject: [PATCH 08/19] tree_test: test slices mode in TestBadInputs --- tree_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tree_test.go b/tree_test.go index 4f3cc19..85b4d4d 100644 --- a/tree_test.go +++ b/tree_test.go @@ -507,6 +507,30 @@ func TestBadInputs(t *testing.T) { if err == nil { t.Error("expecting error, shouldn't be able to reset a tree after pushing") } + err = tree.SetSlice(1, 3) + if err == nil { + t.Error("expecting error, shouldn't be able to reset a tree after pushing") + } + + // Get the proof of a tree that hasn't reached it's index (slice version). + tree2 := New(sha256.New()) + err = tree2.SetSlice(1, 10) + if err != nil { + t.Fatal(err) + } + tree2.Push([]byte{1}) + _, proof, _, _ = tree2.Prove() + if proof != nil { + t.Fatal(err) + } + err = tree2.SetIndex(2) + if err == nil { + t.Error("expecting error, shouldn't be able to reset a tree after pushing") + } + err = tree2.SetSlice(1, 3) + if err == nil { + t.Error("expecting error, shouldn't be able to reset a tree after pushing") + } // Try nil values in VerifyProof. mt := CreateMerkleTester(t) @@ -525,6 +549,23 @@ func TestBadInputs(t *testing.T) { if VerifyProof(sha256.New(), mt.roots[15], mt.proofSets[15][10], 15, 0) { t.Error("VerifyProof should return false when numLeaves is 0") } + + // Try nil values in VerifyProofOfSlice. + if VerifyProofOfSlice(sha256.New(), nil, mt.proofSets[1][0], 0, 1, 1) { + t.Error("VerifyProofOfSlice should return false for nil merkle root") + } + if VerifyProofOfSlice(sha256.New(), []byte{1}, nil, 0, 1, 1) { + t.Error("VerifyProofOfSlice should return false for nil proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][3][1:], 3, 4, 15) { + t.Error("VerifyProofOfSlice should return false for too-short proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][10][1:], 10, 11, 15) { + t.Error("VerifyProofOfSlice should return false for too-short proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][10], 15, 30, 0) { + t.Error("VerifyProofOfSlice should return false when numLeaves is 0") + } } // TestCompatibility runs BuildProof for a large set of trees, and checks that From c76cfc5d48eb045300c0f3ee7df4d86507f8d57e Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Wed, 2 Aug 2017 23:08:25 +0200 Subject: [PATCH 09/19] tree_test: add compatibility tests for slices --- tree_test.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tree_test.go b/tree_test.go index 85b4d4d..51bfbed 100644 --- a/tree_test.go +++ b/tree_test.go @@ -648,6 +648,114 @@ func TestCompatibility(t *testing.T) { } } +func TestCompatibilitySlice(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Brute force all trees up to size 'max'. Running time for this test is max^5. + max := uint64(15) + tree := New(sha256.New()) + for numLeaves := uint64(1); numLeaves < max; numLeaves++ { + // Make merkleRoot using regular Prove. + tree = New(sha256.New()) + err := tree.SetIndex(0) + if err != nil { + t.Fatal(err) + } + for k := uint64(0); k < numLeaves; k++ { + tree.Push([]byte{byte(k)}) + } + merkleRoot, _, _, _ := tree.Prove() + + // Try with proof at every possible slice. + for a := uint64(0); a < numLeaves; a++ { + for b := a + 1; b <= numLeaves; b++ { + tree = New(sha256.New()) + err := tree.SetSlice(a, b) + if err != nil { + t.Fatal(err) + } + for k := uint64(0); k < numLeaves; k++ { + tree.Push([]byte{byte(k)}) + } + + // Build the proof for the tree and run it through verify. + _, proofSet, proofBegin, _ := tree.Prove() + if proofBegin != a { + t.Error("proofBegin=%d, want %d", proofBegin, a) + } + if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, a, b, numLeaves) { + t.Error("proof didn't verify for indices", a, b) + } + + // Check that verification fails for all other indices. + for a1 := uint64(0); a1 < numLeaves; a1++ { + for b1 := a1 + 1; b1 <= numLeaves; b1++ { + if a == a1 && b == b1 { + continue + } + if VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, a1, b1, numLeaves) { + t.Error("proof verify for indices", a1, b1) + } + } + } + } + } + } +} + +func TestCompatibilitySliceLarge(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Check that proofs on larger trees are consistent. + for i := 0; i < 25; i++ { + // Determine a random size for the tree up to 256k elements. + sizeI, err := rand.Int(rand.Reader, big.NewInt(256e3)) + if err != nil { + t.Fatal(err) + } + size := uint64(sizeI.Int64()) + + proofBeginI, err := rand.Int(rand.Reader, sizeI) + if err != nil { + t.Fatal(err) + } + proofBegin := uint64(proofBeginI.Int64()) + + var remainingI big.Int + remainingI.Sub(sizeI, proofBeginI) + sliceI, err := rand.Int(rand.Reader, &remainingI) + if err != nil { + t.Fatal(err) + } + var proofEndI big.Int + proofEndI.Add(proofBeginI, sliceI) + proofEnd := uint64(proofEndI.Int64()) + + // Prepare the tree. + tree := New(sha256.New()) + err = tree.SetSlice(proofBegin, proofEnd) + if err != nil { + t.Fatal(err) + } + + // Insert 'size' unique elements. + for j := 0; j < int(size); j++ { + elem := []byte(strconv.Itoa(j)) + tree.Push(elem) + } + + // Get the proof for the tree and run it through verify. + merkleRoot, proofSet, _, numLeaves := tree.Prove() + if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) { + t.Error("proof didn't verify in long test", size, proofBegin, proofEnd) + } + } +} + // TestLeafCounts checks that the number of leaves in the tree are being // reported correctly. func TestLeafCounts(t *testing.T) { From 83137a2be848258162d09457b82e09f452a6faad Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Thu, 3 Aug 2017 23:08:25 +0200 Subject: [PATCH 10/19] tree_test: test teaves count in slices mode --- tree_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tree_test.go b/tree_test.go index 51bfbed..002b9f8 100644 --- a/tree_test.go +++ b/tree_test.go @@ -779,6 +779,18 @@ func TestLeafCounts(t *testing.T) { if leaves != 1 { t.Error("bad reporting on leaf count") } + + tree = New(sha256.New()) + err = tree.SetSlice(1, 10) + if err != nil { + t.Fatal(err) + } + tree.Push([]byte{}) + tree.Push([]byte{}) + _, _, _, leaves = tree.Prove() + if leaves != 2 { + t.Error("bad reporting on leaf count") + } } // BenchmarkSha256_4MB uses sha256 to hash 4mb of data. From 0df5fa6d0d104b1f289135a721b7c7bd47b8c002 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 4 Aug 2017 23:08:25 +0200 Subject: [PATCH 11/19] tree_test: add manual proofs for slices --- tree_test.go | 645 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 643 insertions(+), 2 deletions(-) diff --git a/tree_test.go b/tree_test.go index 002b9f8..3ba16a2 100644 --- a/tree_test.go +++ b/tree_test.go @@ -27,6 +27,14 @@ type MerkleTester struct { // root of that tree can be found in roots. The second map is the // proofIndex that was used when building the proof. proofSets map[int]map[int][][]byte + + // sliceProofSets contains proofs that certain slices of leaves is in a Merkle tree. The + // first map is the number of leaves in the tree that the proof is for. The + // root of that tree can be found in roots. The second map is the + // proofBegin and the third map is the proofEnd that was used when + // building the proof. + sliceProofSets map[int]map[int]map[int][][]byte + *testing.T } @@ -41,8 +49,9 @@ func (mt *MerkleTester) join(a, b []byte) []byte { // Tree creates. func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { mt = &MerkleTester{ - roots: make(map[int][]byte), - proofSets: make(map[int]map[int][][]byte), + roots: make(map[int][]byte), + proofSets: make(map[int]map[int][][]byte), + sliceProofSets: make(map[int]map[int]map[int][][]byte), } mt.T = t @@ -371,6 +380,550 @@ func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { mt.roots[8], } + // Manually build out some slice proof sets that should should + // match what the Tree creates for the same values. + + mt.sliceProofSets[1] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{mt.data[0]}, + }, + } + + mt.sliceProofSets[2] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + }, + }, + } + + mt.sliceProofSets[3] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.leaves[2], + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.leaves[2], + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.leaves[2], + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + } + + mt.sliceProofSets[4] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.join(mt.leaves[2], mt.leaves[3]), + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.join(mt.leaves[2], mt.leaves[3]), + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.leaves[3], + }, + 4: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.join(mt.leaves[2], mt.leaves[3]), + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + mt.leaves[3], + }, + 4: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + 4: [][]byte{ + mt.data[2], + mt.data[3], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 3: map[int][][]byte{ + 4: [][]byte{ + mt.data[3], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + } + + mt.sliceProofSets[5] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.leaves[4], + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.leaves[4], + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.leaves[3], + mt.leaves[4], + }, + 4: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.leaves[4], + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + mt.leaves[3], + mt.leaves[4], + }, + 4: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[0], + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.leaves[4], + }, + 4: [][]byte{ + mt.data[2], + mt.data[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[2], + mt.data[3], + mt.data[4], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 3: map[int][][]byte{ + 4: [][]byte{ + mt.data[3], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[3], + mt.data[4], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 4: map[int][][]byte{ + 5: [][]byte{ + mt.data[4], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + }, + } + + mt.sliceProofSets[6] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 4: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[5], + }, + 6: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.data[5], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + mt.leaves[3], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 4: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[0], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[0], + mt.leaves[5], + }, + 6: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.data[5], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 4: [][]byte{ + mt.data[2], + mt.data[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[5], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + 6: [][]byte{ + mt.data[2], + mt.data[3], + mt.data[4], + mt.data[5], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 3: map[int][][]byte{ + 4: [][]byte{ + mt.data[3], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[3], + mt.data[4], + mt.leaves[2], + mt.leaves[5], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + 6: [][]byte{ + mt.data[3], + mt.data[4], + mt.data[5], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 4: map[int][][]byte{ + 5: [][]byte{ + mt.data[4], + mt.leaves[5], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + 6: [][]byte{ + mt.data[4], + mt.data[5], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + }, + 5: map[int][][]byte{ + 6: [][]byte{ + mt.data[5], + mt.leaves[4], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + }, + } + + mt.sliceProofSets[100] = map[int]map[int][][]byte{ + 17: map[int][][]byte{ + 43: [][]byte{ + mt.data[17], + mt.data[18], + mt.data[19], + mt.data[20], + mt.data[21], + mt.data[22], + mt.data[23], + mt.data[24], + mt.data[25], + mt.data[26], + mt.data[27], + mt.data[28], + mt.data[29], + mt.data[30], + mt.data[31], + mt.data[32], + mt.data[33], + mt.data[34], + mt.data[35], + mt.data[36], + mt.data[37], + mt.data[38], + mt.data[39], + mt.data[40], + mt.data[41], + mt.data[42], + mt.leaves[16], + mt.leaves[43], + // 44-47. + mt.join( + mt.join(mt.leaves[44], mt.leaves[45]), + mt.join(mt.leaves[46], mt.leaves[47]), + ), + // 0-15. + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + mt.join( + mt.join(mt.leaves[4], mt.leaves[5]), + mt.join(mt.leaves[6], mt.leaves[7]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[8], mt.leaves[9]), + mt.join(mt.leaves[10], mt.leaves[11]), + ), + mt.join( + mt.join(mt.leaves[12], mt.leaves[13]), + mt.join(mt.leaves[14], mt.leaves[15]), + ), + ), + ), + // 48-63. + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[48], mt.leaves[49]), + mt.join(mt.leaves[50], mt.leaves[51]), + ), + mt.join( + mt.join(mt.leaves[52], mt.leaves[53]), + mt.join(mt.leaves[54], mt.leaves[55]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[56], mt.leaves[57]), + mt.join(mt.leaves[58], mt.leaves[59]), + ), + mt.join( + mt.join(mt.leaves[60], mt.leaves[61]), + mt.join(mt.leaves[62], mt.leaves[63]), + ), + ), + ), + mt.join( + // 64-95. + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[64], mt.leaves[65]), + mt.join(mt.leaves[66], mt.leaves[67]), + ), + mt.join( + mt.join(mt.leaves[68], mt.leaves[69]), + mt.join(mt.leaves[70], mt.leaves[71]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[72], mt.leaves[73]), + mt.join(mt.leaves[74], mt.leaves[75]), + ), + mt.join( + mt.join(mt.leaves[76], mt.leaves[77]), + mt.join(mt.leaves[78], mt.leaves[79]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[80], mt.leaves[81]), + mt.join(mt.leaves[82], mt.leaves[83]), + ), + mt.join( + mt.join(mt.leaves[84], mt.leaves[85]), + mt.join(mt.leaves[86], mt.leaves[87]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[88], mt.leaves[89]), + mt.join(mt.leaves[90], mt.leaves[91]), + ), + mt.join( + mt.join(mt.leaves[92], mt.leaves[93]), + mt.join(mt.leaves[94], mt.leaves[95]), + ), + ), + ), + ), + // 96-99. + mt.join( + mt.join(mt.leaves[96], mt.leaves[97]), + mt.join(mt.leaves[98], mt.leaves[99]), + ), + ), + }, + }, + } + return } @@ -479,6 +1032,91 @@ func TestBuildAndVerifyProof(t *testing.T) { } } +// TestBuildAndVerifyProofOfSlice builds a proof using a tree for every single +// manually created proof of slice in the MerkleTester. Then it checks that the proof +// matches the manually created proof, and that the proof is verified by +// VerifyProofOfSlice. Then it checks that the proof fails for all other slices, +// which should happen if all of the leaves are unique. +func TestBuildAndVerifyProofOfSlice(t *testing.T) { + mt := CreateMerkleTester(t) + + // Compare the results of building a Merkle proof to all of the manually + // constructed proofs. + tree := New(sha256.New()) + for i, manualProveSets := range mt.sliceProofSets { + for j, manualProveSets2 := range manualProveSets { + for l, expectedProveSet := range manualProveSets2 { + // Build out the tree. + tree = New(sha256.New()) + err := tree.SetSlice(uint64(j), uint64(l)) + if err != nil { + t.Fatal(err) + } + for k := 0; k < i; k++ { + tree.Push(mt.data[k]) + } + + // Get the proof and check all values. + merkleRoot, proofSet, proofIndex, numSegments := tree.Prove() + if bytes.Compare(merkleRoot, mt.roots[i]) != 0 { + t.Error("incorrect Merkle root returned by Tree for indices", i, j, l) + } + if len(proofSet) != len(expectedProveSet) { + t.Error("proof set is wrong length for indices", i, j, l) + continue + } + if proofIndex != uint64(j) { + t.Error("incorrect proofIndex returned for indices", i, j, l) + } + if numSegments != uint64(i) { + t.Error("incorrect numSegments returned for indices", i, j, l) + } + for k := range proofSet { + if bytes.Compare(proofSet[k], expectedProveSet[k]) != 0 { + t.Error("proof set does not match expected proof set for indices", i, j, l, k) + } + } + + // Check that verification works on for the desired proof index but + // fails for all other indices. + if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, uint64(j), uint64(l), numSegments) { + t.Error("proof set does not verify for indices", i, j, l) + } + for k := uint64(0); k < uint64(i); k++ { + for m := k + 1; m < uint64(i); m++ { + if k == uint64(j) && m == uint64(l) { + continue + } + if VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, uint64(k), uint64(m), uint64(i)) { + t.Error("proof set verifies for wrong slice at indices", i, j, l, k, m) + } + } + } + + // Check that calling Prove a second time results in the same values. + merkleRoot2, proofSet2, proofIndex2, numSegments2 := tree.Prove() + if bytes.Compare(merkleRoot, merkleRoot2) != 0 { + t.Error("tree returned different merkle roots after calling Prove twice for indices", i, j, l) + } + if len(proofSet) != len(proofSet2) { + t.Error("tree returned different proof sets after calling Prove twice for indices", i, j, l) + } + for k := range proofSet { + if bytes.Compare(proofSet[k], proofSet2[k]) != 0 { + t.Error("tree returned different proof sets after calling Prove twice for indices", i, j, l) + } + } + if proofIndex != proofIndex2 { + t.Error("tree returned different proof indexes after calling Prove twice for indices", i, j, l) + } + if numSegments != numSegments2 { + t.Error("tree returned different segment count after calling Prove twice for indices", i, j, l) + } + } + } + } +} + // TestBadInputs provides malicious inputs to the functions of the package, // trying to trigger panics or unexpected behavior. func TestBadInputs(t *testing.T) { @@ -563,6 +1201,9 @@ func TestBadInputs(t *testing.T) { if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][10][1:], 10, 11, 15) { t.Error("VerifyProofOfSlice should return false for too-short proof set") } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.sliceProofSets[4][2][4][1:], 2, 4, 4) { + t.Error("VerifyProofOfSlice should return false for too-short proof set") + } if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][10], 15, 30, 0) { t.Error("VerifyProofOfSlice should return false when numLeaves is 0") } From 6b537ad6acfc6622fb61a5ba18206f6136e242a7 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sat, 5 Aug 2017 23:08:25 +0200 Subject: [PATCH 12/19] cachedtree_test: fix mistake in comment --- cachedtree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cachedtree_test.go b/cachedtree_test.go index 5fd7053..6427ca5 100644 --- a/cachedtree_test.go +++ b/cachedtree_test.go @@ -301,7 +301,7 @@ func TestCachedTreeConstructionAuto(t *testing.T) { t.SkipNow() } - // Build out cached trees with up to 33 cached elements, each height 'h'. + // Build out cached trees with up to 35 cached elements, each height 'h'. for h := uint64(0); h < 5; h++ { n := uint64(1) << h for i := uint64(0); i < 35; i++ { From b38bf5835133a72d0ea85ca288fbc98748abf061 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sun, 6 Aug 2017 23:08:25 +0200 Subject: [PATCH 13/19] readers: implement slice mode --- README.md | 8 ++++++++ readers.go | 16 +++++++++++++--- readers_test.go | 42 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3fe7deb..59881fa 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,14 @@ func main() { merkleRoot, proof, _, numLeaves = tree.Prove() verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, proof, 1, 3, numLeaves) + // Example 8: Build and verify a proof that the elements at segments 5-10 + // are in the merkle root. The proof starts with the elements themselves. + file.Seek(0, 0) // Offset needs to be set back to 0. + proofBegin := uint64(5) + proofEnd := uint64(10) + 1 + merkleRoot, proof, numLeaves, _ = merkletree.BuildReaderProofSlice(file, sha256.New(), segmentSize, proofBegin, proofEnd) + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, proof, proofBegin, proofEnd, numLeaves) + _ = verified _ = collectiveRoot _ = revisedRoot diff --git a/readers.go b/readers.go index 63ced81..01e3103 100644 --- a/readers.go +++ b/readers.go @@ -49,10 +49,20 @@ func ReaderRoot(r io.Reader, h hash.Hash, segmentSize int) (root []byte, err err // 'segmentSize' bytes except the last leaf, which will not be padded out if // there are not enough bytes remaining in the reader. func BuildReaderProof(r io.Reader, h hash.Hash, segmentSize int, index uint64) (root []byte, proofSet [][]byte, numLeaves uint64, err error) { + return BuildReaderProofSlice(r, h, segmentSize, index, index+1) +} + +// BuildReaderProof returns a proof that certain data is in the merkle tree +// created by the data in the reader. The merkle root, set of proofs, and the +// number of leaves in the Merkle tree are all returned. All leaves will we +// 'segmentSize' bytes except the last leaf, which will not be padded out if +// there are not enough bytes remaining in the reader. +// This function creates the proof of slice [proofBegin, proofEnd). +func BuildReaderProofSlice(r io.Reader, h hash.Hash, segmentSize int, proofBegin, proofEnd uint64) (root []byte, proofSet [][]byte, numLeaves uint64, err error) { tree := New(h) - err = tree.SetIndex(index) + err = tree.SetSlice(proofBegin, proofEnd) if err != nil { - // This code should be unreachable - SetIndex will only return an error + // This code should be unreachable - SetSlice will only return an error // if the tree is not empty, and yet the tree should be empty at this // point. panic(err) @@ -63,7 +73,7 @@ func BuildReaderProof(r io.Reader, h hash.Hash, segmentSize int, index uint64) ( } root, proofSet, _, numLeaves = tree.Prove() if len(proofSet) == 0 { - err = errors.New("index was not reached while creating proof") + err = errors.New("proof slice was not reached while creating proof") return } return diff --git a/readers_test.go b/readers_test.go index 2768362..40b5ab0 100644 --- a/readers_test.go +++ b/readers_test.go @@ -102,10 +102,50 @@ func TestBuildReaderProofPadding(t *testing.T) { } } -// TestEmptyReader passes an empty reader into BuildReaderProof. +// TestBuildReaderProofSlicePadding passes BuildReaderProofSlice a reader that has too +// few bytes to fill the last segment. The segment should not be padded out. +func TestBuildReaderProofSlicePadding(t *testing.T) { + bytes1 := []byte{1, 2, 3, 4, 5} + reader := bytes.NewReader(bytes1) + root, proofSet, numLeaves, err := BuildReaderProofSlice(reader, sha256.New(), 2, 1, 3) + if err != nil { + t.Fatal(err) + } + + base12 := sum(sha256.New(), []byte{0, 1, 2}) + base34 := sum(sha256.New(), []byte{0, 3, 4}) + base5 := sum(sha256.New(), []byte{0, 5}) + node1234 := sum(sha256.New(), append(append([]byte{1}, base12...), base34...)) + expectedRoot := sum(sha256.New(), append(append([]byte{1}, node1234...), base5...)) + + if bytes.Compare(root, expectedRoot) != 0 { + t.Error("ReaderRoot returned the wrong root") + } + if len(proofSet) != 3 { + t.Fatalf("proofSet is the incorrect lenght: %d", len(proofSet)) + } + if bytes.Compare(proofSet[0], []byte{3, 4}) != 0 { + t.Error("proofSet is incorrect") + } + if bytes.Compare(proofSet[1], []byte{5}) != 0 { + t.Error("proofSet is incorrect") + } + if bytes.Compare(proofSet[2], base12) != 0 { + t.Error("proofSet is incorrect") + } + if numLeaves != 3 { + t.Error("wrong number of leaves returned") + } +} + +// TestEmptyReader passes an empty reader into BuildReaderProof and BuildReaderProofSlice. func TestEmptyReader(t *testing.T) { _, _, _, err := BuildReaderProof(new(bytes.Reader), sha256.New(), 64, 5) if err == nil { t.Error(err) } + _, _, _, err = BuildReaderProofSlice(new(bytes.Reader), sha256.New(), 64, 5, 6) + if err == nil { + t.Error(err) + } } From e0869cd74f1e9547a22edabf49ef7e905252af54 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Mon, 7 Aug 2017 23:08:25 +0200 Subject: [PATCH 14/19] cachedtree_test: fix mistake in comment --- cachedtree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cachedtree_test.go b/cachedtree_test.go index 6427ca5..4987135 100644 --- a/cachedtree_test.go +++ b/cachedtree_test.go @@ -8,7 +8,7 @@ import ( // addSubTree will create a subtree of the desired height using the dataSeed to // seed the data. addSubTree will add the data created in the subtree to the -// Tree as well. The tree must have the proveIndex set separately. +// Tree as well. The tree must have the SetIndex set separately. func addSubTree(height uint64, dataSeed []byte, subtreeProveIndex uint64, fullTree *Tree) (subTree *Tree) { data := sum(sha256.New(), dataSeed) leaves := 1 << height From c412b6e63ca767c39270dfa9c5fc97080a2e5273 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Tue, 8 Aug 2017 23:08:25 +0200 Subject: [PATCH 15/19] implement slices on cached trees --- README.md | 51 +++++++++++++ cachedtree.go | 50 +++++++++---- cachedtree_test.go | 181 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 268 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 59881fa..08767e8 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,57 @@ func main() { merkleRoot, proof, numLeaves, _ = merkletree.BuildReaderProofSlice(file, sha256.New(), segmentSize, proofBegin, proofEnd) verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, proof, proofBegin, proofEnd, numLeaves) + // Example 9: Cached tree of height 2, with proof slice entirely inside + // one cached subtree. + cachedTree = merkletree.NewCachedTree(sha256.New(), 2) + cachedTree.SetSlice(5, 7) + subtree1 = merkletree.New(sha256.New()) + subtree1.Push([]byte("first leaf, first subtree")) + subtree1.Push([]byte("second leaf, first subtree")) + subtree1.Push([]byte("third leaf, first subtree")) + subtree1.Push([]byte("fourth leaf, first subtree")) + subtree2 = merkletree.New(sha256.New()) + subtree2.SetSlice(1, 3) + subtree2.Push([]byte("first leaf, second subtree")) + subtree2.Push([]byte("second leaf, second subtree")) // in proof slice + subtree2.Push([]byte("third leaf, second subtree")) // in proof slice + subtree2.Push([]byte("fourth leaf, second subtree")) + cachedTree.Push(subtree1.Root()) + cachedTree.Push(subtree2.Root()) + _, subtreeProof, _, _ = subtree2.Prove() + // Now we can create the full proof for the cached tree, without having to + // rehash any of the elements from subtree1. + merkleRoot, fullProof, _, numLeaves = cachedTree.Prove(subtreeProof) + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, fullProof, 1, 3, numLeaves) + + // Example 10: Cached tree of height 1, with proof slice consisting + // of several full subtrees. + cachedTree = merkletree.NewCachedTree(sha256.New(), 1) + cachedTree.SetSlice(2, 6) + subtree1 = merkletree.New(sha256.New()) + subtree1.Push([]byte("first leaf, first subtree")) + subtree1.Push([]byte("second leaf, first subtree")) + subtree2 = merkletree.New(sha256.New()) + subtree2.SetSlice(0, 2) + subtree2.Push([]byte("first leaf, second subtree")) // in proof slice + subtree2.Push([]byte("second leaf, second subtree")) // in proof slice + subtree3 := merkletree.New(sha256.New()) + subtree3.SetSlice(0, 2) + subtree3.Push([]byte("first leaf, third subtree")) // in proof slice + subtree3.Push([]byte("second leaf, third subtree")) // in proof slice + subtree4 := merkletree.New(sha256.New()) + subtree4.Push([]byte("first leaf, fourth subtree")) + subtree4.Push([]byte("second leaf, fourth subtree")) + cachedTree.Push(subtree1.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree4.Root()) + _, subtreeProof1, _, _ := subtree2.Prove() + _, subtreeProof2, _, _ := subtree3.Prove() + subtreeProof = append(subtreeProof1, subtreeProof2...) + merkleRoot, fullProof, _, numLeaves = cachedTree.Prove(subtreeProof) + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, fullProof, 2, 6, numLeaves) + _ = verified _ = collectiveRoot _ = revisedRoot diff --git a/cachedtree.go b/cachedtree.go index 7b6bc6f..ffd0da1 100644 --- a/cachedtree.go +++ b/cachedtree.go @@ -10,8 +10,9 @@ import ( // meaning every element added to the CachedTree is the root of a full Merkle // tree containing 2^height leaves. type CachedTree struct { - cachedNodeHeight uint64 - trueProofIndex uint64 + cachedNodeHeight uint64 + trueProofBegin, trueProofEnd uint64 + cachedBegin, cachedEnd uint64 Tree } @@ -31,22 +32,27 @@ func NewCachedTree(h hash.Hash, cachedNodeHeight uint64) *CachedTree { // Prove will create a proof that the leaf at the indicated index is a part of // the data represented by the Merkle root of the Cached Tree. The CachedTree -// needs the proof set proving that the index is an element of the cached -// element in order to create a correct proof. After proof is called, the -// CachedTree is unchanged, and can receive more elements. +// needs the proof set proving that the index or slice belongs to the cached +// element in order to create a correct proof. If SetSlice was called on a slice +// covering multiple cached elements (which means all affected cached elements +// must be covered entirely), cachedProofSet is concatenation of proofs of +// cached elements. After proof is called, the CachedTree is unchanged, and +// can receive more elements. func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { // Determine the proof index within the full tree, and the number of leaves // within the full tree. leavesPerCachedNode := uint64(1) << ct.cachedNodeHeight numLeaves = leavesPerCachedNode * ct.currentIndex + cut := ct.cachedEnd - ct.cachedBegin + // Get the proof set tail, which is generated based entirely on cached // nodes. merkleRoot, proofSetTail, _, _ := ct.Tree.Prove() - if len(proofSetTail) < 1 { + if len(proofSetTail) < int(cut) { // The proof was invalid, return 'nil' for the proof set but accurate // values for everything else. - return merkleRoot, nil, ct.trueProofIndex, numLeaves + return merkleRoot, nil, ct.trueProofBegin, numLeaves } // The full proof set is going to be the input cachedProofSet combined with @@ -54,18 +60,36 @@ func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSe // extra piece of data at the first element - the verifier will assume that // this data exists and therefore it needs to be omitted from the proof // set. - proofSet = append(cachedProofSet, proofSetTail[1:]...) - return merkleRoot, proofSet, ct.trueProofIndex, numLeaves + proofSet = append(cachedProofSet, proofSetTail[cut:]...) + return merkleRoot, proofSet, ct.trueProofBegin, numLeaves } // SetIndex will inform the CachedTree of the index of the leaf for which a // storage proof is being created. The index should be the index of the actual // leaf, and not the index of the cached element containing the leaf. SetIndex -// must be called on empty CachedTree. +// or SetSlice must be called on empty CachedTree. func (ct *CachedTree) SetIndex(i uint64) error { + return ct.SetSlice(i, i+1) +} + +// SetSlice will inform the CachedTree of the slice of leafs for which a +// storage proof is being created. Indices should be the indices of the actual +// leafs, and not the indices of the cached elements containing the leafs. +// SetIndex or SetSlice must be called on empty CachedTree. +// If SetSlice was called on a slice covering multiple cached elements, then +// all affected cached elements must be covered entirely. +func (ct *CachedTree) SetSlice(proofBegin, proofEnd uint64) error { if ct.head != nil { - return errors.New("cannot call SetIndex on Tree if Tree has not been reset") + return errors.New("cannot call SetIndex or SetSlice on Tree if Tree has not been reset") + } + ct.trueProofBegin = proofBegin + ct.trueProofEnd = proofEnd + ct.cachedBegin = proofBegin / (1 << ct.cachedNodeHeight) + ct.cachedEnd = (proofEnd-1)/(1< Date: Wed, 9 Aug 2017 23:08:25 +0200 Subject: [PATCH 16/19] implement slice proofs of cached element values --- README.md | 23 +++++++++++++++++++++++ cachedtree.go | 25 ++++++++++++++++++++++++ cachedtree_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++ verify.go | 19 ++++++++++++++++++- 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 08767e8..89fb5b0 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,29 @@ func main() { merkleRoot, fullProof, _, numLeaves = cachedTree.Prove(subtreeProof) verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, fullProof, 2, 6, numLeaves) + // Example 11: Cached tree of height 1, with proof slice consisting + // of cached elements hashes. + cachedTree = merkletree.NewCachedTree(sha256.New(), 1) + cachedTree.SetSlice(2, 6) + subtree1 = merkletree.New(sha256.New()) + subtree1.Push([]byte("first leaf, first subtree")) + subtree1.Push([]byte("second leaf, first subtree")) + subtree2 = merkletree.New(sha256.New()) + subtree2.Push([]byte("first leaf, second subtree")) // in proof slice + subtree2.Push([]byte("second leaf, second subtree")) // in proof slice + subtree3 = merkletree.New(sha256.New()) + subtree3.Push([]byte("first leaf, third subtree")) // in proof slice + subtree3.Push([]byte("second leaf, third subtree")) // in proof slice + subtree4 = merkletree.New(sha256.New()) + subtree4.Push([]byte("first leaf, fourth subtree")) + subtree4.Push([]byte("second leaf, fourth subtree")) + cachedTree.Push(subtree1.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree4.Root()) + merkleRoot, fullProof, _, numLeaves = cachedTree.ProveCached() + verified = merkletree.VerifyProofOfCachedElements(sha256.New(), merkleRoot, fullProof, 1, 3, numLeaves) + _ = verified _ = collectiveRoot _ = revisedRoot diff --git a/cachedtree.go b/cachedtree.go index ffd0da1..82a57ab 100644 --- a/cachedtree.go +++ b/cachedtree.go @@ -38,6 +38,7 @@ func NewCachedTree(h hash.Hash, cachedNodeHeight uint64) *CachedTree { // must be covered entirely), cachedProofSet is concatenation of proofs of // cached elements. After proof is called, the CachedTree is unchanged, and // can receive more elements. +// Use VerifyProof or VerifyProofOfSlice to verify proofSet returned by this method. func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { // Determine the proof index within the full tree, and the number of leaves // within the full tree. @@ -64,6 +65,30 @@ func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSe return merkleRoot, proofSet, ct.trueProofBegin, numLeaves } +// ProveCached will create a proof of cached element values. +// SetSlice must be called on a slice of leaves belonging to entire +// cached elements. +// Use VerifyProofOfCachedElements to verify proofSet returned by this method. +func (ct *CachedTree) ProveCached() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { + // Determine the proof index within the full tree, and the number of leaves + // within the full tree. + leavesPerCachedNode := uint64(1) << ct.cachedNodeHeight + numLeaves = leavesPerCachedNode * ct.currentIndex + + // Get the proof set, which is generated based entirely on cached nodes. + merkleRoot, proofSet, _, _ = ct.Tree.Prove() + if len(proofSet) < 1 { + // The proof was invalid, return 'nil' for the proof set but accurate + // values for everything else. + return merkleRoot, nil, ct.trueProofBegin, numLeaves + } + if (ct.trueProofEnd-ct.trueProofBegin)%(1< Date: Thu, 10 Aug 2017 23:08:25 +0200 Subject: [PATCH 17/19] fix golint warnings --- readers.go | 2 +- verify.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/readers.go b/readers.go index 01e3103..da9ae96 100644 --- a/readers.go +++ b/readers.go @@ -52,7 +52,7 @@ func BuildReaderProof(r io.Reader, h hash.Hash, segmentSize int, index uint64) ( return BuildReaderProofSlice(r, h, segmentSize, index, index+1) } -// BuildReaderProof returns a proof that certain data is in the merkle tree +// BuildReaderProofSlice returns a proof that certain data is in the merkle tree // created by the data in the reader. The merkle root, set of proofs, and the // number of leaves in the Merkle tree are all returned. All leaves will we // 'segmentSize' bytes except the last leaf, which will not be padded out if diff --git a/verify.go b/verify.go index 3829b62..237ee8c 100644 --- a/verify.go +++ b/verify.go @@ -22,11 +22,11 @@ func VerifyProofOfSlice(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proof return verifyProofOfSlice(h, false, merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) } -// VerifyProofOfSlice takes a Merkle root, a proofSet, and the slice and returns -// true if the first proofEnd-proofBegin elements of the proof set are roots -// of cached elements in the Merkle root. False is returned if the proof set or Merkle -// root is nil, and if 'numLeaves' equals 0. Can be used with proofs returned -// by CachedTree.ProveCached. +// VerifyProofOfCachedElements takes a Merkle root, a proofSet, and the slice +// and returns true if the first proofEnd-proofBegin elements of the proof set +// are roots of cached elements in the Merkle root. False is returned if +// the proof set or Merkle root is nil, and if 'numLeaves' equals 0. +// Can be used with proofs returned by CachedTree.ProveCached. func VerifyProofOfCachedElements(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofBegin, proofEnd, numLeaves uint64) bool { return verifyProofOfSlice(h, true, merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) } @@ -76,7 +76,7 @@ func verifyProofOfSlice(h hash.Hash, proveCached bool, merkleRoot []byte, proofS left := proofSet[0] proofSet = proofSet[1:] sums = append([][]byte{left}, sums...) - proofBegin -= 1 + proofBegin-- } if len(sums)%2 == 1 && proofEnd < numLeaves { // Example: addition of % on level <- @@ -90,7 +90,7 @@ func verifyProofOfSlice(h hash.Hash, proveCached bool, merkleRoot []byte, proofS right := proofSet[0] proofSet = proofSet[1:] sums = append(sums, right) - proofEnd += 1 + proofEnd++ } var sums2 [][]byte for len(sums) >= 2 { From 6c301d88cf338744db649f5577f7a89f33793bdb Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 11 Aug 2017 23:08:25 +0200 Subject: [PATCH 18/19] use the correct error function in a test --- tree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree_test.go b/tree_test.go index 3ba16a2..0639c9c 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1324,7 +1324,7 @@ func TestCompatibilitySlice(t *testing.T) { // Build the proof for the tree and run it through verify. _, proofSet, proofBegin, _ := tree.Prove() if proofBegin != a { - t.Error("proofBegin=%d, want %d", proofBegin, a) + t.Errorf("proofBegin=%d, want %d", proofBegin, a) } if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, a, b, numLeaves) { t.Error("proof didn't verify for indices", a, b) From cd88361a091ccb2c036ed213cd1baf48cd80367e Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Thu, 5 Oct 2017 10:16:39 +0200 Subject: [PATCH 19/19] update docs of VerifyProofOfCachedElements --- verify.go | 1 + 1 file changed, 1 insertion(+) diff --git a/verify.go b/verify.go index 237ee8c..aa2853d 100644 --- a/verify.go +++ b/verify.go @@ -27,6 +27,7 @@ func VerifyProofOfSlice(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proof // are roots of cached elements in the Merkle root. False is returned if // the proof set or Merkle root is nil, and if 'numLeaves' equals 0. // Can be used with proofs returned by CachedTree.ProveCached. +// 'numLeaves' is the total number of cached elements. func VerifyProofOfCachedElements(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofBegin, proofEnd, numLeaves uint64) bool { return verifyProofOfSlice(h, true, merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) }