diff --git a/README.md b/README.md index db36a1c..89fb5b0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -merkletree ----------- +# merkletree merkletree is a Go package for working with [Merkle trees](http://en.wikipedia.org/wiki/Merkle_tree). Specifically, this package is @@ -12,15 +11,13 @@ piece is part of the full file. When sha256 is used as the hashing algorithm, the implementation matches the merkle tree described in RFC 6962, 'Certificate Transparency'. -Usage ------ +## Usage ```go package main import ( "crypto/sha256" - "log" "os" "github.com/NebulousLabs/merkletree" @@ -38,7 +35,7 @@ func main() { file.Seek(0, 0) // Offset needs to be set back to 0. proofIndex := uint64(7) merkleRoot, proof, numLeaves, _ := merkletree.BuildReaderProof(file, sha256.New(), segmentSize, proofIndex) - verified := VerifyProof(sha256.New(), merkleRoot, proof, proofIndex, numLeaves) + verified := merkletree.VerifyProof(sha256.New(), merkleRoot, proof, proofIndex, numLeaves) // Example 3: Using a Tree to build a merkle tree and get a proof for a // specific index for non-file objects. @@ -48,11 +45,11 @@ func main() { tree.Push([]byte("another object")) // The merkle root could be obtained by calling tree.Root(), but will also // be provided by tree.Prove() - merkleRoot, proof, proofIndex, numLeaves := tree.Prove() + merkleRoot, proof, proofIndex, numLeaves = tree.Prove() - //////////////////////////////////////////////// - /// Remaining examples deal with cached trees // - //////////////////////////////////////////////// + //////////////////////////////////////////////////// + /// Next group of examples deal with cached trees // + //////////////////////////////////////////////////// // Example 4: Creating a cached set of Merkle roots and then using them in // a cached tree. The cached tree is height 1, meaning that all elements of @@ -96,14 +93,115 @@ func main() { // Now we can create the full proof for the cached tree, without having to // rehash any of the elements from subtree1. _, fullProof, _, _ := cachedTree.Prove(subtreeProof) + + //////////////////////////////////////////////////////// + /// Next group of examples deal with proofs of slices // + //////////////////////////////////////////////////////// + + // Example 7: Using a Tree to build a merkle tree and get a proof for a + // specific slice for non-file objects. + tree = merkletree.New(sha256.New()) + tree.SetSlice(1, 3) // Objects 1 and 2. + tree.Push([]byte("an object - the tree will hash the data after it is pushed")) + tree.Push([]byte("the first part of the slice")) + tree.Push([]byte("the second part of the slice")) + tree.Push([]byte("another object")) + merkleRoot, proof, _, numLeaves = tree.Prove() + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, proof, 1, 3, numLeaves) + + // Example 8: Build and verify a proof that the elements at segments 5-10 + // are in the merkle root. The proof starts with the elements themselves. + file.Seek(0, 0) // Offset needs to be set back to 0. + proofBegin := uint64(5) + proofEnd := uint64(10) + 1 + merkleRoot, proof, numLeaves, _ = merkletree.BuildReaderProofSlice(file, sha256.New(), segmentSize, proofBegin, proofEnd) + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, proof, proofBegin, proofEnd, numLeaves) + + // Example 9: Cached tree of height 2, with proof slice entirely inside + // one cached subtree. + cachedTree = merkletree.NewCachedTree(sha256.New(), 2) + cachedTree.SetSlice(5, 7) + subtree1 = merkletree.New(sha256.New()) + subtree1.Push([]byte("first leaf, first subtree")) + subtree1.Push([]byte("second leaf, first subtree")) + subtree1.Push([]byte("third leaf, first subtree")) + subtree1.Push([]byte("fourth leaf, first subtree")) + subtree2 = merkletree.New(sha256.New()) + subtree2.SetSlice(1, 3) + subtree2.Push([]byte("first leaf, second subtree")) + subtree2.Push([]byte("second leaf, second subtree")) // in proof slice + subtree2.Push([]byte("third leaf, second subtree")) // in proof slice + subtree2.Push([]byte("fourth leaf, second subtree")) + cachedTree.Push(subtree1.Root()) + cachedTree.Push(subtree2.Root()) + _, subtreeProof, _, _ = subtree2.Prove() + // Now we can create the full proof for the cached tree, without having to + // rehash any of the elements from subtree1. + merkleRoot, fullProof, _, numLeaves = cachedTree.Prove(subtreeProof) + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, fullProof, 1, 3, numLeaves) + + // Example 10: Cached tree of height 1, with proof slice consisting + // of several full subtrees. + cachedTree = merkletree.NewCachedTree(sha256.New(), 1) + cachedTree.SetSlice(2, 6) + subtree1 = merkletree.New(sha256.New()) + subtree1.Push([]byte("first leaf, first subtree")) + subtree1.Push([]byte("second leaf, first subtree")) + subtree2 = merkletree.New(sha256.New()) + subtree2.SetSlice(0, 2) + subtree2.Push([]byte("first leaf, second subtree")) // in proof slice + subtree2.Push([]byte("second leaf, second subtree")) // in proof slice + subtree3 := merkletree.New(sha256.New()) + subtree3.SetSlice(0, 2) + subtree3.Push([]byte("first leaf, third subtree")) // in proof slice + subtree3.Push([]byte("second leaf, third subtree")) // in proof slice + subtree4 := merkletree.New(sha256.New()) + subtree4.Push([]byte("first leaf, fourth subtree")) + subtree4.Push([]byte("second leaf, fourth subtree")) + cachedTree.Push(subtree1.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree4.Root()) + _, subtreeProof1, _, _ := subtree2.Prove() + _, subtreeProof2, _, _ := subtree3.Prove() + subtreeProof = append(subtreeProof1, subtreeProof2...) + merkleRoot, fullProof, _, numLeaves = cachedTree.Prove(subtreeProof) + verified = merkletree.VerifyProofOfSlice(sha256.New(), merkleRoot, fullProof, 2, 6, numLeaves) + + // Example 11: Cached tree of height 1, with proof slice consisting + // of cached elements hashes. + cachedTree = merkletree.NewCachedTree(sha256.New(), 1) + cachedTree.SetSlice(2, 6) + subtree1 = merkletree.New(sha256.New()) + subtree1.Push([]byte("first leaf, first subtree")) + subtree1.Push([]byte("second leaf, first subtree")) + subtree2 = merkletree.New(sha256.New()) + subtree2.Push([]byte("first leaf, second subtree")) // in proof slice + subtree2.Push([]byte("second leaf, second subtree")) // in proof slice + subtree3 = merkletree.New(sha256.New()) + subtree3.Push([]byte("first leaf, third subtree")) // in proof slice + subtree3.Push([]byte("second leaf, third subtree")) // in proof slice + subtree4 = merkletree.New(sha256.New()) + subtree4.Push([]byte("first leaf, fourth subtree")) + subtree4.Push([]byte("second leaf, fourth subtree")) + cachedTree.Push(subtree1.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree2.Root()) + cachedTree.Push(subtree4.Root()) + merkleRoot, fullProof, _, numLeaves = cachedTree.ProveCached() + verified = merkletree.VerifyProofOfCachedElements(sha256.New(), merkleRoot, fullProof, 1, 3, numLeaves) + + _ = verified + _ = collectiveRoot + _ = revisedRoot + _ = fullProof } ``` For more extensive documentation, refer to the [godoc](http://godoc.org/github.com/NebulousLabs/merkletree). -Notes ------ +## Notes This implementation does not retain the entire Merkle tree in memory. Rather, as each new leaf is added to the tree, is it pushed onto a stack as a "subtree @@ -127,3 +225,84 @@ hashed multiple times. When using the Reader functions (ReaderRoot and BuildReaderProof), the last segment will not be padded if there are not 'segmentSize' bytes remaining. + +## Format of proof + +### What is included to the proof + +A proof is a slice of slices of bytes. It begins with the leave data, +then hashes of subtrees follow. Combining all leaves which are covered in +these two groups (as leaves from the beginning of the proof or as leaves +from the subtrees whose hashes constitute the second part of the proof) +we get all leaves of the tree and each leave presents once. + +Example. Proof built in a tree of 5 leaves for element at index 2: + +``` + ┌───┴──* + *──┴──┐ │ +┌─┴─┐ ┌─┴─* │ +0 1 2 3 4 + * +``` + +Parts of the proof are marked with asterisks (*). + +If we build a proof for a slice, the rule is the same: first include all +leaves from the target slice, then add hashes of all subtrees so that +together with the target slice they cover all leaves, once. + +Example. Proof built in a tree of 7 leaves for the slice [2, 5). + +``` + ┌─────┴─────┐ + *──┴──┐ ┌──┴──* +┌─┴─┐ ┌─┴─┐ ┌─┴─* │ +0 1 2 3 4 5 6 + * * * +``` + +Example. Proof built in a tree of 7 leaves for the slice [3, 5). + +``` + ┌─────┴─────┐ + *──┴──┐ ┌──┴──* +┌─┴─┐ *─┴─┐ ┌─┴─* │ +0 1 2 3 4 5 6 + * * +``` + +### The order of stuff in the proof + +The proof starts with the data items. For a proof of one element +it is the element itself (one item in the main proof slice). +In case of slice the data is represented as multiple items in the main +proof slice, in the order of occurrence in the source data. + +Hashes of subtrees (constituting the second half of a proof) are sorted +by height (ascending), then by occurrence in the source data. The height +of an orphan subtree is equal to the height of its parent minus one. + +Some examples of how parts of proofs are ordered. A number corresponds +to the place of this leave or subtree hash in the proof. + +``` + ┌────┴───┐ + 5──┴──┐ │ +┌─┴─┐ 3─┴─┐ ┌─┴─4 + 1 2 +``` + +``` + ┌────┴───4 + ┌──┴──┐ │ +3─┴─┐ ┌─┴─┐ ┌─┴─┐ + 1 2 3 +``` + +``` + ┌────┴───┐ + 5──┴──┐ │ +┌─┴─┐ ┌─┴─┐ ┌─┴─┐ + 1 2 3 4 +``` diff --git a/cachedtree.go b/cachedtree.go index 7b6bc6f..82a57ab 100644 --- a/cachedtree.go +++ b/cachedtree.go @@ -10,8 +10,9 @@ import ( // meaning every element added to the CachedTree is the root of a full Merkle // tree containing 2^height leaves. type CachedTree struct { - cachedNodeHeight uint64 - trueProofIndex uint64 + cachedNodeHeight uint64 + trueProofBegin, trueProofEnd uint64 + cachedBegin, cachedEnd uint64 Tree } @@ -31,22 +32,28 @@ func NewCachedTree(h hash.Hash, cachedNodeHeight uint64) *CachedTree { // Prove will create a proof that the leaf at the indicated index is a part of // the data represented by the Merkle root of the Cached Tree. The CachedTree -// needs the proof set proving that the index is an element of the cached -// element in order to create a correct proof. After proof is called, the -// CachedTree is unchanged, and can receive more elements. +// needs the proof set proving that the index or slice belongs to the cached +// element in order to create a correct proof. If SetSlice was called on a slice +// covering multiple cached elements (which means all affected cached elements +// must be covered entirely), cachedProofSet is concatenation of proofs of +// cached elements. After proof is called, the CachedTree is unchanged, and +// can receive more elements. +// Use VerifyProof or VerifyProofOfSlice to verify proofSet returned by this method. func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { // Determine the proof index within the full tree, and the number of leaves // within the full tree. leavesPerCachedNode := uint64(1) << ct.cachedNodeHeight numLeaves = leavesPerCachedNode * ct.currentIndex + cut := ct.cachedEnd - ct.cachedBegin + // Get the proof set tail, which is generated based entirely on cached // nodes. merkleRoot, proofSetTail, _, _ := ct.Tree.Prove() - if len(proofSetTail) < 1 { + if len(proofSetTail) < int(cut) { // The proof was invalid, return 'nil' for the proof set but accurate // values for everything else. - return merkleRoot, nil, ct.trueProofIndex, numLeaves + return merkleRoot, nil, ct.trueProofBegin, numLeaves } // The full proof set is going to be the input cachedProofSet combined with @@ -54,18 +61,60 @@ func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSe // extra piece of data at the first element - the verifier will assume that // this data exists and therefore it needs to be omitted from the proof // set. - proofSet = append(cachedProofSet, proofSetTail[1:]...) - return merkleRoot, proofSet, ct.trueProofIndex, numLeaves + proofSet = append(cachedProofSet, proofSetTail[cut:]...) + return merkleRoot, proofSet, ct.trueProofBegin, numLeaves +} + +// ProveCached will create a proof of cached element values. +// SetSlice must be called on a slice of leaves belonging to entire +// cached elements. +// Use VerifyProofOfCachedElements to verify proofSet returned by this method. +func (ct *CachedTree) ProveCached() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { + // Determine the proof index within the full tree, and the number of leaves + // within the full tree. + leavesPerCachedNode := uint64(1) << ct.cachedNodeHeight + numLeaves = leavesPerCachedNode * ct.currentIndex + + // Get the proof set, which is generated based entirely on cached nodes. + merkleRoot, proofSet, _, _ = ct.Tree.Prove() + if len(proofSet) < 1 { + // The proof was invalid, return 'nil' for the proof set but accurate + // values for everything else. + return merkleRoot, nil, ct.trueProofBegin, numLeaves + } + if (ct.trueProofEnd-ct.trueProofBegin)%(1< 2 { + panic("More than 2 proofs of same height in proofSet") + } + for _, proof := range step { + proofs = append(proofs, proof) + } + } + return proofs +} + // New creates a new Tree. The provided hash will be used for all hashing // operations within the Tree. func New(h hash.Hash) *Tree { return &Tree{ - hash: h, + hash: h, + proofBegin: 0, + proofEnd: 1, } } // Prove creates a proof that the leaf at the established index (established by -// SetIndex) is an element of the Merkle tree. Prove will return a nil proof -// set if used incorrectly. Prove does not modify the Tree. -func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { - // Return nil if the Tree is empty, or if the proofIndex hasn't yet been +// SetIndex) or the slice of leaves (established by SetSlice) belongs to the +// Merkle tree. Prove will return a nil proof set if used incorrectly. +// Prove does not modify the Tree. +func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofBegin uint64, numLeaves uint64) { + // Return nil if the Tree is empty, or if the proofEnd hasn't yet been // reached. - if t.head == nil || len(t.proofSet) == 0 { - return t.Root(), nil, t.proofIndex, t.currentIndex + if t.head == nil || t.currentIndex < t.proofEnd { + return t.Root(), nil, t.proofBegin, t.currentIndex } - proofSet = t.proofSet + proofSet = make([][]byte, len(t.bases)) + copy(proofSet, t.bases) - // The set of subtrees must now be collapsed into a single root. The proof - // set already contains all of the elements that are members of a complete - // subtree. Of what remains, there will be at most 1 element provided from - // a sibling on the right, and all of the other proofs will be provided - // from a sibling on the left. This results from the way orphans are - // treated. All subtrees smaller than the subtree containing the proofIndex - // will be combined into a single subtree that gets combined with the - // proofIndex subtree as a single right sibling. All subtrees larger than - // the subtree containing the proofIndex will be combined with the subtree - // containing the proof index as left siblings. - - // Start at the smallest subtree and combine it with larger subtrees until - // it would be combining with the subtree that contains the proof index. We - // can recognize the subtree containing the proof index because the height - // of that subtree will be one less than the current length of the proof - // set. + // The set of subtrees must now be collapsed into a single root. + // Unlike Push, now we ignore previous height of the right subtree, + // because we have to collapse all leaves anyway. + // All needed hashes are added to the ladder. current := t.head - for current.next != nil && current.next.height < len(proofSet)-1 { + lader := cloneLader(t.lader) + for current.next != nil { + // The left subtree is higher or equal to the right subtree. + // If the right subtree is incomplete, its height is considered + // to be equal to its left sibling. + height := current.next.height + if current.contains(t) && !current.next.contains(t) { + lader = addToLader(lader, height, current.next.sum) + } else if !current.contains(t) && current.next.contains(t) { + lader = addToLader(lader, height, current.sum) + } + current = joinSubTrees(t.hash, current.next, current) } + proofSet = append(proofSet, foldLader(lader)...) // Sanity check - check that either 'current' or 'current.next' is the - // subtree containing the proof index. + // subtree containing the proof slice. if DEBUG { - if current.height != len(t.proofSet)-1 && (current.next != nil && current.next.height != len(t.proofSet)-1) { - panic("could not find the subtree containing the proof index") + if !current.contains(t) && !current.next.contains(t) { + panic("could not find the subtree containing the proof slice") } } - - // If the current subtree is not the subtree containing the proof index, - // then it must be an aggregate subtree that is to the right of the subtree - // containing the proof index, and the next subtree is the subtree - // containing the proof index. - if current.next != nil && current.next.height == len(proofSet)-1 { - proofSet = append(proofSet, current.sum) - current = current.next - } - - // The current subtree must be the subtree containing the proof index. This - // subtree does not need an entry, as the entry was created during the - // construction of the Tree. Instead, skip to the next subtree. - current = current.next - - // All remaining subtrees will be added to the proof set as a left sibling, - // completing the proof set. - for current != nil { - proofSet = append(proofSet, current.sum) - current = current.next - } - return t.Root(), proofSet, t.proofIndex, t.currentIndex + return t.Root(), proofSet, t.proofBegin, t.currentIndex } // Push will add data to the set, building out the Merkle tree and Root. The @@ -166,8 +205,8 @@ func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, func (t *Tree) Push(data []byte) { // The first element of a proof is the data at the proof index. If this // data is being inserted at the proof index, it is added to the proof set. - if t.currentIndex == t.proofIndex { - t.proofSet = append(t.proofSet, data) + if t.proofBegin <= t.currentIndex && t.currentIndex < t.proofEnd { + t.bases = append(t.bases, data) } // Hash the data to create a subtree of height 0. The sum of the new node @@ -177,6 +216,8 @@ func (t *Tree) Push(data []byte) { t.head = &subTree{ next: t.head, height: 0, + begin: t.currentIndex, + end: t.currentIndex + 1, } if t.cachedTree { t.head.sum = data @@ -189,32 +230,18 @@ func (t *Tree) Push(data []byte) { // be combined into a single subTree of height n+1. for t.head.next != nil && t.head.height == t.head.next.height { // Before combining subtrees, check whether one of the subtree hashes - // needs to be added to the proof set. This is going to be true IFF the - // subtrees being combined are one height higher than the previous - // subtree added to the proof set. The height of the previous subtree - // added to the proof set is equal to len(t.proofSet) - 1. - if t.head.height == len(t.proofSet)-1 { - // One of the subtrees needs to be added to the proof set. The - // subtree that needs to be added is the subtree that does not - // contain the proofIndex. Because the subtrees being compared are - // the smallest and rightmost trees in the Tree, this can be - // determined by rounding the currentIndex down to the number of - // nodes in the subtree and comparing that index to the proofIndex. - leaves := uint64(1 << uint(t.head.height)) - mid := (t.currentIndex / leaves) * leaves - if t.proofIndex < mid { - t.proofSet = append(t.proofSet, t.head.sum) - } else { - t.proofSet = append(t.proofSet, t.head.next.sum) - } + // needs to be added to the proof set. This is going to be true IFF + // one of the subtrees being combined overlaps with the target slice + // and another does not. + var proof []byte + if t.head.contains(t) && !t.head.next.contains(t) { + proof = t.head.next.sum + } else if !t.head.contains(t) && t.head.next.contains(t) { + proof = t.head.sum + } - // Sanity check - the proofIndex should never be less than the - // midpoint minus the number of leaves in each subtree. - if DEBUG { - if t.proofIndex < mid-leaves { - panic("proof being added with weird values") - } - } + if proof != nil { + t.lader = addToLader(t.lader, t.head.height, proof) } // Join the two subTrees into one subTree with a greater height. Then @@ -258,9 +285,16 @@ func (t *Tree) Root() []byte { // SetIndex will tell the Tree to create a storage proof for the leaf at the // input index. SetIndex must be called on an empty tree. func (t *Tree) SetIndex(i uint64) error { + return t.SetSlice(i, i+1) +} + +// SetSlice will tell the Tree to create a storage proof for the leaves +// within the slice [begin, end). SetSlice must be called on an empty tree. +func (t *Tree) SetSlice(begin, end uint64) error { if t.head != nil { - return errors.New("cannot call SetIndex on Tree if Tree has not been reset") + return errors.New("cannot call SetIndex or SetSlice on Tree if Tree has not been reset") } - t.proofIndex = i + t.proofBegin = begin + t.proofEnd = end return nil } diff --git a/tree_test.go b/tree_test.go index dab1320..0639c9c 100644 --- a/tree_test.go +++ b/tree_test.go @@ -18,7 +18,7 @@ type MerkleTester struct { // leaves is the hashes of the data, and should be the same length. leaves [][]byte - // roots contains the root hashes of Merkle trees of various heights using + // roots contains the root hashes of Merkle trees of various lengths using // the data for input. roots map[int][]byte @@ -27,6 +27,14 @@ type MerkleTester struct { // root of that tree can be found in roots. The second map is the // proofIndex that was used when building the proof. proofSets map[int]map[int][][]byte + + // sliceProofSets contains proofs that certain slices of leaves is in a Merkle tree. The + // first map is the number of leaves in the tree that the proof is for. The + // root of that tree can be found in roots. The second map is the + // proofBegin and the third map is the proofEnd that was used when + // building the proof. + sliceProofSets map[int]map[int]map[int][][]byte + *testing.T } @@ -41,13 +49,14 @@ func (mt *MerkleTester) join(a, b []byte) []byte { // Tree creates. func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { mt = &MerkleTester{ - roots: make(map[int][]byte), - proofSets: make(map[int]map[int][][]byte), + roots: make(map[int][]byte), + proofSets: make(map[int]map[int][][]byte), + sliceProofSets: make(map[int]map[int]map[int][][]byte), } mt.T = t // Fill out the data and leaves values. - size := 16 + size := 100 for i := 0; i < size; i++ { mt.data = append(mt.data, []byte{byte(i)}) } @@ -110,6 +119,155 @@ func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { ), ) + mt.roots[100] = mt.join( + mt.join( + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + mt.join( + mt.join(mt.leaves[4], mt.leaves[5]), + mt.join(mt.leaves[6], mt.leaves[7]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[8], mt.leaves[9]), + mt.join(mt.leaves[10], mt.leaves[11]), + ), + mt.join( + mt.join(mt.leaves[12], mt.leaves[13]), + mt.join(mt.leaves[14], mt.leaves[15]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[16], mt.leaves[17]), + mt.join(mt.leaves[18], mt.leaves[19]), + ), + mt.join( + mt.join(mt.leaves[20], mt.leaves[21]), + mt.join(mt.leaves[22], mt.leaves[23]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[24], mt.leaves[25]), + mt.join(mt.leaves[26], mt.leaves[27]), + ), + mt.join( + mt.join(mt.leaves[28], mt.leaves[29]), + mt.join(mt.leaves[30], mt.leaves[31]), + ), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[32], mt.leaves[33]), + mt.join(mt.leaves[34], mt.leaves[35]), + ), + mt.join( + mt.join(mt.leaves[36], mt.leaves[37]), + mt.join(mt.leaves[38], mt.leaves[39]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[40], mt.leaves[41]), + mt.join(mt.leaves[42], mt.leaves[43]), + ), + mt.join( + mt.join(mt.leaves[44], mt.leaves[45]), + mt.join(mt.leaves[46], mt.leaves[47]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[48], mt.leaves[49]), + mt.join(mt.leaves[50], mt.leaves[51]), + ), + mt.join( + mt.join(mt.leaves[52], mt.leaves[53]), + mt.join(mt.leaves[54], mt.leaves[55]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[56], mt.leaves[57]), + mt.join(mt.leaves[58], mt.leaves[59]), + ), + mt.join( + mt.join(mt.leaves[60], mt.leaves[61]), + mt.join(mt.leaves[62], mt.leaves[63]), + ), + ), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[64], mt.leaves[65]), + mt.join(mt.leaves[66], mt.leaves[67]), + ), + mt.join( + mt.join(mt.leaves[68], mt.leaves[69]), + mt.join(mt.leaves[70], mt.leaves[71]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[72], mt.leaves[73]), + mt.join(mt.leaves[74], mt.leaves[75]), + ), + mt.join( + mt.join(mt.leaves[76], mt.leaves[77]), + mt.join(mt.leaves[78], mt.leaves[79]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[80], mt.leaves[81]), + mt.join(mt.leaves[82], mt.leaves[83]), + ), + mt.join( + mt.join(mt.leaves[84], mt.leaves[85]), + mt.join(mt.leaves[86], mt.leaves[87]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[88], mt.leaves[89]), + mt.join(mt.leaves[90], mt.leaves[91]), + ), + mt.join( + mt.join(mt.leaves[92], mt.leaves[93]), + mt.join(mt.leaves[94], mt.leaves[95]), + ), + ), + ), + ), + mt.join( + mt.join(mt.leaves[96], mt.leaves[97]), + mt.join(mt.leaves[98], mt.leaves[99]), + ), + ), + ) + // Manually build out some proof sets that should should match what the // Tree creates for the same values. mt.proofSets[1] = make(map[int][][]byte) @@ -222,6 +380,550 @@ func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { mt.roots[8], } + // Manually build out some slice proof sets that should should + // match what the Tree creates for the same values. + + mt.sliceProofSets[1] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{mt.data[0]}, + }, + } + + mt.sliceProofSets[2] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + }, + }, + } + + mt.sliceProofSets[3] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.leaves[2], + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.leaves[2], + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.leaves[2], + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + } + + mt.sliceProofSets[4] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.join(mt.leaves[2], mt.leaves[3]), + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.join(mt.leaves[2], mt.leaves[3]), + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.leaves[3], + }, + 4: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.join(mt.leaves[2], mt.leaves[3]), + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + mt.leaves[3], + }, + 4: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + 4: [][]byte{ + mt.data[2], + mt.data[3], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 3: map[int][][]byte{ + 4: [][]byte{ + mt.data[3], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + } + + mt.sliceProofSets[5] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.leaves[4], + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.leaves[4], + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.leaves[3], + mt.leaves[4], + }, + 4: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.leaves[4], + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + mt.leaves[3], + mt.leaves[4], + }, + 4: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[0], + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.leaves[4], + }, + 4: [][]byte{ + mt.data[2], + mt.data[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[2], + mt.data[3], + mt.data[4], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 3: map[int][][]byte{ + 4: [][]byte{ + mt.data[3], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.leaves[4], + }, + 5: [][]byte{ + mt.data[3], + mt.data[4], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 4: map[int][][]byte{ + 5: [][]byte{ + mt.data[4], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + }, + } + + mt.sliceProofSets[6] = map[int]map[int][][]byte{ + 0: map[int][][]byte{ + 1: [][]byte{ + mt.data[0], + mt.leaves[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 2: [][]byte{ + mt.data[0], + mt.data[1], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 3: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 4: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[5], + }, + 6: [][]byte{ + mt.data[0], + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.data[5], + }, + }, + 1: map[int][][]byte{ + 2: [][]byte{ + mt.data[1], + mt.leaves[0], + mt.join(mt.leaves[2], mt.leaves[3]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 3: [][]byte{ + mt.data[1], + mt.data[2], + mt.leaves[0], + mt.leaves[3], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 4: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.leaves[0], + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[0], + mt.leaves[5], + }, + 6: [][]byte{ + mt.data[1], + mt.data[2], + mt.data[3], + mt.data[4], + mt.data[5], + mt.leaves[0], + }, + }, + 2: map[int][][]byte{ + 3: [][]byte{ + mt.data[2], + mt.leaves[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 4: [][]byte{ + mt.data[2], + mt.data[3], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[2], + mt.data[3], + mt.data[4], + mt.leaves[5], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + 6: [][]byte{ + mt.data[2], + mt.data[3], + mt.data[4], + mt.data[5], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 3: map[int][][]byte{ + 4: [][]byte{ + mt.data[3], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[4], mt.leaves[5]), + }, + 5: [][]byte{ + mt.data[3], + mt.data[4], + mt.leaves[2], + mt.leaves[5], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + 6: [][]byte{ + mt.data[3], + mt.data[4], + mt.data[5], + mt.leaves[2], + mt.join(mt.leaves[0], mt.leaves[1]), + }, + }, + 4: map[int][][]byte{ + 5: [][]byte{ + mt.data[4], + mt.leaves[5], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + 6: [][]byte{ + mt.data[4], + mt.data[5], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + }, + 5: map[int][][]byte{ + 6: [][]byte{ + mt.data[5], + mt.leaves[4], + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + }, + }, + } + + mt.sliceProofSets[100] = map[int]map[int][][]byte{ + 17: map[int][][]byte{ + 43: [][]byte{ + mt.data[17], + mt.data[18], + mt.data[19], + mt.data[20], + mt.data[21], + mt.data[22], + mt.data[23], + mt.data[24], + mt.data[25], + mt.data[26], + mt.data[27], + mt.data[28], + mt.data[29], + mt.data[30], + mt.data[31], + mt.data[32], + mt.data[33], + mt.data[34], + mt.data[35], + mt.data[36], + mt.data[37], + mt.data[38], + mt.data[39], + mt.data[40], + mt.data[41], + mt.data[42], + mt.leaves[16], + mt.leaves[43], + // 44-47. + mt.join( + mt.join(mt.leaves[44], mt.leaves[45]), + mt.join(mt.leaves[46], mt.leaves[47]), + ), + // 0-15. + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[0], mt.leaves[1]), + mt.join(mt.leaves[2], mt.leaves[3]), + ), + mt.join( + mt.join(mt.leaves[4], mt.leaves[5]), + mt.join(mt.leaves[6], mt.leaves[7]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[8], mt.leaves[9]), + mt.join(mt.leaves[10], mt.leaves[11]), + ), + mt.join( + mt.join(mt.leaves[12], mt.leaves[13]), + mt.join(mt.leaves[14], mt.leaves[15]), + ), + ), + ), + // 48-63. + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[48], mt.leaves[49]), + mt.join(mt.leaves[50], mt.leaves[51]), + ), + mt.join( + mt.join(mt.leaves[52], mt.leaves[53]), + mt.join(mt.leaves[54], mt.leaves[55]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[56], mt.leaves[57]), + mt.join(mt.leaves[58], mt.leaves[59]), + ), + mt.join( + mt.join(mt.leaves[60], mt.leaves[61]), + mt.join(mt.leaves[62], mt.leaves[63]), + ), + ), + ), + mt.join( + // 64-95. + mt.join( + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[64], mt.leaves[65]), + mt.join(mt.leaves[66], mt.leaves[67]), + ), + mt.join( + mt.join(mt.leaves[68], mt.leaves[69]), + mt.join(mt.leaves[70], mt.leaves[71]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[72], mt.leaves[73]), + mt.join(mt.leaves[74], mt.leaves[75]), + ), + mt.join( + mt.join(mt.leaves[76], mt.leaves[77]), + mt.join(mt.leaves[78], mt.leaves[79]), + ), + ), + ), + mt.join( + mt.join( + mt.join( + mt.join(mt.leaves[80], mt.leaves[81]), + mt.join(mt.leaves[82], mt.leaves[83]), + ), + mt.join( + mt.join(mt.leaves[84], mt.leaves[85]), + mt.join(mt.leaves[86], mt.leaves[87]), + ), + ), + mt.join( + mt.join( + mt.join(mt.leaves[88], mt.leaves[89]), + mt.join(mt.leaves[90], mt.leaves[91]), + ), + mt.join( + mt.join(mt.leaves[92], mt.leaves[93]), + mt.join(mt.leaves[94], mt.leaves[95]), + ), + ), + ), + ), + // 96-99. + mt.join( + mt.join(mt.leaves[96], mt.leaves[97]), + mt.join(mt.leaves[98], mt.leaves[99]), + ), + ), + }, + }, + } + return } @@ -330,6 +1032,91 @@ func TestBuildAndVerifyProof(t *testing.T) { } } +// TestBuildAndVerifyProofOfSlice builds a proof using a tree for every single +// manually created proof of slice in the MerkleTester. Then it checks that the proof +// matches the manually created proof, and that the proof is verified by +// VerifyProofOfSlice. Then it checks that the proof fails for all other slices, +// which should happen if all of the leaves are unique. +func TestBuildAndVerifyProofOfSlice(t *testing.T) { + mt := CreateMerkleTester(t) + + // Compare the results of building a Merkle proof to all of the manually + // constructed proofs. + tree := New(sha256.New()) + for i, manualProveSets := range mt.sliceProofSets { + for j, manualProveSets2 := range manualProveSets { + for l, expectedProveSet := range manualProveSets2 { + // Build out the tree. + tree = New(sha256.New()) + err := tree.SetSlice(uint64(j), uint64(l)) + if err != nil { + t.Fatal(err) + } + for k := 0; k < i; k++ { + tree.Push(mt.data[k]) + } + + // Get the proof and check all values. + merkleRoot, proofSet, proofIndex, numSegments := tree.Prove() + if bytes.Compare(merkleRoot, mt.roots[i]) != 0 { + t.Error("incorrect Merkle root returned by Tree for indices", i, j, l) + } + if len(proofSet) != len(expectedProveSet) { + t.Error("proof set is wrong length for indices", i, j, l) + continue + } + if proofIndex != uint64(j) { + t.Error("incorrect proofIndex returned for indices", i, j, l) + } + if numSegments != uint64(i) { + t.Error("incorrect numSegments returned for indices", i, j, l) + } + for k := range proofSet { + if bytes.Compare(proofSet[k], expectedProveSet[k]) != 0 { + t.Error("proof set does not match expected proof set for indices", i, j, l, k) + } + } + + // Check that verification works on for the desired proof index but + // fails for all other indices. + if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, uint64(j), uint64(l), numSegments) { + t.Error("proof set does not verify for indices", i, j, l) + } + for k := uint64(0); k < uint64(i); k++ { + for m := k + 1; m < uint64(i); m++ { + if k == uint64(j) && m == uint64(l) { + continue + } + if VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, uint64(k), uint64(m), uint64(i)) { + t.Error("proof set verifies for wrong slice at indices", i, j, l, k, m) + } + } + } + + // Check that calling Prove a second time results in the same values. + merkleRoot2, proofSet2, proofIndex2, numSegments2 := tree.Prove() + if bytes.Compare(merkleRoot, merkleRoot2) != 0 { + t.Error("tree returned different merkle roots after calling Prove twice for indices", i, j, l) + } + if len(proofSet) != len(proofSet2) { + t.Error("tree returned different proof sets after calling Prove twice for indices", i, j, l) + } + for k := range proofSet { + if bytes.Compare(proofSet[k], proofSet2[k]) != 0 { + t.Error("tree returned different proof sets after calling Prove twice for indices", i, j, l) + } + } + if proofIndex != proofIndex2 { + t.Error("tree returned different proof indexes after calling Prove twice for indices", i, j, l) + } + if numSegments != numSegments2 { + t.Error("tree returned different segment count after calling Prove twice for indices", i, j, l) + } + } + } + } +} + // TestBadInputs provides malicious inputs to the functions of the package, // trying to trigger panics or unexpected behavior. func TestBadInputs(t *testing.T) { @@ -358,6 +1145,30 @@ func TestBadInputs(t *testing.T) { if err == nil { t.Error("expecting error, shouldn't be able to reset a tree after pushing") } + err = tree.SetSlice(1, 3) + if err == nil { + t.Error("expecting error, shouldn't be able to reset a tree after pushing") + } + + // Get the proof of a tree that hasn't reached it's index (slice version). + tree2 := New(sha256.New()) + err = tree2.SetSlice(1, 10) + if err != nil { + t.Fatal(err) + } + tree2.Push([]byte{1}) + _, proof, _, _ = tree2.Prove() + if proof != nil { + t.Fatal(err) + } + err = tree2.SetIndex(2) + if err == nil { + t.Error("expecting error, shouldn't be able to reset a tree after pushing") + } + err = tree2.SetSlice(1, 3) + if err == nil { + t.Error("expecting error, shouldn't be able to reset a tree after pushing") + } // Try nil values in VerifyProof. mt := CreateMerkleTester(t) @@ -376,6 +1187,26 @@ func TestBadInputs(t *testing.T) { if VerifyProof(sha256.New(), mt.roots[15], mt.proofSets[15][10], 15, 0) { t.Error("VerifyProof should return false when numLeaves is 0") } + + // Try nil values in VerifyProofOfSlice. + if VerifyProofOfSlice(sha256.New(), nil, mt.proofSets[1][0], 0, 1, 1) { + t.Error("VerifyProofOfSlice should return false for nil merkle root") + } + if VerifyProofOfSlice(sha256.New(), []byte{1}, nil, 0, 1, 1) { + t.Error("VerifyProofOfSlice should return false for nil proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][3][1:], 3, 4, 15) { + t.Error("VerifyProofOfSlice should return false for too-short proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][10][1:], 10, 11, 15) { + t.Error("VerifyProofOfSlice should return false for too-short proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.sliceProofSets[4][2][4][1:], 2, 4, 4) { + t.Error("VerifyProofOfSlice should return false for too-short proof set") + } + if VerifyProofOfSlice(sha256.New(), mt.roots[15], mt.proofSets[15][10], 15, 30, 0) { + t.Error("VerifyProofOfSlice should return false when numLeaves is 0") + } } // TestCompatibility runs BuildProof for a large set of trees, and checks that @@ -424,7 +1255,7 @@ func TestCompatibility(t *testing.T) { // Check that proofs on larger trees are consistent. for i := 0; i < 25; i++ { - // Determine a random size for the tree up to 64M elements. + // Determine a random size for the tree up to 256k elements. sizeI, err := rand.Int(rand.Reader, big.NewInt(256e3)) if err != nil { t.Fatal(err) @@ -458,6 +1289,114 @@ func TestCompatibility(t *testing.T) { } } +func TestCompatibilitySlice(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Brute force all trees up to size 'max'. Running time for this test is max^5. + max := uint64(15) + tree := New(sha256.New()) + for numLeaves := uint64(1); numLeaves < max; numLeaves++ { + // Make merkleRoot using regular Prove. + tree = New(sha256.New()) + err := tree.SetIndex(0) + if err != nil { + t.Fatal(err) + } + for k := uint64(0); k < numLeaves; k++ { + tree.Push([]byte{byte(k)}) + } + merkleRoot, _, _, _ := tree.Prove() + + // Try with proof at every possible slice. + for a := uint64(0); a < numLeaves; a++ { + for b := a + 1; b <= numLeaves; b++ { + tree = New(sha256.New()) + err := tree.SetSlice(a, b) + if err != nil { + t.Fatal(err) + } + for k := uint64(0); k < numLeaves; k++ { + tree.Push([]byte{byte(k)}) + } + + // Build the proof for the tree and run it through verify. + _, proofSet, proofBegin, _ := tree.Prove() + if proofBegin != a { + t.Errorf("proofBegin=%d, want %d", proofBegin, a) + } + if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, a, b, numLeaves) { + t.Error("proof didn't verify for indices", a, b) + } + + // Check that verification fails for all other indices. + for a1 := uint64(0); a1 < numLeaves; a1++ { + for b1 := a1 + 1; b1 <= numLeaves; b1++ { + if a == a1 && b == b1 { + continue + } + if VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, a1, b1, numLeaves) { + t.Error("proof verify for indices", a1, b1) + } + } + } + } + } + } +} + +func TestCompatibilitySliceLarge(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Check that proofs on larger trees are consistent. + for i := 0; i < 25; i++ { + // Determine a random size for the tree up to 256k elements. + sizeI, err := rand.Int(rand.Reader, big.NewInt(256e3)) + if err != nil { + t.Fatal(err) + } + size := uint64(sizeI.Int64()) + + proofBeginI, err := rand.Int(rand.Reader, sizeI) + if err != nil { + t.Fatal(err) + } + proofBegin := uint64(proofBeginI.Int64()) + + var remainingI big.Int + remainingI.Sub(sizeI, proofBeginI) + sliceI, err := rand.Int(rand.Reader, &remainingI) + if err != nil { + t.Fatal(err) + } + var proofEndI big.Int + proofEndI.Add(proofBeginI, sliceI) + proofEnd := uint64(proofEndI.Int64()) + + // Prepare the tree. + tree := New(sha256.New()) + err = tree.SetSlice(proofBegin, proofEnd) + if err != nil { + t.Fatal(err) + } + + // Insert 'size' unique elements. + for j := 0; j < int(size); j++ { + elem := []byte(strconv.Itoa(j)) + tree.Push(elem) + } + + // Get the proof for the tree and run it through verify. + merkleRoot, proofSet, _, numLeaves := tree.Prove() + if !VerifyProofOfSlice(sha256.New(), merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) { + t.Error("proof didn't verify in long test", size, proofBegin, proofEnd) + } + } +} + // TestLeafCounts checks that the number of leaves in the tree are being // reported correctly. func TestLeafCounts(t *testing.T) { @@ -481,6 +1420,18 @@ func TestLeafCounts(t *testing.T) { if leaves != 1 { t.Error("bad reporting on leaf count") } + + tree = New(sha256.New()) + err = tree.SetSlice(1, 10) + if err != nil { + t.Fatal(err) + } + tree.Push([]byte{}) + tree.Push([]byte{}) + _, _, _, leaves = tree.Prove() + if leaves != 2 { + t.Error("bad reporting on leaf count") + } } // BenchmarkSha256_4MB uses sha256 to hash 4mb of data. diff --git a/verify.go b/verify.go index e3f5aa6..aa2853d 100644 --- a/verify.go +++ b/verify.go @@ -10,111 +10,109 @@ import ( // root. False is returned if the proof set or Merkle root is nil, and if // 'numLeaves' equals 0. func VerifyProof(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) bool { - // Return false for nonsense input. A switch statement is used so that the - // cover tool will reveal if a case is not covered by the test suite. This - // would not be possible using a single if statement due to the limitations - // of the cover tool. + return VerifyProofOfSlice(h, merkleRoot, proofSet, proofIndex, proofIndex+1, numLeaves) +} + +// VerifyProofOfSlice takes a Merkle root, a proofSet, and the slice and returns +// true if the first proofEnd-proofBegin elements of the proof set are leaves +// of data in the Merkle root. False is returned if the proof set or Merkle +// root is nil, and if 'numLeaves' equals 0. Can be used with proofs returned +// by Tree.Prove and CachedTree.Prove. +func VerifyProofOfSlice(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofBegin, proofEnd, numLeaves uint64) bool { + return verifyProofOfSlice(h, false, merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) +} + +// VerifyProofOfCachedElements takes a Merkle root, a proofSet, and the slice +// and returns true if the first proofEnd-proofBegin elements of the proof set +// are roots of cached elements in the Merkle root. False is returned if +// the proof set or Merkle root is nil, and if 'numLeaves' equals 0. +// Can be used with proofs returned by CachedTree.ProveCached. +// 'numLeaves' is the total number of cached elements. +func VerifyProofOfCachedElements(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofBegin, proofEnd, numLeaves uint64) bool { + return verifyProofOfSlice(h, true, merkleRoot, proofSet, proofBegin, proofEnd, numLeaves) +} + +func verifyProofOfSlice(h hash.Hash, proveCached bool, merkleRoot []byte, proofSet [][]byte, proofBegin, proofEnd, numLeaves uint64) bool { + // Return false for nonsense input. if merkleRoot == nil { return false } - if proofIndex >= numLeaves { + if proofBegin >= proofEnd { return false } - - // In a Merkle tree, every node except the root node has a sibling. - // Combining the two siblings in the correct order will create the parent - // node. Each of the remaining hashes in the proof set is a sibling to a - // node that can be built from all of the previous elements of the proof - // set. The next node is built by taking: - // - // H(0x01 || sibling A || sibling B) - // - // The difficulty of the algorithm lies in determining whether the supplied - // hash is sibling A or sibling B. This information can be determined by - // using the proof index and the total number of leaves in the tree. - // - // A pair of two siblings forms a subtree. The subtree is complete if it - // has 1 << height total leaves. When the subtree is complete, the position - // of the proof index within the subtree can be determined by looking at - // the bounds of the subtree and determining if the proof index is in the - // first or second half of the subtree. - // - // When the subtree is not complete, either 1 or 0 of the remaining hashes - // will be sibling B. All remaining hashes after that will be sibling A. - // This is true because of the way that orphans are merged into the Merkle - // tree - an orphan at height n is elevated to height n + 1, and only - // hashed when it is no longer an orphan. Each subtree will therefore merge - // with at most 1 orphan to the right before becoming an orphan itself. - // Orphan nodes are always merged with larger subtrees to the left. - // - // One vulnerability with the proof verification is that the proofSet may - // not be long enough. Before looking at an element of proofSet, a check - // needs to be made that the element exists. - - // The first element of the set is the original data. A sibling at height 1 - // is created by getting the leafSum of the original data. - height := 0 - if len(proofSet) <= height { + if proofEnd > numLeaves { return false } - sum := leafSum(h, proofSet[height]) - height++ - // While the current subtree (of height 'height') is complete, determine - // the position of the next sibling using the complete subtree algorithm. - // 'stableEnd' tells us the ending index of the last full subtree. It gets - // initialized to 'proofIndex' because the first full subtree was the - // subtree of height 1, created above (and had an ending index of - // 'proofIndex'). - stableEnd := proofIndex - for { - // Determine if the subtree is complete. This is accomplished by - // rounding down the proofIndex to the nearest 1 << 'height', adding 1 - // << 'height', and comparing the result to the number of leaves in the - // Merkle tree. - subTreeStartIndex := (proofIndex / (1 << uint(height))) * (1 << uint(height)) // round down to the nearest 1 << height - subTreeEndIndex := subTreeStartIndex + (1 << (uint(height))) - 1 // subtract 1 because the start index is inclusive - if subTreeEndIndex >= numLeaves { - // If the Merkle tree does not have a leaf at index - // 'subTreeEndIndex', then the subtree of the current height is not - // a complete subtree. - break - } - stableEnd = subTreeEndIndex - - // Determine if the proofIndex is in the first or the second half of - // the subtree. - if len(proofSet) <= height { + // Create the list of hashes on the level of leaves. + var sums [][]byte + for i := proofBegin; i < proofEnd; i++ { + if len(proofSet) == 0 { return false } - if proofIndex-subTreeStartIndex < 1< 1. + + for numLeaves > 1 { + if proofBegin%2 == 1 { + // Example: addition of % on level <- + // ┌───┴──┐ + // %──┴──* │ <- + // ┌─┴─┐ ┌─┴─┐ │ + // * * + if len(proofSet) == 0 { + return false + } + left := proofSet[0] + proofSet = proofSet[1:] + sums = append([][]byte{left}, sums...) + proofBegin-- + } + if len(sums)%2 == 1 && proofEnd < numLeaves { + // Example: addition of % on level <- + // ┌───┴──┐ + // *──┴──% │ <- + // ┌─┴─┐ ┌─┴─┐ │ + // * * + if len(proofSet) == 0 { + return false + } + right := proofSet[0] + proofSet = proofSet[1:] + sums = append(sums, right) + proofEnd++ } - sum = nodeSum(h, sum, proofSet[height]) - height++ + var sums2 [][]byte + for len(sums) >= 2 { + left, right := sums[0], sums[1] + sums = sums[2:] + sums2 = append(sums2, nodeSum(h, left, right)) + } + if len(sums) == 1 { + sums2 = append(sums2, sums[0]) + } + sums = sums2 + proofBegin /= 2 + // proofEnd and numLeaves need +1 because they are not inclusive. + proofEnd = (proofEnd + 1) / 2 + numLeaves = (numLeaves + 1) / 2 } - // All remaining elements in the proof set will belong to a left sibling. - for height < len(proofSet) { - sum = nodeSum(h, proofSet[height], sum) - height++ + if len(proofSet) != 0 { + return false } // Compare our calculated Merkle root to the desired Merkle root. - if bytes.Compare(sum, merkleRoot) == 0 { - return true - } - return false + return bytes.Compare(sums[0], merkleRoot) == 0 }