-
Notifications
You must be signed in to change notification settings - Fork 60
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
accumulator: Add CacheSim type that can simulate pollard modification #185
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
package accumulator | ||
|
||
import ( | ||
"testing" | ||
) | ||
|
||
func TestCacheSim(t *testing.T) { | ||
// New simulation chain with a lookahead cache of 32 blocks | ||
chain := NewSimChain(64 - 1) | ||
chain.lookahead = 32 | ||
|
||
// New cache simulator for the pollard | ||
cacheSimulator := NewCacheSimulator(0) | ||
|
||
// Empty forest and pollard | ||
forest := NewForest(nil, false) | ||
var pollard Pollard | ||
|
||
for i := 0; i < 64; i++ { | ||
adds, _, delHashes := chain.NextBlock(100) | ||
|
||
proof, err := forest.ProveBatch(delHashes) | ||
if err != nil { | ||
t.Fatal("ProveBatch failed", err) | ||
} | ||
proof.SortTargets() | ||
_, err = forest.Modify(adds, proof.Targets) | ||
if err != nil { | ||
t.Fatal("Modify failed", err) | ||
} | ||
remember := make([]bool, len(adds)) | ||
for i, add := range adds { | ||
remember[i] = add.Remember | ||
} | ||
|
||
proofPositions, _ := ProofPositions(proof.Targets, pollard.numLeaves, pollard.rows()) | ||
// Run the simulator to retrieve the positions of the partial proof. | ||
neededPositions := cacheSimulator.Simulate(proof.Targets, remember) | ||
|
||
// check that the size of the partial proof is actually smaller than a regular proof. | ||
// if the partial proof is bigger it's not partial. | ||
if len(neededPositions) > len(proof.Proof) { | ||
t.Fatal("more positions needed than regular proof") | ||
} | ||
|
||
// check that the partial proof is minimal by ensuring that all the `neededPositions` are not cached. | ||
for _, pos := range neededPositions { | ||
n, _, _, _ := pollard.readPos(pos) | ||
if n != nil { | ||
t.Fatal("partial proof is not minimal. position", pos, "is cached but included in the partial proof") | ||
} | ||
} | ||
|
||
// check that all the positions that the simulator claims to be cached are actually cached. | ||
cached := sortedUint64SliceDiff( | ||
mergeSortedSlices(proofPositions, proof.Targets), neededPositions) | ||
for _, pos := range cached { | ||
n, _, _, err := pollard.readPos(pos) | ||
if err != nil || n == nil || n.data == empty { | ||
t.Fatal("simulated cache claimed to have", pos, "but did not", err) | ||
} | ||
} | ||
err = pollard.IngestBatchProof(proof) | ||
if err != nil { | ||
t.Fatal("IngestBatchProof failed", err) | ||
} | ||
err = pollard.Modify(adds, proof.Targets) | ||
if err != nil { | ||
t.Fatal("Modify failed", err) | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ package accumulator | |
|
||
import ( | ||
"fmt" | ||
"math/big" | ||
) | ||
|
||
// IngestBatchProof populates the Pollard with all needed data to delete the | ||
|
@@ -208,3 +209,65 @@ func (p *Pollard) verifyBatchProof( | |
|
||
return true, proofmap | ||
} | ||
|
||
// CacheSim is a type capable of simulating pollard caching/remembering. | ||
type CacheSim struct { | ||
// A bit set to mark cached positions. | ||
// This is not optimal since it has a spatial complexity of O(n) where n is the number of UTXOs | ||
// BUT only 1 bit per UTXO. | ||
// currently(#643584) there are 66,391,686 UTXOs that means size(cached) = ~8MB. | ||
cached *big.Int | ||
// The number of leaves in the simulator. | ||
numLeaves uint64 | ||
} | ||
|
||
// NewCacheSimulator initialises a new cache simulator. | ||
func NewCacheSimulator(numLeaves uint64) *CacheSim { | ||
return &CacheSim{cached: big.NewInt(0), numLeaves: numLeaves} | ||
} | ||
|
||
// Simulate simulates one modification to the pollard. | ||
// Takes a slice of target positions and a slice of bools representing new leaves (true means "cache the leaf"). | ||
// Returns the proof positions that are needed to prove the targets (including the unkown targets). | ||
func (c *CacheSim) Simulate(targets []uint64, adds []bool) []uint64 { | ||
// Figure out which targets are known/cached and which aren't. | ||
var knownTargets, unknownTargets []uint64 | ||
for _, target := range targets { | ||
if c.cached.Bit(int(target)) > 0 { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit since it'll be eons until we get to 1 << 63 leaves but this could error out right? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @kcalvinalvin are you saying it will be the int casting that overflows? in that case would it be a problem on 32-bit platforms? are those supposed to be supported? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. on a 64-bit system it would cast to a signed int64 and There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i don't know what happens on a 32-bit system with all the uint64s we are using 😄 |
||
knownTargets = append(knownTargets, target) | ||
// Set cached bit to zero because that positions gets deleted. | ||
c.cached.SetBit(c.cached, int(target), 0) | ||
continue | ||
} | ||
|
||
unknownTargets = append(unknownTargets, target) | ||
} | ||
knownProof, computable := ProofPositions(knownTargets, c.numLeaves, treeRows(c.numLeaves)) | ||
unknownProof, _ := ProofPositions(unknownTargets, c.numLeaves, treeRows(c.numLeaves)) | ||
|
||
// Apply remove transformation swaps to cached positions. | ||
leafSwaps := floorTransform(targets, c.numLeaves, treeRows(c.numLeaves)) | ||
for _, swap := range leafSwaps { | ||
fromBit := c.cached.Bit(int(swap.from)) | ||
toBit := c.cached.Bit(int(swap.to)) | ||
c.cached.SetBit(c.cached, int(swap.from), toBit) | ||
c.cached.SetBit(c.cached, int(swap.to), fromBit) | ||
} | ||
c.numLeaves -= uint64(len(targets)) | ||
|
||
// Add the new leaves to the cache. | ||
for i, add := range adds { | ||
if add { | ||
c.cached.SetBit(c.cached, int(c.numLeaves+uint64(i)), 1) | ||
} else { | ||
c.cached.SetBit(c.cached, int(c.numLeaves+uint64(i)), 0) | ||
} | ||
} | ||
c.numLeaves += uint64(len(adds)) | ||
|
||
neededProof := sortedUint64SliceDiff( | ||
mergeSortedSlices(unknownProof, unknownTargets), | ||
mergeSortedSlices(knownProof, computable), | ||
) | ||
return neededProof | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,6 +9,85 @@ import ( | |
// verbose is a global const to get lots of printfs for debugging | ||
var verbose = false | ||
|
||
// ProofPositions returns the positions that are needed to prove that the targets exist. | ||
func ProofPositions(targets []uint64, numLeaves uint64, forestRows uint8) ([]uint64, []uint64) { | ||
// the proofPositions needed without caching. | ||
var proofPositions, computedPositions []uint64 | ||
for row := uint8(0); row < forestRows; row++ { | ||
computedPositions = append(computedPositions, targets...) | ||
if numLeaves&(1<<row) > 0 && len(targets) > 0 && | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we could probably do this in a lot of places or maybe we create a function that checks for roots that we can call in various places, pretty sure the go compiler would inline simple functions like that. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. wasn't there some example very recently where Go didn't inline a really simple function? probably something i read from @kcalvinalvin There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. it should inline all functions that meet these criteria. |
||
targets[len(targets)-1] == rootPosition(numLeaves, row, forestRows) { | ||
// remove roots from targets | ||
targets = targets[:len(targets)-1] | ||
} | ||
|
||
var nextTargets []uint64 | ||
for len(targets) > 0 { | ||
switch { | ||
// look at the first 4 targets | ||
case len(targets) > 3: | ||
if (targets[0]|1)^2 == targets[3]|1 { | ||
// the first and fourth target are cousins | ||
// => target 2 and 3 are also targets, both parents are targets of next row | ||
nextTargets = append(nextTargets, | ||
parent(targets[0], forestRows), parent(targets[3], forestRows)) | ||
targets = targets[4:] | ||
break | ||
} | ||
// handle first three targets | ||
fallthrough | ||
|
||
// look at the first 3 targets | ||
case len(targets) > 2: | ||
if (targets[0]|1)^2 == targets[2]|1 { | ||
// the first and third target are cousins | ||
// => the second target is either the sibling of the first | ||
// OR the sibiling of the third | ||
// => only the sibling that is not a target is appended to the proof positions | ||
if targets[1]|1 == targets[0]|1 { | ||
proofPositions = append(proofPositions, targets[2]^1) | ||
} else { | ||
proofPositions = append(proofPositions, targets[0]^1) | ||
} | ||
// both parents are targets of next row | ||
nextTargets = append(nextTargets, | ||
parent(targets[0], forestRows), parent(targets[2], forestRows)) | ||
targets = targets[3:] | ||
break | ||
} | ||
// handle first two targets | ||
fallthrough | ||
|
||
// look at the first 2 targets | ||
case len(targets) > 1: | ||
if targets[0]|1 == targets[1] { | ||
nextTargets = append(nextTargets, parent(targets[0], forestRows)) | ||
targets = targets[2:] | ||
break | ||
} | ||
if (targets[0]|1)^2 == targets[1]|1 { | ||
proofPositions = append(proofPositions, targets[0]^1, targets[1]^1) | ||
nextTargets = append(nextTargets, | ||
parent(targets[0], forestRows), parent(targets[1], forestRows)) | ||
targets = targets[2:] | ||
break | ||
} | ||
// not related, handle first target | ||
fallthrough | ||
|
||
// look at the first target | ||
default: | ||
proofPositions = append(proofPositions, targets[0]^1) | ||
nextTargets = append(nextTargets, parent(targets[0], forestRows)) | ||
targets = targets[1:] | ||
} | ||
} | ||
targets = nextTargets | ||
} | ||
|
||
return proofPositions, computedPositions | ||
} | ||
|
||
// takes a slice of dels, removes the twins (in place) and returns a slice | ||
// of parents of twins | ||
// | ||
|
@@ -413,6 +492,28 @@ func mergeSortedSlices(a []uint64, b []uint64) (c []uint64) { | |
return | ||
} | ||
|
||
// returns a \ b | ||
// (eg [1, 5, 8, 9], [2, 3, 4, 5, 8] -> [1, 9] | ||
func sortedUint64SliceDiff(a []uint64, b []uint64) (diff []uint64) { | ||
for i, elemA := range a { | ||
for len(b) > 0 && b[0] < elemA { | ||
b = b[1:] | ||
} | ||
|
||
if len(b) == 0 { | ||
diff = append(diff, a[i:]...) | ||
break | ||
} | ||
|
||
if len(b) > 0 && elemA < b[0] { | ||
diff = append(diff, elemA) | ||
continue | ||
} | ||
} | ||
|
||
return | ||
} | ||
|
||
// dedupeSwapDirt is kind of like mergeSortedSlices. Takes 2 sorted slices | ||
// a, b and removes all elements of b from a and returns a. | ||
// in this case b is arrow.to | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Another nit but there's a spelling error on unkown