Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

accumulator: Add CacheSim type that can simulate pollard modification #185

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions accumulator/cachesim_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package accumulator

import (
"testing"
)

func TestCacheSim(t *testing.T) {
// New simulation chain with a lookahead cache of 32 blocks
chain := NewSimChain(64 - 1)
chain.lookahead = 32

// New cache simulator for the pollard
cacheSimulator := NewCacheSimulator(0)

// Empty forest and pollard
forest := NewForest(nil, false)
var pollard Pollard

for i := 0; i < 64; i++ {
adds, _, delHashes := chain.NextBlock(100)

proof, err := forest.ProveBatch(delHashes)
if err != nil {
t.Fatal("ProveBatch failed", err)
}
proof.SortTargets()
_, err = forest.Modify(adds, proof.Targets)
if err != nil {
t.Fatal("Modify failed", err)
}
remember := make([]bool, len(adds))
for i, add := range adds {
remember[i] = add.Remember
}

proofPositions, _ := ProofPositions(proof.Targets, pollard.numLeaves, pollard.rows())
// Run the simulator to retrieve the positions of the partial proof.
neededPositions := cacheSimulator.Simulate(proof.Targets, remember)

// check that the size of the partial proof is actually smaller than a regular proof.
// if the partial proof is bigger it's not partial.
if len(neededPositions) > len(proof.Proof) {
t.Fatal("more positions needed than regular proof")
}

// check that the partial proof is minimal by ensuring that all the `neededPositions` are not cached.
for _, pos := range neededPositions {
n, _, _, _ := pollard.readPos(pos)
if n != nil {
t.Fatal("partial proof is not minimal. position", pos, "is cached but included in the partial proof")
}
}

// check that all the positions that the simulator claims to be cached are actually cached.
cached := sortedUint64SliceDiff(
mergeSortedSlices(proofPositions, proof.Targets), neededPositions)
for _, pos := range cached {
n, _, _, err := pollard.readPos(pos)
if err != nil || n == nil || n.data == empty {
t.Fatal("simulated cache claimed to have", pos, "but did not", err)
}
}
err = pollard.IngestBatchProof(proof)
if err != nil {
t.Fatal("IngestBatchProof failed", err)
}
err = pollard.Modify(adds, proof.Targets)
if err != nil {
t.Fatal("Modify failed", err)
}
}
}
3 changes: 3 additions & 0 deletions accumulator/pollard.go
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,9 @@ func (p *Pollard) readPos(pos uint64) (
lrSib := lr ^ 1
if h == 0 { // if at bottom, done
n, nsib = n.niece[lrSib], n.niece[lr]
if n == nil && nsib != nil {
n = nsib.niece[0]
}
return
}

Expand Down
63 changes: 63 additions & 0 deletions accumulator/pollardproof.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package accumulator

import (
"fmt"
"math/big"
)

// IngestBatchProof populates the Pollard with all needed data to delete the
Expand Down Expand Up @@ -208,3 +209,65 @@ func (p *Pollard) verifyBatchProof(

return true, proofmap
}

// CacheSim is a type capable of simulating pollard caching/remembering.
type CacheSim struct {
// A bit set to mark cached positions.
// This is not optimal since it has a spatial complexity of O(n) where n is the number of UTXOs
// BUT only 1 bit per UTXO.
// currently(#643584) there are 66,391,686 UTXOs that means size(cached) = ~8MB.
cached *big.Int
// The number of leaves in the simulator.
numLeaves uint64
}

// NewCacheSimulator initialises a new cache simulator.
func NewCacheSimulator(numLeaves uint64) *CacheSim {
return &CacheSim{cached: big.NewInt(0), numLeaves: numLeaves}
}

// Simulate simulates one modification to the pollard.
// Takes a slice of target positions and a slice of bools representing new leaves (true means "cache the leaf").
// Returns the proof positions that are needed to prove the targets (including the unkown targets).
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another nit but there's a spelling error on unkown

func (c *CacheSim) Simulate(targets []uint64, adds []bool) []uint64 {
// Figure out which targets are known/cached and which aren't.
var knownTargets, unknownTargets []uint64
for _, target := range targets {
if c.cached.Bit(int(target)) > 0 {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit since it'll be eons until we get to 1 << 63 leaves but this could error out right?

https://play.golang.org/p/wo0-tE8s1KN

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kcalvinalvin are you saying it will be the int casting that overflows? in that case would it be a problem on 32-bit platforms? are those supposed to be supported?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

on a 64-bit system it would cast to a signed int64 and target is unsigned, so the casted int could be negative

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't know what happens on a 32-bit system with all the uint64s we are using 😄

knownTargets = append(knownTargets, target)
// Set cached bit to zero because that positions gets deleted.
c.cached.SetBit(c.cached, int(target), 0)
continue
}

unknownTargets = append(unknownTargets, target)
}
knownProof, computable := ProofPositions(knownTargets, c.numLeaves, treeRows(c.numLeaves))
unknownProof, _ := ProofPositions(unknownTargets, c.numLeaves, treeRows(c.numLeaves))

// Apply remove transformation swaps to cached positions.
leafSwaps := floorTransform(targets, c.numLeaves, treeRows(c.numLeaves))
for _, swap := range leafSwaps {
fromBit := c.cached.Bit(int(swap.from))
toBit := c.cached.Bit(int(swap.to))
c.cached.SetBit(c.cached, int(swap.from), toBit)
c.cached.SetBit(c.cached, int(swap.to), fromBit)
}
c.numLeaves -= uint64(len(targets))

// Add the new leaves to the cache.
for i, add := range adds {
if add {
c.cached.SetBit(c.cached, int(c.numLeaves+uint64(i)), 1)
} else {
c.cached.SetBit(c.cached, int(c.numLeaves+uint64(i)), 0)
}
}
c.numLeaves += uint64(len(adds))

neededProof := sortedUint64SliceDiff(
mergeSortedSlices(unknownProof, unknownTargets),
mergeSortedSlices(knownProof, computable),
)
return neededProof
}
101 changes: 101 additions & 0 deletions accumulator/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,85 @@ import (
// verbose is a global const to get lots of printfs for debugging
var verbose = false

// ProofPositions returns the positions that are needed to prove that the targets exist.
func ProofPositions(targets []uint64, numLeaves uint64, forestRows uint8) ([]uint64, []uint64) {
// the proofPositions needed without caching.
var proofPositions, computedPositions []uint64
for row := uint8(0); row < forestRows; row++ {
computedPositions = append(computedPositions, targets...)
if numLeaves&(1<<row) > 0 && len(targets) > 0 &&
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

numLeaves&(1<<row) > 0 is checking to see if a root exists right? Comment or maybe even splitting this up to make something like rootExists := numLeaves&(1<<row) > 0 could be better for readability.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could probably do this in a lot of places or maybe we create a function that checks for roots that we can call in various places, pretty sure the go compiler would inline simple functions like that.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wasn't there some example very recently where Go didn't inline a really simple function? probably something i read from @kcalvinalvin

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it should inline all functions that meet these criteria.
we can also verify that it inlines them using -gcflags=-m when building.

targets[len(targets)-1] == rootPosition(numLeaves, row, forestRows) {
// remove roots from targets
targets = targets[:len(targets)-1]
}

var nextTargets []uint64
for len(targets) > 0 {
switch {
// look at the first 4 targets
case len(targets) > 3:
if (targets[0]|1)^2 == targets[3]|1 {
// the first and fourth target are cousins
// => target 2 and 3 are also targets, both parents are targets of next row
nextTargets = append(nextTargets,
parent(targets[0], forestRows), parent(targets[3], forestRows))
targets = targets[4:]
break
}
// handle first three targets
fallthrough

// look at the first 3 targets
case len(targets) > 2:
if (targets[0]|1)^2 == targets[2]|1 {
// the first and third target are cousins
// => the second target is either the sibling of the first
// OR the sibiling of the third
// => only the sibling that is not a target is appended to the proof positions
if targets[1]|1 == targets[0]|1 {
proofPositions = append(proofPositions, targets[2]^1)
} else {
proofPositions = append(proofPositions, targets[0]^1)
}
// both parents are targets of next row
nextTargets = append(nextTargets,
parent(targets[0], forestRows), parent(targets[2], forestRows))
targets = targets[3:]
break
}
// handle first two targets
fallthrough

// look at the first 2 targets
case len(targets) > 1:
if targets[0]|1 == targets[1] {
nextTargets = append(nextTargets, parent(targets[0], forestRows))
targets = targets[2:]
break
}
if (targets[0]|1)^2 == targets[1]|1 {
proofPositions = append(proofPositions, targets[0]^1, targets[1]^1)
nextTargets = append(nextTargets,
parent(targets[0], forestRows), parent(targets[1], forestRows))
targets = targets[2:]
break
}
// not related, handle first target
fallthrough

// look at the first target
default:
proofPositions = append(proofPositions, targets[0]^1)
nextTargets = append(nextTargets, parent(targets[0], forestRows))
targets = targets[1:]
}
}
targets = nextTargets
}

return proofPositions, computedPositions
}

// takes a slice of dels, removes the twins (in place) and returns a slice
// of parents of twins
//
Expand Down Expand Up @@ -413,6 +492,28 @@ func mergeSortedSlices(a []uint64, b []uint64) (c []uint64) {
return
}

// returns a \ b
// (eg [1, 5, 8, 9], [2, 3, 4, 5, 8] -> [1, 9]
func sortedUint64SliceDiff(a []uint64, b []uint64) (diff []uint64) {
for i, elemA := range a {
for len(b) > 0 && b[0] < elemA {
b = b[1:]
}

if len(b) == 0 {
diff = append(diff, a[i:]...)
break
}

if len(b) > 0 && elemA < b[0] {
diff = append(diff, elemA)
continue
}
}

return
}

// dedupeSwapDirt is kind of like mergeSortedSlices. Takes 2 sorted slices
// a, b and removes all elements of b from a and returns a.
// in this case b is arrow.to
Expand Down