Skip to content

feat: use BitArray for bit-packed trie node path encoding #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
weiihann wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: use BitArray for bit-packed trie node path encoding #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,16 +344,16 @@ func TestEndToEndWithGenesisBinaryTrie(t *testing.T) {
t.Errorf("SnapshotRoot mismatch: got %x, want %s", snapshotRoot, stats.StateRoot.Hex())
}

// Verify chain config has EnableVerkleAtGenesis
// Verify chain config has EnableUBTAtGenesis
chainConfig := rawdb.ReadChainConfig(db, block.Hash())
if chainConfig == nil {
t.Error("Chain config not found")
} else if !chainConfig.EnableVerkleAtGenesis {
t.Error("Chain config should have EnableVerkleAtGenesis=true for binary trie mode")
} else if !chainConfig.EnableUBTAtGenesis {
t.Error("Chain config should have EnableUBTAtGenesis=true for binary trie mode")
}

// Verify expected counts
expectedAccounts := 1 + 50 // 1 genesis EOA + 50 generated
expectedAccounts := 1 + 50 // 1 genesis EOA + 50 generated
expectedContracts := 1 + 20 // 1 genesis contract + 20 generated

if stats.AccountsCreated != expectedAccounts {
Expand Down
46 changes: 24 additions & 22 deletions generator/binary_stack_trie.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import (
"encoding/binary"
"fmt"
"log"
"time"
"math/bits"
"runtime"
"sort"
"sync"
"time"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
Expand All @@ -20,8 +20,8 @@ import (
)

const (
stemSize = bintrie.StemSize // 31
hashSize = bintrie.HashSize // 32
stemSize = bintrie.StemSize // 31
hashSize = bintrie.HashSize // 32
stemNodeWidth = bintrie.StemNodeWidth // 256

// Node type markers matching bintrie/binary_node.go
Expand Down Expand Up @@ -67,8 +67,9 @@ type groupChild struct {
}

// trieNodeWriter batches serialized trie node writes to Pebble.
// Each node is written at key "vA" + path, where path is one byte per
// tree level (0x00=left, 0x01=right). Flushes when batch exceeds 256MB.
// Each node is written at key "vA" + path, where path is the bit-packed
// encoding of the trie path (top N bits right-justified in ceil(N/8)
// bytes). Flushes when batch exceeds 256MB.
type trieNodeWriter struct {
batch ethdb.Batch
db ethdb.KeyValueStore
Expand Down Expand Up @@ -455,11 +456,11 @@ func commonPrefixLenBits(a, b []byte) int {
// group's bottom-layer boundary. This eliminates the need for a post-hoc
// regroupTrieNodes pass. Memory overhead: O(groupDepth) per group.
type streamingBuilder struct {
stack [maxDepth]common.Hash // pending child hash at each depth
occupied [maxDepth]bool // whether stack[d] is valid
isRight [maxDepth]bool // true if stack[d] is a right child
stemBits [maxDepth][stemSize]byte // stem that placed each pending hash
w *trieNodeWriter // optional: writes serialized nodes to DB
stack [maxDepth]common.Hash // pending child hash at each depth
occupied [maxDepth]bool // whether stack[d] is valid
isRight [maxDepth]bool // true if stack[d] is a right child
stemBits [maxDepth][stemSize]byte // stem that placed each pending hash
w *trieNodeWriter // optional: writes serialized nodes to DB

// Grouped emission: when groupDepth > 0, internal nodes are written in
// grouped format at boundary depths. groupBuf collects bottom-layer
Expand All @@ -480,14 +481,17 @@ func stemBitAt(stem []byte, depth int) byte {
return (stem[depth/8] >> uint(7-(depth%8))) & 1
}

// makePath builds the bit-path from root to `depth` for the given stem.
// Each byte is 0x00 (left) or 0x01 (right). Used for trie node DB keys.
// makePath builds a bit-packed path from root to `depth` for the given
// stem using BitArray, matching go-ethereum's PathDB node key format.
func makePath(stem []byte, depth int) []byte {
path := make([]byte, depth)
for i := 0; i < depth; i++ {
path[i] = stemBitAt(stem, i)
if depth <= 0 {
return nil
}
return path
var ba bintrie.BitArray
ba.SetBytes(uint8(len(stem)*8), stem)
var path bintrie.BitArray
path.MSBs(&ba, uint8(depth))
return path.ActiveBytes()
}

// recordGroupChild records a hash at a boundary depth as a bottom-layer
Expand Down Expand Up @@ -829,7 +833,6 @@ func computeBinaryRootStreaming(iter ethdb.Iterator, db ethdb.KeyValueStore, gro
return root, tnStats
}


// parallelStorageThreshold is the minimum number of storage slots needed
// to justify worker pool overhead for parallel key derivation.
const parallelStorageThreshold = 64
Expand Down Expand Up @@ -867,7 +870,6 @@ func collectAccountEntriesParallel(
return entries
}


// --- Parallel Phase 2 pipeline ---

// stemWork is a unit of work sent from the reader to worker goroutines.
Expand Down Expand Up @@ -912,10 +914,10 @@ func computeBinaryRootStreamingParallel(

// Channels
const maxInFlight = 64
sem := make(chan struct{}, maxInFlight) // bounds total in-flight stems
workCh := make(chan *stemWork, 2*numWorkers) // reader -> workers
resultCh := make(chan *stemResult, 2*numWorkers) // workers -> resequencer
builderCh := make(chan *stemResult, 128) // resequencer -> builder
sem := make(chan struct{}, maxInFlight) // bounds total in-flight stems
workCh := make(chan *stemWork, 2*numWorkers) // reader -> workers
resultCh := make(chan *stemResult, 2*numWorkers) // workers -> resequencer
builderCh := make(chan *stemResult, 128) // resequencer -> builder

// Error collection
errCh := make(chan error, numWorkers+3) // enough for all goroutines
Expand Down
4 changes: 2 additions & 2 deletions generator/generator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,7 +691,7 @@ func TestBinaryTrieStateRootValue(t *testing.T) {
t.Fatalf("Failed to generate state: %v", err)
}

expected := common.HexToHash("0xee656cf3921d8cbd1aa003a881128846feed2f2c670fa9110cac78f6f8e9d263")
expected := common.HexToHash("0x4aa03e3a6f1cbafe5879f67088aca4ea6cbbcfcfff61251752017d3c859294a8")
if stats.StateRoot != expected {
t.Errorf("Binary trie state root mismatch:\n got: %s\n want: %s\nThis may indicate an upstream bintrie API change.",
stats.StateRoot.Hex(), expected.Hex())
Expand Down Expand Up @@ -810,7 +810,7 @@ func TestBinaryTrieCommitIntervalGoldenHash(t *testing.T) {
}

// Must match the same golden hash as TestBinaryTrieStateRootValue.
expected := common.HexToHash("0xee656cf3921d8cbd1aa003a881128846feed2f2c670fa9110cac78f6f8e9d263")
expected := common.HexToHash("0x4aa03e3a6f1cbafe5879f67088aca4ea6cbbcfcfff61251752017d3c859294a8")
if stats.StateRoot != expected {
t.Errorf("CommitInterval golden hash mismatch:\n got: %s\n want: %s",
stats.StateRoot.Hex(), expected.Hex())
Expand Down
Loading