Skip to content

list of changes needed to replay at block height ~ 17M #474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: jsign-replay-kaustinen6
Choose a base branch
from
Draft

list of changes needed to replay at block height ~ 17M #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/geth/chaincmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ func exportOverlayPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, _ := utils.MakeChain(ctx, stack, true)
chain, _ := utils.MakeChain(ctx, stack, false)

var root common.Hash
if ctx.String(utils.TreeRootFlag.Name) != "" {
Expand Down
4 changes: 4 additions & 0 deletions cmd/utils/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,10 @@ func ImportChain(chain *core.BlockChain, fn string) error {
}
defer fh.Close()

// if _, err := fh.Seek(18224628422, 0); err != nil {
// panic(err)
// }

var reader io.Reader = fh
if strings.HasSuffix(fn, ".gz") {
if reader, err = gzip.NewReader(reader); err != nil {
Expand Down
135 changes: 58 additions & 77 deletions consensus/beacon/consensus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/utils"
"github.com/ethereum/go-verkle"
// "github.com/ethereum/go-ethereum/trie/utils"
// "github.com/ethereum/go-verkle"
// "github.com/pk910/dynamic-ssz"
)

// Proof-of-stake protocol constants.
Expand Down Expand Up @@ -373,9 +374,9 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.

// FinalizeAndAssemble implements consensus.Engine, setting the final state and
// assembling the block.
func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, withdrawals []*types.Withdrawal) (*types.Block, error) {
func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, withdrawals []*types.Withdrawal) (*types.Block, error) {
if !beacon.IsPoSHeader(header) {
return beacon.ethone.FinalizeAndAssemble(chain, header, state, txs, uncles, receipts, nil)
return beacon.ethone.FinalizeAndAssemble(chain, header, statedb, txs, uncles, receipts, nil)
}
shanghai := chain.Config().IsShanghai(header.Number, header.Time)
if shanghai {
Expand All @@ -389,86 +390,66 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
}
}
// Finalize and assemble the block.
beacon.Finalize(chain, header, state, txs, uncles, withdrawals)
beacon.Finalize(chain, header, statedb, txs, uncles, withdrawals)

// Assign the final state root to header.
header.Root = state.IntermediateRoot(true)
header.Root = statedb.IntermediateRoot(true)
// Associate current conversion state to computed state
// root and store it in the database for later recovery.
state.Database().SaveTransitionState(header.Root)

var (
p *verkle.VerkleProof
k verkle.StateDiff
keys = state.Witness().Keys()
)
if chain.Config().IsPrague(header.Number, header.Time) {
// Open the pre-tree to prove the pre-state against
parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1)
if parent == nil {
return nil, fmt.Errorf("nil parent header for block %d", header.Number)
}

// Load transition state at beginning of block, because
// OpenTrie needs to know what the conversion status is.
state.Database().LoadTransitionState(parent.Root)

if chain.Config().ProofInBlocks {
preTrie, err := state.Database().OpenTrie(parent.Root)
if err != nil {
return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
}

var okpre, okpost bool
var vtrpre, vtrpost *trie.VerkleTrie
switch pre := preTrie.(type) {
case *trie.VerkleTrie:
vtrpre, okpre = preTrie.(*trie.VerkleTrie)
switch tr := state.GetTrie().(type) {
case *trie.VerkleTrie:
vtrpost = tr
okpost = true
// This is to handle a situation right at the start of the conversion:
// the post trie is a transition tree when the pre tree is an empty
// verkle tree.
case *trie.TransitionTrie:
vtrpost = tr.Overlay()
okpost = true
default:
okpost = false
}
case *trie.TransitionTrie:
vtrpre = pre.Overlay()
okpre = true
post, _ := state.GetTrie().(*trie.TransitionTrie)
vtrpost = post.Overlay()
okpost = true
default:
// This should only happen for the first block of the
// conversion, when the previous tree is a merkle tree.
// Logically, the "previous" verkle tree is an empty tree.
okpre = true
vtrpre = trie.NewVerkleTrie(verkle.New(), state.Database().TrieDB(), utils.NewPointCache(), false)
post := state.GetTrie().(*trie.TransitionTrie)
vtrpost = post.Overlay()
okpost = true
}
if okpre && okpost {
if len(keys) > 0 {
p, k, err = trie.ProveAndSerialize(vtrpre, vtrpost, keys, vtrpre.FlatdbNodeResolver)
if err != nil {
return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
}
}
}
}
}
statedb.Database().SaveTransitionState(header.Root)

// var (
// p *verkle.VerkleProof
// k verkle.StateDiff
// keys = statedb.Witness().Keys()
// )
// if chain.Config().IsPrague(header.Number, header.Time) {
// // Open the pre-tree to prove the pre-state against
// parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1)
// if parent == nil {
// return nil, fmt.Errorf("nil parent header for block %d", header.Number)
// }

// // Load transition state at beginning of block, because
// // OpenTrie needs to know what the conversion status is.
// statedb.Database().LoadTransitionState(parent.Root)

// preTrie, err := statedb.Database().OpenTrie(parent.Root)
// if err != nil {
// return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
// }

// var vtrpre *trie.VerkleTrie
// switch pre := preTrie.(type) {
// case *trie.VerkleTrie:
// vtrpre = preTrie.(*trie.VerkleTrie)
// case *trie.TransitionTrie:
// vtrpre = pre.Overlay()
// default:
// panic("should not happen")
// }
// if len(keys) > 0 {
// p, k, err = trie.ProveAndSerialize(vtrpre, nil, keys, vtrpre.FlatdbNodeResolver)
// if err != nil {
// return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
// }
// }

// ew := types.ExecutionWitness{StateDiff: k, VerkleProof: p}
// encoder := dynssz.NewDynSsz(map[string]any{})
// encoded, err := encoder.MarshalSSZ(&ew)
// if err != nil {
// spew.Dump(ew)
// panic(err)
// }
// state.AppendBytesToFile("witness_size.csv", []byte(fmt.Sprintf("%d,%d", header.Number, len(encoded))))
// }

// Assemble and return the final block.
block := types.NewBlockWithWithdrawals(header, txs, uncles, receipts, withdrawals, trie.NewStackTrie(nil))
if chain.Config().IsPrague(header.Number, header.Time) && chain.Config().ProofInBlocks {
block.SetVerkleProof(p, k)
}
// if chain.Config().IsPrague(header.Number, header.Time) && chain.Config().ProofInBlocks {
// block.SetVerkleProof(p, k)
// }
return block, nil
}

Expand Down
1 change: 1 addition & 0 deletions core/block_validator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
// if receiptSha != header.ReceiptHash {
// return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
// }

// // Validate the state root against the received state root and throw
// // an error if they don't match.
// if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
Expand Down
4 changes: 2 additions & 2 deletions core/blockchain.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
}
if overrides != nil {
if overrides.OverrideProofInBlock != nil {
chainConfig.ProofInBlocks = *overrides.OverrideProofInBlock
chainConfig.ProofInBlocks = true // *overrides.OverrideProofInBlock
}
if overrides.OverrideOverlayStride != nil {
chainConfig.OverlayStride = *overrides.OverrideOverlayStride
Expand Down Expand Up @@ -322,7 +322,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// for it to be able to recover if interrupted during the transition
// but that's left out to a later PR since there's not really a need
// right now.
bc.stateCache.InitTransitionStatus(true, true)
bc.stateCache.InitTransitionStatus(false, false)
bc.stateCache.EndVerkleTransition()
}

Expand Down
60 changes: 45 additions & 15 deletions core/overlay/conversion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,29 +95,25 @@ func (kvm *keyValueMigrator) addStorageSlot(addr []byte, slotNumber []byte, slot
func (kvm *keyValueMigrator) addAccount(addr []byte, acc *types.StateAccount) {
leafNodeData := kvm.getOrInitLeafNodeData(newBranchKey(addr, &zeroTreeIndex))

var version [verkle.LeafValueSize]byte
leafNodeData.Values[utils.VersionLeafKey] = version[:]
var basicData [verkle.LeafValueSize]byte
basicData[utils.BasicDataVersionOffset] = 0
binary.BigEndian.PutUint64(basicData[utils.BasicDataNonceOffset:], acc.Nonce)

var balance [verkle.LeafValueSize]byte
for i, b := range acc.Balance.Bytes() {
balance[len(acc.Balance.Bytes())-1-i] = b
}
leafNodeData.Values[utils.BalanceLeafKey] = balance[:]

var nonce [verkle.LeafValueSize]byte
binary.LittleEndian.PutUint64(nonce[:8], acc.Nonce)
leafNodeData.Values[utils.NonceLeafKey] = nonce[:]
// get the lower 16 bytes of water and change its endianness
balanceBytes := acc.Balance.Bytes()
copy(basicData[32-len(balanceBytes):], balanceBytes[:])

leafNodeData.Values[utils.BasicDataLeafKey] = basicData[:]
leafNodeData.Values[utils.CodeHashLeafKey] = acc.CodeHash[:]
}

func (kvm *keyValueMigrator) addAccountCode(addr []byte, codeSize uint64, chunks []byte) {
leafNodeData := kvm.getOrInitLeafNodeData(newBranchKey(addr, &zeroTreeIndex))

// Save the code size.
var codeSizeBytes [verkle.LeafValueSize]byte
binary.LittleEndian.PutUint64(codeSizeBytes[:8], codeSize)
leafNodeData.Values[utils.CodeSizeLeafKey] = codeSizeBytes[:]
var cs [4]byte
binary.BigEndian.PutUint32(cs[:], uint32(codeSize))
copy(leafNodeData.Values[utils.BasicDataLeafKey][utils.BasicDataCodeSizeOffset:utils.BasicDataNonceOffset], cs[:])

// The first 128 chunks are stored in the account header leaf.
for i := 0; i < 128 && i < len(chunks)/32; i++ {
Expand Down Expand Up @@ -218,12 +214,27 @@ func (kvm *keyValueMigrator) migrateCollectedKeyValues(tree *trie.VerkleTrie) er
return nil
}

var dump bool = false

// OverlayVerkleTransition contains the overlay conversion logic
func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedCount uint64) error {
migrdb := statedb.Database()
migrdb.LockCurrentTransitionState()
defer migrdb.UnLockCurrentTransitionState()

// Open or create file for writing; append data if file exists
var (
file *os.File
err error
)
if dump {
file, err = os.OpenFile("prout.bin", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer file.Close()
}

// verkle transition: if the conversion process is in progress, move
// N values from the MPT into the verkle tree.
if migrdb.InTransition() {
Expand Down Expand Up @@ -271,6 +282,13 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
if len(addr) != 20 {
return fmt.Errorf("addr len is zero is not 32: %d", len(addr))
}
if dump {
n, err := file.Write(addr.Bytes())
if err != nil {
fmt.Println(n, addr, err, file)
panic(err)
}
}
}
migrdb.SetCurrentAccountAddress(addr)
if migrdb.GetCurrentAccountHash() != accIt.Hash() {
Expand Down Expand Up @@ -344,7 +362,13 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
} else {
slotnr = rawdb.ReadPreimage(migrdb.DiskDB(), stIt.Hash())
if len(slotnr) != 32 {
return fmt.Errorf("slotnr len is zero is not 32: %d", len(slotnr))
panic(fmt.Errorf("slotnr len is zero is not 32: %d", len(slotnr)))
}
if dump {
_, err = file.Write(slotnr)
if err != nil {
panic(err)
}
}
}
log.Trace("found slot number", "number", slotnr)
Expand Down Expand Up @@ -400,6 +424,12 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
if len(addr) != 20 {
return fmt.Errorf("account address len is zero is not 20: %d", len(addr))
}
if dump {
_, err = file.Write(addr.Bytes())
if err != nil {
panic(err)
}
}
}
if crypto.Keccak256Hash(addr[:]) != accIt.Hash() {
return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
Expand Down
Loading