diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0ed8c0b2c08..3c4f9e94649 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -664,6 +664,11 @@ var ( Usage: "Download snapshots up to the given block number (exclusive). Disabled by default. Useful for testing and shadow forks.", Aliases: []string{"shadow.fork.block"}, } + SnapDownloadToBlockWithRebuildCommitmentFlag = cli.BoolFlag{ + Name: "snap.download.to.block.with.rebuild.commitment", + Usage: "Rebuild commitment domain files based on storage/acc/code. Only used with snap.download.to.block. Needed when all downloaded commitment files are purified.", + Value: false, + } TorrentVerbosityFlag = cli.IntFlag{ Name: "torrent.verbosity", Value: 1, diff --git a/db/snapshotsync/snapshotsync.go b/db/snapshotsync/snapshotsync.go index 1d6c8537143..aac338a820a 100644 --- a/db/snapshotsync/snapshotsync.go +++ b/db/snapshotsync/snapshotsync.go @@ -478,7 +478,7 @@ func SyncSnapshots( continue } - if filterToBlock(p.Name, toBlock, toStep, headerchain) { + if filterToBlock(p.Name, toBlock, toStep, headerchain, syncCfg.SnapshotDownloadToBlockWithRebuildCommitment) { continue } @@ -512,7 +512,7 @@ func SyncSnapshots( return nil } -func filterToBlock(name string, toBlock uint64, toStep uint64, headerchain bool) bool { +func filterToBlock(name string, toBlock uint64, toStep uint64, headerchain bool, rebuildCommitment bool) bool { if toBlock == 0 { return false // toBlock filtering is not enabled } @@ -526,6 +526,9 @@ func filterToBlock(name string, toBlock uint64, toStep uint64, headerchain bool) if strings.HasPrefix(name, "caplin/") { return false // not applicable, caplin files are slot-based } + if rebuildCommitment && strings.Contains(name, kv.CommitmentDomain.String()) && strings.HasPrefix(name, "domain") { + return true + } if stateFile { return fileInfo.To > toStep } diff --git a/db/state/squeeze.go b/db/state/squeeze.go index 59490dbf7b3..008f9bb3c5f 100644 --- a/db/state/squeeze.go +++ b/db/state/squeeze.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/erigontech/erigon/db/version" "math" "os" "path/filepath" @@ -28,6 +27,7 @@ import ( downloadertype "github.com/erigontech/erigon/db/snaptype" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/db/state/statecfg" + "github.com/erigontech/erigon/db/version" "github.com/erigontech/erigon/execution/commitment/commitmentdb" ) @@ -336,8 +336,8 @@ func CheckCommitmentForPrint(ctx context.Context, rwDb kv.TemporalRwDB) (string, // RebuildCommitmentFiles recreates commitment files from existing accounts and storage kv files // If some commitment exists, they will be accepted as correct and next kv range will be processed. // DB expected to be empty, committed into db keys will be not processed. -func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsReader *rawdbv3.TxNumsReader, logger log.Logger, squeeze bool) (latestRoot []byte, err error) { - a := rwDb.(HasAgg).Agg().(*Aggregator) +func RebuildCommitmentFiles(ctx context.Context, db kv.TemporalRoDB, txNumsReader *rawdbv3.TxNumsReader, logger log.Logger, squeeze bool) (latestRoot []byte, err error) { + a := db.(HasAgg).Agg().(*Aggregator) // disable hard alignment; allowing commitment and storage/account to have // different visibleFiles @@ -468,22 +468,22 @@ func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsRea return true, k } - rwTx, err := rwDb.BeginTemporalRw(ctx) + tx, err := db.BeginTemporalRo(ctx) if err != nil { return nil, err } - defer rwTx.Rollback() + defer tx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) + domains, err := execctx.NewSharedDomains(ctx, tx, log.New()) if err != nil { return nil, err } domains.SetBlockNum(blockNum) domains.SetTxNum(lastTxnumInShard - 1) - domains.GetCommitmentCtx().SetLimitedHistoryStateReader(rwTx, lastTxnumInShard) // this helps to read state from correct file during commitment + domains.GetCommitmentCtx().SetLimitedHistoryStateReader(tx, lastTxnumInShard) // this helps to read state from correct file during commitment - rebuiltCommit, err = rebuildCommitmentShard(ctx, domains, rwTx, nextKey, &rebuiltCommitment{ + rebuiltCommit, err = rebuildCommitmentShard(ctx, domains, tx, nextKey, &rebuiltCommitment{ StepFrom: shardFrom, StepTo: shardTo, TxnFrom: rangeFromTxNum, @@ -503,7 +503,7 @@ func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsRea a.dirtyFilesLock.Lock() a.recalcVisibleFiles(a.dirtyFilesEndTxNumMinimax()) a.dirtyFilesLock.Unlock() - rwTx.Rollback() + tx.Rollback() if shardTo+shardStepsSize > lastShard && shardStepsSize > 1 { shardStepsSize /= 2 diff --git a/execution/stagedsync/stage_snapshots.go b/execution/stagedsync/stage_snapshots.go index bf8dfc06812..e525447447b 100644 --- a/execution/stagedsync/stage_snapshots.go +++ b/execution/stagedsync/stage_snapshots.go @@ -275,6 +275,17 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R temporal.ForceReopenAggCtx() // otherwise next stages will not see just-indexed-files } + if cfg.syncConfig.SnapshotDownloadToBlockWithRebuildCommitment && s.BlockNumber == 0 { + txNumsReader := cfg.blockReader.TxnumReader(ctx) + _, err = state.RebuildCommitmentFiles(ctx, cfg.db, &txNumsReader, logger, true /*squeeze*/) + if err != nil { + return fmt.Errorf("failed to rebuild commitment files: %w", err) + } + if temporal, ok := tx.(*temporal.RwTx); ok { + temporal.ForceReopenAggCtx() // otherwise next stages will not see new files + } + } + // It's ok to notify before tx.Commit(), because RPCDaemon does read list of files by gRPC (not by reading from db) if cfg.notifier.Events != nil { cfg.notifier.Events.OnNewSnapshot() diff --git a/node/cli/default_flags.go b/node/cli/default_flags.go index eb95e00b6d9..e7971de0337 100644 --- a/node/cli/default_flags.go +++ b/node/cli/default_flags.go @@ -112,6 +112,7 @@ var DefaultFlags = []cli.Flag{ &utils.SnapStateStopFlag, &utils.SnapSkipStateSnapshotDownloadFlag, &utils.SnapDownloadToBlockFlag, + &utils.SnapDownloadToBlockWithRebuildCommitmentFlag, &utils.DbPageSizeFlag, &utils.DbSizeLimitFlag, &utils.DbWriteMapFlag, diff --git a/node/cli/flags.go b/node/cli/flags.go index 58a7508351a..f49e2fb11e7 100644 --- a/node/cli/flags.go +++ b/node/cli/flags.go @@ -294,6 +294,7 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. if ctx.IsSet(utils.SnapDownloadToBlockFlag.Name) { cfg.Sync.SnapshotDownloadToBlock = ctx.Uint64(utils.SnapDownloadToBlockFlag.Name) + cfg.Sync.SnapshotDownloadToBlockWithRebuildCommitment = ctx.Bool(utils.SnapDownloadToBlockWithRebuildCommitmentFlag.Name) } if stage := ctx.String(SyncLoopBreakAfterFlag.Name); len(stage) > 0 { diff --git a/node/ethconfig/config.go b/node/ethconfig/config.go index 83c10b99f0c..1d8eff72236 100644 --- a/node/ethconfig/config.go +++ b/node/ethconfig/config.go @@ -301,5 +301,7 @@ type Sync struct { MaxReorgDepth uint64 KeepExecutionProofs bool PersistReceiptsCacheV2 bool - SnapshotDownloadToBlock uint64 // exclusive [0,toBlock) + + SnapshotDownloadToBlock uint64 // exclusive [0,toBlock) + SnapshotDownloadToBlockWithRebuildCommitment bool }