From b4fc88d73eb73e139af2783567ff2380de6cbd7e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 16 Oct 2024 17:41:05 +0700 Subject: [PATCH 01/27] save --- erigon-lib/kv/mdbx/kv_mdbx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index 4bdfcd87c4d..a69fc2e5ec9 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -419,7 +419,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) { MaxBatchDelay: DefaultMaxBatchDelay, } - customBuckets := opts.bucketsCfg(kv.ChaindataTablesCfg) + customBuckets := opts.bucketsCfg(kv.TablesCfgByLabel(opts.label)) for name, cfg := range customBuckets { // copy map to avoid changing global variable db.buckets[name] = cfg } From 3e03d7ba10d2d5c068c6b6c70d28f3901375ca06 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 16 Oct 2024 17:42:25 +0700 Subject: [PATCH 02/27] save --- erigon-lib/kv/tables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 4391e16bb00..5cb5e6e2ebf 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -772,7 +772,7 @@ var ReconTablesCfg = TableCfg{ func TablesCfgByLabel(label Label) TableCfg { switch label { - case ChainDB: + case ChainDB, InMem: return ChaindataTablesCfg case TxPoolDB: return TxpoolTablesCfg From aba9881c84712849c03781edd7fe178662f1af50 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 16 Oct 2024 17:44:29 +0700 Subject: [PATCH 03/27] save --- erigon-lib/kv/tables.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 5cb5e6e2ebf..cf51fe09e85 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -644,6 +644,7 @@ var TxPoolTables = []string{ PoolInfo, } var SentryTables = []string{} +var HeimdallTables = []string{} var DownloaderTables = []string{ BittorrentCompletion, BittorrentInfo, @@ -764,6 +765,7 @@ var TxpoolTablesCfg = TableCfg{} var SentryTablesCfg = TableCfg{} var DownloaderTablesCfg = TableCfg{} var DiagnosticsTablesCfg = TableCfg{} +var HeimdallTablesCfg = TableCfg{} var ReconTablesCfg = TableCfg{ PlainStateD: {Flags: DupSort}, CodeD: {Flags: DupSort}, @@ -782,6 +784,8 @@ func TablesCfgByLabel(label Label) TableCfg { return DownloaderTablesCfg case DiagnosticsDB: return DiagnosticsTablesCfg + case HeimdallDB: + return HeimdallTablesCfg default: panic(fmt.Sprintf("unexpected label: %s", label)) } @@ -850,6 +854,13 @@ func reinit() { DiagnosticsTablesCfg[name] = TableCfgItem{} } } + + for _, name := range HeimdallTables { + _, ok := HeimdallTablesCfg[name] + if !ok { + HeimdallTablesCfg[name] = TableCfgItem{} + } + } } // Temporal From d313b686badee288cc47a1875e12327c17981488 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 16 Oct 2024 17:47:39 +0700 Subject: [PATCH 04/27] save --- erigon-lib/kv/tables.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index cf51fe09e85..eb36786910c 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -645,6 +645,7 @@ var TxPoolTables = []string{ } var SentryTables = []string{} var HeimdallTables = []string{} +var PolygonBridgeTables = []string{} var DownloaderTables = []string{ BittorrentCompletion, BittorrentInfo, @@ -766,6 +767,7 @@ var SentryTablesCfg = TableCfg{} var DownloaderTablesCfg = TableCfg{} var DiagnosticsTablesCfg = TableCfg{} var HeimdallTablesCfg = TableCfg{} +var PolygonBridgeTablesCfg = TableCfg{} var ReconTablesCfg = TableCfg{ PlainStateD: {Flags: DupSort}, CodeD: {Flags: DupSort}, @@ -786,6 +788,8 @@ func TablesCfgByLabel(label Label) TableCfg { return DiagnosticsTablesCfg case HeimdallDB: return HeimdallTablesCfg + case PolygonBridgeDB: + return PolygonBridgeTablesCfg default: panic(fmt.Sprintf("unexpected label: %s", label)) } @@ -861,6 +865,12 @@ func reinit() { HeimdallTablesCfg[name] = TableCfgItem{} } } + for _, name := range PolygonBridgeTables { + _, ok := PolygonBridgeTablesCfg[name] + if !ok { + PolygonBridgeTablesCfg[name] = TableCfgItem{} + } + } } // Temporal From 0c839a8c042eca4a5b8bbbc7d0d565ff5edb3d6f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 17 Oct 2024 21:42:33 +0700 Subject: [PATCH 05/27] save --- erigon-lib/kv/tables.go | 5 ++++- node/node.go | 39 ++++++++++++++++++++------------------- node/node_test.go | 4 ++-- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index eb36786910c..18a620ce35b 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -643,7 +643,10 @@ var TxPoolTables = []string{ PoolTransaction, PoolInfo, } -var SentryTables = []string{} +var SentryTables = []string{ + Inodes, + NodeRecords, +} var HeimdallTables = []string{} var PolygonBridgeTables = []string{} var DownloaderTables = []string{ diff --git a/node/node.go b/node/node.go index 2116c91bec5..e54eade55be 100644 --- a/node/node.go +++ b/node/node.go @@ -377,30 +377,31 @@ func OpenDatabase(ctx context.Context, config *nodecfg.Config, label kv.Label, n return nil, err } - migrator := migrations.NewMigrator(label) - if err := migrator.VerifyVersion(db, dbPath); err != nil { - return nil, err - } - - has, err := migrator.HasPendingMigrations(db) - if err != nil { - return nil, err - } - if has && !dbg.OnlyCreateDB { - logger.Info("Re-Opening DB in exclusive mode to apply migrations") - db.Close() - db, err = openFunc(true) - if err != nil { + if label == kv.ChainDB { + migrator := migrations.NewMigrator(label) + if err := migrator.VerifyVersion(db, dbPath); err != nil { return nil, err } - if err = migrator.Apply(db, config.Dirs.DataDir, dbPath, logger); err != nil { - return nil, err - } - db.Close() - db, err = openFunc(false) + has, err := migrator.HasPendingMigrations(db) if err != nil { return nil, err } + if has && !dbg.OnlyCreateDB { + logger.Info("Re-Opening DB in exclusive mode to apply migrations") + db.Close() + db, err = openFunc(true) + if err != nil { + return nil, err + } + if err = migrator.Apply(db, config.Dirs.DataDir, dbPath, logger); err != nil { + return nil, err + } + db.Close() + db, err = openFunc(false) + if err != nil { + return nil, err + } + } } if err := db.Update(context.Background(), func(tx kv.RwTx) (err error) { diff --git a/node/node_test.go b/node/node_test.go index c554f8b0ec2..a968688bafe 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -156,14 +156,14 @@ func TestNodeCloseClosesDB(t *testing.T) { t.Fatal("can't open DB:", err) } if err = db.Update(context.Background(), func(tx kv.RwTx) error { - return tx.Put(kv.HashedAccounts, []byte("testK"), []byte{}) + return tx.Put(kv.Inodes, []byte("testK"), []byte{}) }); err != nil { t.Fatal("can't Put on open DB:", err) } stack.Close() //if err = db.Update(context.Background(), func(tx kv.RwTx) error { - // return tx.Put(kv.HashedAccounts, []byte("testK"), []byte{}) + // return tx.Put(kv.Inodes, []byte("testK"), []byte{}) //}); err == nil { // t.Fatal("Put succeeded after node is closed") //} From 96e45800551b49ecf33dae454b07767f3b41f1a2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 17 Oct 2024 21:43:55 +0700 Subject: [PATCH 06/27] save --- node/node.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/node/node.go b/node/node.go index e54eade55be..68cbf11e5f9 100644 --- a/node/node.go +++ b/node/node.go @@ -31,13 +31,13 @@ import ( "github.com/c2h5oh/datasize" "github.com/erigontech/erigon-lib/common/dbg" + "github.com/erigontech/erigon/params" "golang.org/x/sync/semaphore" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon/cmd/utils" "github.com/erigontech/erigon/node/nodecfg" - "github.com/erigontech/erigon/params" "github.com/erigontech/erigon/turbo/debug" "github.com/gofrs/flock" @@ -402,12 +402,11 @@ func OpenDatabase(ctx context.Context, config *nodecfg.Config, label kv.Label, n return nil, err } } - } - - if err := db.Update(context.Background(), func(tx kv.RwTx) (err error) { - return params.SetErigonVersion(tx, params.VersionKeyCreated) - }); err != nil { - return nil, err + if err := db.Update(context.Background(), func(tx kv.RwTx) (err error) { + return params.SetErigonVersion(tx, params.VersionKeyCreated) + }); err != nil { + return nil, err + } } return db, nil From 084c150d492d5a9203b067721485834cf8dd6f04 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 17 Oct 2024 21:44:56 +0700 Subject: [PATCH 07/27] save --- cmd/integration/commands/root.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/integration/commands/root.go b/cmd/integration/commands/root.go index 42b46cecd53..2dbd49d222e 100644 --- a/cmd/integration/commands/root.go +++ b/cmd/integration/commands/root.go @@ -23,6 +23,7 @@ import ( "path/filepath" "strings" + "github.com/erigontech/erigon/migrations" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" @@ -33,7 +34,6 @@ import ( kv2 "github.com/erigontech/erigon-lib/kv/mdbx" "github.com/erigontech/erigon/cmd/utils" - "github.com/erigontech/erigon/migrations" "github.com/erigontech/erigon/turbo/debug" "github.com/erigontech/erigon/turbo/logging" ) @@ -90,25 +90,25 @@ func dbCfg(label kv.Label, path string) kv2.MdbxOpts { func openDB(opts kv2.MdbxOpts, applyMigrations bool, logger log.Logger) (kv.RwDB, error) { db := opts.MustOpen() - if applyMigrations { - migrator := migrations.NewMigrator(opts.GetLabel()) - has, err := migrator.HasPendingMigrations(db) - if err != nil { - return nil, err - } - if has { - logger.Info("Re-Opening DB in exclusive mode to apply DB migrations") - db.Close() - db = opts.Exclusive().MustOpen() - if err := migrator.Apply(db, datadirCli, "", logger); err != nil { + if opts.GetLabel() == kv.ChainDB { + if applyMigrations { + migrator := migrations.NewMigrator(opts.GetLabel()) + has, err := migrator.HasPendingMigrations(db) + if err != nil { return nil, err } - db.Close() - db = opts.MustOpen() + if has { + logger.Info("Re-Opening DB in exclusive mode to apply DB migrations") + db.Close() + db = opts.Exclusive().MustOpen() + if err := migrator.Apply(db, datadirCli, "", logger); err != nil { + return nil, err + } + db.Close() + db = opts.MustOpen() + } } - } - if opts.GetLabel() == kv.ChainDB { _, _, agg, _ := allSnapshots(context.Background(), db, logger) tdb, err := temporal.New(db, agg) if err != nil { From 52ad370fd3ba675705e0226d49acaabf2fb2652e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 12:26:45 +0700 Subject: [PATCH 08/27] save --- .../hex_patricia_hashed_fuzz_test.go | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go index 19129463e59..bbc16f9f18f 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go @@ -21,7 +21,6 @@ package commitment import ( "bytes" "context" - "encoding/binary" "encoding/hex" "math" "math/rand" @@ -80,24 +79,15 @@ func Fuzz_ProcessUpdate(f *testing.F) { }) } -// go test -trimpath -v -fuzz=Fuzz_ProcessUpdates_ArbitraryUpdateCount -fuzztime=300s ./commitment +// go test -trimpath -v -fuzz=Fuzz_ProcessUpdates_ArbitraryUpdateCount2 -fuzztime=300s ./commitment -func Fuzz_ProcessUpdates_ArbitraryUpdateCount(f *testing.F) { - ha, _ := hex.DecodeString("0008852883b2850c7a48f4b0eea3ccc4c04e6cb6025e9e8f7db2589c7dae81517c514790cfd6f668903161349e") +func Fuzz_ProcessUpdates_ArbitraryUpdateCount2(f *testing.F) { + //ha, _ := hex.DecodeString("0008852883b2850c7a48f4b0eea3ccc4c04e6cb6025e9e8f7db2589c7dae81517c514790cfd6f668903161349e") ctx := context.Background() - f.Add(ha) + f.Add(uint16(10_000), uint32(1), uint32(2)) - f.Fuzz(func(t *testing.T, build []byte) { - if len(build) < 12 { - t.Skip() - } - i := 0 - keysCount := binary.BigEndian.Uint32(build[i : i+4]) - i += 4 - ks := binary.BigEndian.Uint32(build[i : i+4]) + f.Fuzz(func(t *testing.T, keysCount uint16, ks, us uint32) { keysSeed := rand.New(rand.NewSource(int64(ks))) - i += 4 - us := binary.BigEndian.Uint32(build[i : i+4]) updateSeed := rand.New(rand.NewSource(int64(us))) t.Logf("fuzzing %d keys keysSeed=%d updateSeed=%d", keysCount, ks, us) @@ -105,7 +95,7 @@ func Fuzz_ProcessUpdates_ArbitraryUpdateCount(f *testing.F) { plainKeys := make([][]byte, keysCount) updates := make([]Update, keysCount) - for k := uint32(0); k < keysCount; k++ { + for k := uint16(0); k < keysCount; k++ { aux := make([]byte, 32) From 8234a290283ab314c078668b6e358fbfa823efbb Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 14:10:38 +0700 Subject: [PATCH 09/27] save --- erigon-lib/kv/tables.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 18a620ce35b..faf916cf782 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -647,6 +647,7 @@ var SentryTables = []string{ Inodes, NodeRecords, } +var ConsensusTables = []string{} var HeimdallTables = []string{} var PolygonBridgeTables = []string{} var DownloaderTables = []string{ @@ -767,6 +768,7 @@ var BorTablesCfg = TableCfg{ var TxpoolTablesCfg = TableCfg{} var SentryTablesCfg = TableCfg{} +var ConsensusTablesCfg = TableCfg{} var DownloaderTablesCfg = TableCfg{} var DiagnosticsTablesCfg = TableCfg{} var HeimdallTablesCfg = TableCfg{} @@ -793,6 +795,8 @@ func TablesCfgByLabel(label Label) TableCfg { return HeimdallTablesCfg case PolygonBridgeDB: return PolygonBridgeTablesCfg + case ConsensusDB: + return ConsensusTablesCfg default: panic(fmt.Sprintf("unexpected label: %s", label)) } From 2292b279e00273b47f92356c81516ae533100d77 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 14:12:33 +0700 Subject: [PATCH 10/27] save --- erigon-lib/kv/tables.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index faf916cf782..40cf03cf005 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -506,8 +506,6 @@ var ChaindataTables = []string{ ConfigTable, DatabaseInfo, IncarnationMap, - CliqueSeparate, - CliqueLastSnapshot, SyncStageProgress, PlainState, PlainContractCode, @@ -647,7 +645,10 @@ var SentryTables = []string{ Inodes, NodeRecords, } -var ConsensusTables = []string{} +var ConsensusTables = []string{ + CliqueSeparate, + CliqueLastSnapshot, +} var HeimdallTables = []string{} var PolygonBridgeTables = []string{} var DownloaderTables = []string{ @@ -845,6 +846,13 @@ func reinit() { } } + for _, name := range ConsensusTables { + _, ok := ConsensusTablesCfg[name] + if !ok { + ConsensusTablesCfg[name] = TableCfgItem{} + } + } + for _, name := range DownloaderTables { _, ok := DownloaderTablesCfg[name] if !ok { From 5c83d51b3c2daceeff5b1a87049a2bbc7de2ef49 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:10:45 +0700 Subject: [PATCH 11/27] save --- consensus/clique/clique_test.go | 2 +- consensus/clique/snapshot_test.go | 2 +- erigon-lib/kv/memdb/memory_database.go | 26 +++++++------------ .../temporaltest/kv_temporal_testdb.go | 4 +-- node/node.go | 2 +- turbo/stages/mock/mock_sentry.go | 2 +- 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 5a9f44795d1..0f5cb32947b 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -48,7 +48,7 @@ import ( func TestReimportMirroredState(t *testing.T) { // Initialize a Clique chain with a single signer var ( - cliqueDB = memdb.NewTestDB(t) + cliqueDB = memdb.NewTestDB(t, kv.ChainDB) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) engine = clique.New(params.AllCliqueProtocolChanges, params.CliqueSnapshot, cliqueDB, log.New()) diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 49d6b5443d5..5b1e10aab6b 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -428,7 +428,7 @@ func TestClique(t *testing.T) { Epoch: tt.epoch, } - cliqueDB := memdb.NewTestDB(t) + cliqueDB := memdb.NewTestDB(t, kv.ConsensusDB) engine := clique.New(&config, params.CliqueSnapshot, cliqueDB, log.New()) engine.FakeDiff = true diff --git a/erigon-lib/kv/memdb/memory_database.go b/erigon-lib/kv/memdb/memory_database.go index d33c870748b..f9557aeb377 100644 --- a/erigon-lib/kv/memdb/memory_database.go +++ b/erigon-lib/kv/memdb/memory_database.go @@ -27,8 +27,8 @@ import ( "github.com/erigontech/erigon-lib/log/v3" ) -func New(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).MustOpen() +func New(tmpDir string, label kv.Label) kv.RwDB { + return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(label).MustOpen() } func NewStateDB(tmpDir string) kv.RwDB { @@ -38,21 +38,15 @@ func NewStateDB(tmpDir string) kv.RwDB { }).MustOpen() } -func NewPoolDB(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(kv.TxPoolDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.TxpoolTablesCfg }).MustOpen() -} -func NewDownloaderDB(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(kv.DownloaderDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.DownloaderTablesCfg }).MustOpen() -} -func NewSentryDB(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(kv.SentryDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.SentryTablesCfg }).MustOpen() +func NewWithLabel(tmpDir string, label kv.Label) kv.RwDB { + return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(label).MustOpen() } -func NewTestDB(tb testing.TB) kv.RwDB { +func NewTestDB(tb testing.TB, label kv.Label) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() tb.Helper() - db := New(tmpDir) + db := New(tmpDir, label) tb.Cleanup(db.Close) return db } @@ -80,7 +74,7 @@ func BeginRo(tb testing.TB, db kv.RoDB) kv.Tx { func NewTestPoolDB(tb testing.TB) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() - db := NewPoolDB(tmpDir) + db := New(tmpDir, kv.TxPoolDB) tb.Cleanup(db.Close) return db } @@ -88,7 +82,7 @@ func NewTestPoolDB(tb testing.TB) kv.RwDB { func NewTestDownloaderDB(tb testing.TB) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() - db := NewDownloaderDB(tmpDir) + db := New(tmpDir, kv.DownloaderDB) tb.Cleanup(db.Close) return db } @@ -96,7 +90,7 @@ func NewTestDownloaderDB(tb testing.TB) kv.RwDB { func NewTestSentrylDB(tb testing.TB) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() - db := NewPoolDB(tmpDir) + db := New(tmpDir, kv.TxPoolDB) tb.Cleanup(db.Close) return db } @@ -104,7 +98,7 @@ func NewTestSentrylDB(tb testing.TB) kv.RwDB { func NewTestTx(tb testing.TB) (kv.RwDB, kv.RwTx) { tb.Helper() tmpDir := tb.TempDir() - db := New(tmpDir) + db := New(tmpDir, kv.ChainDB) tb.Cleanup(db.Close) tx, err := db.BeginRw(context.Background()) //nolint:gocritic if err != nil { diff --git a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go index 0076d15daac..5c2df633b9e 100644 --- a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go +++ b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go @@ -36,9 +36,9 @@ func NewTestDB(tb testing.TB, dirs datadir.Dirs) (db kv.RwDB, agg *state.Aggrega } if tb != nil { - db = memdb.NewTestDB(tb) + db = memdb.NewTestDB(tb, kv.ChainDB) } else { - db = memdb.New(dirs.DataDir) + db = memdb.New(dirs.DataDir, kv.ChainDB) } var err error diff --git a/node/node.go b/node/node.go index 68cbf11e5f9..3e29ffc5b8a 100644 --- a/node/node.go +++ b/node/node.go @@ -316,7 +316,7 @@ func OpenDatabase(ctx context.Context, config *nodecfg.Config, label kv.Label, n var db kv.RwDB if config.Dirs.DataDir == "" { - db = memdb.New("") + db = memdb.New("", label) return db, nil } diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index d31ee10e72c..6e60da68589 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -343,7 +343,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK if err != nil { tb.Fatal(err) } - mock.txPoolDB = memdb.NewPoolDB(tmpdir) + mock.txPoolDB = memdb.NewWithLabel(tmpdir, kv.TxPoolDB) stateChangesClient := direct.NewStateDiffClientDirect(erigonGrpcServeer) From d6f51b440ecd9c0d1fb50fca41c51435dfe56be9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:12:55 +0700 Subject: [PATCH 12/27] save --- cl/phase1/stages/stage_history_download.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index 08d5f738d27..76c9f9e58ce 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -26,7 +26,6 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon-lib/kv/memdb" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/cl/antiquary" "github.com/erigontech/erigon/cl/persistence/beacon_indicies" @@ -265,14 +264,6 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co } cfg.downloader.SetThrottle(cfg.backfillingThrottling) // throttle to 0.6 second for backfilling cfg.downloader.SetNeverSkip(false) - // If i do not give it a database, erigon lib starts to cry uncontrollably - db2 := memdb.New(cfg.tmpdir) - defer db2.Close() - tx2, err := db2.BeginRw(ctx) - if err != nil { - return err - } - defer tx2.Rollback() isBackfilling.Store(true) cfg.logger.Info("Ready to insert history, waiting for sync cycle to finish") From c61717a0ee36760a881bf0b18c6135b9f0941f28 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:30:47 +0700 Subject: [PATCH 13/27] save --- erigon-lib/kv/membatch/database_test.go | 4 ++-- erigon-lib/kv/membatch/mapmutation_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/kv/membatch/database_test.go b/erigon-lib/kv/membatch/database_test.go index 9b02711f4c5..35919f62fcd 100644 --- a/erigon-lib/kv/membatch/database_test.go +++ b/erigon-lib/kv/membatch/database_test.go @@ -99,7 +99,7 @@ func TestPutGet(t *testing.T) { } func TestNoPanicAfterDbClosed(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) tx, err := db.BeginRo(context.Background()) require.NoError(t, err) defer tx.Rollback() @@ -136,7 +136,7 @@ func TestNoPanicAfterDbClosed(t *testing.T) { } func TestParallelPutGet(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) const n = 8 var pending sync.WaitGroup diff --git a/erigon-lib/kv/membatch/mapmutation_test.go b/erigon-lib/kv/membatch/mapmutation_test.go index c997ab63958..91aa5906ed2 100644 --- a/erigon-lib/kv/membatch/mapmutation_test.go +++ b/erigon-lib/kv/membatch/mapmutation_test.go @@ -30,7 +30,7 @@ import ( ) func TestMapmutation_Flush_Close(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) tx, err := db.BeginRw(context.Background()) require.NoError(t, err) From 572b34cc2323cf1ccda38212889b0099e7fbb8ce Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:31:26 +0700 Subject: [PATCH 14/27] save --- erigon-lib/kv/stream/stream_test.go | 2 +- erigon-lib/txpool/fetch_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/kv/stream/stream_test.go b/erigon-lib/kv/stream/stream_test.go index ff6e1f3e708..ffcf63ee441 100644 --- a/erigon-lib/kv/stream/stream_test.go +++ b/erigon-lib/kv/stream/stream_test.go @@ -79,7 +79,7 @@ func TestUnion(t *testing.T) { }) } func TestUnionPairs(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) ctx := context.Background() t.Run("simple", func(t *testing.T) { require := require.New(t) diff --git a/erigon-lib/txpool/fetch_test.go b/erigon-lib/txpool/fetch_test.go index a33308e56ef..ea3d7b4d947 100644 --- a/erigon-lib/txpool/fetch_test.go +++ b/erigon-lib/txpool/fetch_test.go @@ -230,7 +230,7 @@ func decodeHex(in string) []byte { func TestOnNewBlock(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - coreDB, db := memdb.NewTestDB(t), memdb.NewTestDB(t) + coreDB, db := memdb.NewTestDB(t, kv.ChainDB), memdb.NewTestDB(t, kv.TxPoolDB) ctrl := gomock.NewController(t) stream := remote.NewMockKV_StateChangesClient(ctrl) From ddade0c5ae683d09447059c4dee40c3d29055994 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:31:55 +0700 Subject: [PATCH 15/27] save --- erigon-lib/kv/remotedbserver/remotedbserver_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/kv/remotedbserver/remotedbserver_test.go b/erigon-lib/kv/remotedbserver/remotedbserver_test.go index e448363d37d..d1652da8b46 100644 --- a/erigon-lib/kv/remotedbserver/remotedbserver_test.go +++ b/erigon-lib/kv/remotedbserver/remotedbserver_test.go @@ -36,7 +36,7 @@ func TestKvServer_renew(t *testing.T) { t.Skip("fix me on win please") } - require, ctx, db := require.New(t), context.Background(), memdb.NewTestDB(t) + require, ctx, db := require.New(t), context.Background(), memdb.NewTestDB(t, kv.ChainDB) require.NoError(db.Update(ctx, func(tx kv.RwTx) error { wc, err := tx.RwCursorDupSort(kv.AccountChangeSet) require.NoError(err) From 8eb1bbfac7582b03a60510e6beb2b8149af00d35 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:35:12 +0700 Subject: [PATCH 16/27] save --- erigon-lib/kv/mdbx/kv_abstract_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/kv/mdbx/kv_abstract_test.go b/erigon-lib/kv/mdbx/kv_abstract_test.go index 3aaa4fad0f3..cd877302ae0 100644 --- a/erigon-lib/kv/mdbx/kv_abstract_test.go +++ b/erigon-lib/kv/mdbx/kv_abstract_test.go @@ -206,7 +206,7 @@ func TestRemoteKvRange(t *testing.T) { t.Skip("fix me on win please") } logger := log.New() - ctx, writeDB := context.Background(), memdb.NewTestDB(t) + ctx, writeDB := context.Background(), memdb.NewTestDB(t, kv.ChainDB) grpcServer, conn := grpc.NewServer(), bufconn.Listen(1024*1024) go func() { kvServer := remotedbserver.NewKvServer(ctx, writeDB, nil, nil, nil, logger) From db4cea4afb988cc5f52458832b2491057eba32a2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:36:04 +0700 Subject: [PATCH 17/27] save --- cl/beacon/handler/utils_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cl/beacon/handler/utils_test.go b/cl/beacon/handler/utils_test.go index 681be8c7140..8f66eceee98 100644 --- a/cl/beacon/handler/utils_test.go +++ b/cl/beacon/handler/utils_test.go @@ -65,8 +65,8 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge blocks, preState, postState = tests.GetCapellaRandom() } fcu = mock_services2.NewForkChoiceStorageMock(t) - db = memdb.NewTestDB(t) - blobDb := memdb.NewTestDB(t) + db = memdb.NewTestDB(t, kv.ChainDB) + blobDb := memdb.NewTestDB(t, kv.ChainDB) var reader *tests.MockBlockReader reader = tests.LoadChain(blocks, postState, db, t) firstBlockRoot, _ := blocks[0].Block.HashSSZ() From 7e0fbfbb510f4b0aea4726165b0d98b1ef436091 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:40:22 +0700 Subject: [PATCH 18/27] save --- cmd/rpcdaemon/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/rpcdaemon/main.go b/cmd/rpcdaemon/main.go index 7f0215a741e..ea00cbc6551 100644 --- a/cmd/rpcdaemon/main.go +++ b/cmd/rpcdaemon/main.go @@ -39,7 +39,7 @@ func main() { rootCtx, rootCancel := common.RootContext() cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - logger := debug.SetupCobra(cmd, "sentry") + logger := debug.SetupCobra(cmd, "rpcdaemon") db, backend, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, heimdallReader, err := cli.RemoteServices(ctx, cfg, logger, rootCancel) if err != nil { if !errors.Is(err, context.Canceled) { From 8e9a4d8073596622659d7707d239bf396d4e8a54 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 15:44:32 +0700 Subject: [PATCH 19/27] save --- cl/spectest/consensus_tests/fork_choice.go | 3 ++- cmd/devnet/services/polygon/proofgenerator_test.go | 2 +- cmd/evm/runner.go | 8 +++++--- cmd/pics/state.go | 2 +- polygon/bor/bor_test.go | 3 ++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cl/spectest/consensus_tests/fork_choice.go b/cl/spectest/consensus_tests/fork_choice.go index b52135deb12..f087469808d 100644 --- a/cl/spectest/consensus_tests/fork_choice.go +++ b/cl/spectest/consensus_tests/fork_choice.go @@ -23,6 +23,7 @@ import ( "math" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon/spectest" "github.com/spf13/afero" @@ -203,7 +204,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err emitters := beaconevents.NewEventEmitter() _, beaconConfig := clparams.GetConfigsByNetwork(clparams.MainnetNetwork) ethClock := eth_clock.NewEthereumClock(genesisState.GenesisTime(), genesisState.GenesisValidatorsRoot(), beaconConfig) - blobStorage := blob_storage.NewBlobStore(memdb.New("/tmp"), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock) + blobStorage := blob_storage.NewBlobStore(memdb.New("/tmp", kv.ChainDB), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock) validatorMonitor := monitor.NewValidatorMonitor(false, nil, nil, nil) forkStore, err := forkchoice.NewForkChoiceStore( diff --git a/cmd/devnet/services/polygon/proofgenerator_test.go b/cmd/devnet/services/polygon/proofgenerator_test.go index 9744425a695..f3e39c00d2d 100644 --- a/cmd/devnet/services/polygon/proofgenerator_test.go +++ b/cmd/devnet/services/polygon/proofgenerator_test.go @@ -67,7 +67,7 @@ type requestGenerator struct { } func newRequestGenerator(sentry *mock.MockSentry, chain *core.ChainPack) (*requestGenerator, error) { - db := memdb.New("") + db := memdb.New("", kv.ChainDB) if err := db.Update(context.Background(), func(tx kv.RwTx) error { if err := rawdb.WriteHeader(tx, chain.TopBlock.Header()); err != nil { return err diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 57c09bcceea..f36d6b37c9a 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -24,8 +24,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/erigontech/erigon-lib/config3" - "github.com/erigontech/erigon-lib/kv/temporal" "io" "math/big" "os" @@ -34,6 +32,10 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/config3" + "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/temporal" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/holiman/uint256" @@ -159,7 +161,7 @@ func runCmd(ctx *cli.Context) error { } else { debugLogger = logger.NewStructLogger(logconfig) } - db := memdb.New(os.TempDir()) + db := memdb.New(os.TempDir(), kv.ChainDB) defer db.Close() if ctx.String(GenesisFlag.Name) != "" { gen := readGenesis(ctx.String(GenesisFlag.Name)) diff --git a/cmd/pics/state.go b/cmd/pics/state.go index 7da9a520212..f995d09379e 100644 --- a/cmd/pics/state.go +++ b/cmd/pics/state.go @@ -437,7 +437,7 @@ func initialState1() error { return err } - emptyKv := memdb.New("") + emptyKv := memdb.New("", kv.ChainDB) if err = stateDatabaseComparison(emptyKv, m.DB, 0); err != nil { return err } diff --git a/polygon/bor/bor_test.go b/polygon/bor/bor_test.go index ef9a0b8ce2a..c6cfcee012e 100644 --- a/polygon/bor/bor_test.go +++ b/polygon/bor/bor_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -314,7 +315,7 @@ func newValidator(t *testing.T, heimdall *test_heimdall, blocks map[uint64]*type validatorAddress := crypto.PubkeyToAddress(validatorKey.PublicKey) bor := bor.New( heimdall.chainConfig, - memdb.New(""), + memdb.New("", kv.ChainDB), nil, /* blockReader */ &spanner{ ChainSpanner: bor.NewChainSpanner(borabi.ValidatorSetContractABI(), heimdall.chainConfig, false, logger), From 7899a1fb99a32f9a1b06230b5f678544d2e70f39 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 16:16:25 +0700 Subject: [PATCH 20/27] save --- migrations/migrations_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/migrations/migrations_test.go b/migrations/migrations_test.go index 5ff59dd8eb1..48bad188140 100644 --- a/migrations/migrations_test.go +++ b/migrations/migrations_test.go @@ -31,7 +31,7 @@ import ( ) func TestApplyWithInit(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { "one", @@ -96,7 +96,7 @@ func TestApplyWithInit(t *testing.T) { } func TestApplyWithoutInit(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { "one", @@ -161,7 +161,7 @@ func TestApplyWithoutInit(t *testing.T) { } func TestWhenNonFirstMigrationAlreadyApplied(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { "one", @@ -243,7 +243,7 @@ func TestMarshalStages(t *testing.T) { } func TestValidation(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { Name: "repeated_name", @@ -293,7 +293,7 @@ func TestValidation(t *testing.T) { } func TestCommitCallRequired(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { Name: "one", From 13fc59cdfc308f6c2440d24fbccac64cd95cdff0 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 16:18:20 +0700 Subject: [PATCH 21/27] save --- .../historical_states_reader/historical_states_reader_test.go | 3 ++- cl/sentinel/sentinel_requests_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go index 38151494504..df0299c1246 100644 --- a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go +++ b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" libcommon "github.com/erigontech/erigon-lib/common" @@ -36,7 +37,7 @@ import ( ) func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) reader := tests.LoadChain(blocks, postState, db, t) ctx := context.Background() diff --git a/cl/sentinel/sentinel_requests_test.go b/cl/sentinel/sentinel_requests_test.go index 1ca1a4972d5..5217e5584e6 100644 --- a/cl/sentinel/sentinel_requests_test.go +++ b/cl/sentinel/sentinel_requests_test.go @@ -50,7 +50,7 @@ import ( func loadChain(t *testing.T) (db kv.RwDB, blocks []*cltypes.SignedBeaconBlock, f afero.Fs, preState, postState *state.CachingBeaconState, reader *tests.MockBlockReader) { blocks, preState, postState = tests.GetPhase0Random() - db = memdb.NewTestDB(t) + db = memdb.NewTestDB(t, kv.ChainDB) reader = tests.LoadChain(blocks, postState, db, t) ctx := context.Background() From a653f98919662d9023fde3a1a6fb33f91db238c3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 18 Oct 2024 16:21:06 +0700 Subject: [PATCH 22/27] save --- cl/persistence/beacon_indicies/indicies_test.go | 2 +- cl/persistence/blob_storage/blob_db_test.go | 2 +- cl/phase1/network/services/block_service_test.go | 3 ++- cl/sentinel/handlers/blobs_test.go | 5 +++-- cl/sentinel/handlers/utils_test.go | 2 +- consensus/aura/aura_test.go | 4 +++- eth/stagedsync/stagedsynctest/harness.go | 2 +- polygon/bor/finality/whitelist/service_test.go | 8 ++++---- 8 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cl/persistence/beacon_indicies/indicies_test.go b/cl/persistence/beacon_indicies/indicies_test.go index 622f3637e30..a1f05af7f27 100644 --- a/cl/persistence/beacon_indicies/indicies_test.go +++ b/cl/persistence/beacon_indicies/indicies_test.go @@ -30,7 +30,7 @@ import ( func setupTestDB(t *testing.T) kv.RwDB { // Create an in-memory SQLite DB for testing purposes - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) return db } diff --git a/cl/persistence/blob_storage/blob_db_test.go b/cl/persistence/blob_storage/blob_db_test.go index 1481b65c12e..981f44af4c4 100644 --- a/cl/persistence/blob_storage/blob_db_test.go +++ b/cl/persistence/blob_storage/blob_db_test.go @@ -31,7 +31,7 @@ import ( ) func setupTestDB(t *testing.T) kv.RwDB { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) return db } diff --git a/cl/phase1/network/services/block_service_test.go b/cl/phase1/network/services/block_service_test.go index 772e8a85a57..c8fc80b9a8e 100644 --- a/cl/phase1/network/services/block_service_test.go +++ b/cl/phase1/network/services/block_service_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -34,7 +35,7 @@ import ( ) func setupBlockService(t *testing.T, ctrl *gomock.Controller) (BlockService, *synced_data.SyncedDataManager, *eth_clock.MockEthereumClock, *mock_services.ForkChoiceStorageMock) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) cfg := &clparams.MainnetBeaconConfig syncedDataManager := synced_data.NewSyncedDataManager(true, cfg) ethClock := eth_clock.NewMockEthereumClock(ctrl) diff --git a/cl/sentinel/handlers/blobs_test.go b/cl/sentinel/handlers/blobs_test.go index 12fee5be072..bf693892636 100644 --- a/cl/sentinel/handlers/blobs_test.go +++ b/cl/sentinel/handlers/blobs_test.go @@ -25,6 +25,7 @@ import ( "math" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/golang/snappy" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" @@ -88,7 +89,7 @@ func TestBlobsByRangeHandler(t *testing.T) { require.NoError(t, err) peersPool := peers.NewPool() - blobDb := memdb.NewTestDB(t) + blobDb := memdb.NewTestDB(t, kv.ChainDB) _, indiciesDB := setupStore(t) store := tests.NewMockBlockReader() @@ -209,7 +210,7 @@ func TestBlobsByIdentifiersHandler(t *testing.T) { require.NoError(t, err) peersPool := peers.NewPool() - blobDb := memdb.NewTestDB(t) + blobDb := memdb.NewTestDB(t, kv.ChainDB) _, indiciesDB := setupStore(t) store := tests.NewMockBlockReader() diff --git a/cl/sentinel/handlers/utils_test.go b/cl/sentinel/handlers/utils_test.go index 29ccd40ae60..9471fd083a3 100644 --- a/cl/sentinel/handlers/utils_test.go +++ b/cl/sentinel/handlers/utils_test.go @@ -33,7 +33,7 @@ import ( ) func setupStore(t *testing.T) (freezeblocks.BeaconSnapshotReader, kv.RwDB) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) return tests.NewMockBlockReader(), db } diff --git a/consensus/aura/aura_test.go b/consensus/aura/aura_test.go index af080be3825..a7b98115ffc 100644 --- a/consensus/aura/aura_test.go +++ b/consensus/aura/aura_test.go @@ -18,6 +18,8 @@ package aura_test import ( "github.com/erigontech/erigon-lib/common/datadir" + "github.com/erigontech/erigon-lib/kv" + "math/big" "strings" "testing" @@ -48,7 +50,7 @@ func TestEmptyBlock(t *testing.T) { genesis.Config.TerminalTotalDifficultyPassed = false chainConfig := genesis.Config - auraDB := memdb.NewTestDB(t) + auraDB := memdb.NewTestDB(t, kv.ChainDB) engine, err := aura.NewAuRa(chainConfig.Aura, auraDB) require.NoError(err) checkStateRoot := true diff --git a/eth/stagedsync/stagedsynctest/harness.go b/eth/stagedsync/stagedsynctest/harness.go index b01c21347b3..abd100fe96e 100644 --- a/eth/stagedsync/stagedsynctest/harness.go +++ b/eth/stagedsync/stagedsynctest/harness.go @@ -61,7 +61,7 @@ func InitHarness(ctx context.Context, t *testing.T, cfg HarnessCfg) Harness { m := mock.MockWithGenesis(t, genesisInit.genesis, genesisInit.genesisAllocPrivateKey, false) chainDataDB := m.DB blockReader := m.BlockReader - borConsensusDB := memdb.NewTestDB(t) + borConsensusDB := memdb.NewTestDB(t, kv.ChainDB) ctrl := gomock.NewController(t) heimdallClient := heimdall.NewMockHeimdallClient(ctrl) miningState := stagedsync.NewMiningState(ðconfig.Defaults.Miner) diff --git a/polygon/bor/finality/whitelist/service_test.go b/polygon/bor/finality/whitelist/service_test.go index 57012569b66..8b0dcc8fc44 100644 --- a/polygon/bor/finality/whitelist/service_test.go +++ b/polygon/bor/finality/whitelist/service_test.go @@ -65,7 +65,7 @@ func NewMockService(db kv.RwDB) *Service { func TestWhitelistedCheckpoint(t *testing.T) { t.Parallel() - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) //Creating the service for the whitelisting the checkpoints s := NewMockService(db) @@ -116,7 +116,7 @@ func TestWhitelistedCheckpoint(t *testing.T) { func TestMilestone(t *testing.T) { t.Parallel() - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) s := NewMockService(db) @@ -265,7 +265,7 @@ func TestMilestone(t *testing.T) { func TestIsValidChain(t *testing.T) { t.Parallel() - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) s := NewMockService(db) chainA := createMockChain(1, 20) // A1->A2...A19->A20 @@ -478,7 +478,7 @@ func TestIsValidChain(t *testing.T) { } func TestPropertyBasedTestingMilestone(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) rapid.Check(t, func(t *rapid.T) { From bc7ace366dc2d68c653486321148e252a5649173 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 19 Oct 2024 13:25:56 +0700 Subject: [PATCH 23/27] save --- cl/antiquary/state_antiquary_test.go | 3 ++- consensus/aura/aura_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cl/antiquary/state_antiquary_test.go b/cl/antiquary/state_antiquary_test.go index 08e37c4fc6c..5f33094c994 100644 --- a/cl/antiquary/state_antiquary_test.go +++ b/cl/antiquary/state_antiquary_test.go @@ -23,6 +23,7 @@ import ( _ "embed" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" "github.com/erigontech/erigon-lib/common/datadir" @@ -36,7 +37,7 @@ import ( ) func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) reader := tests.LoadChain(blocks, postState, db, t) ctx := context.Background() diff --git a/consensus/aura/aura_test.go b/consensus/aura/aura_test.go index a7b98115ffc..b26c59a0c49 100644 --- a/consensus/aura/aura_test.go +++ b/consensus/aura/aura_test.go @@ -89,7 +89,7 @@ func TestAuRaSkipGasLimit(t *testing.T) { genesis.Config.Aura.BlockGasLimitContractTransitions = map[uint64]libcommon.Address{0: libcommon.HexToAddress("0x4000000000000000000000000000000000000001")} chainConfig := genesis.Config - auraDB := memdb.NewTestDB(t) + auraDB := memdb.NewTestDB(t, kv.ChainDB) engine, err := aura.NewAuRa(chainConfig.Aura, auraDB) require.NoError(err) checkStateRoot := true From 331deac3a4d7ba5302be2d8516e026a2fcab046d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 19 Oct 2024 13:26:21 +0700 Subject: [PATCH 24/27] save --- erigon-lib/kv/memdb/memory_database.go | 44 -------------------------- 1 file changed, 44 deletions(-) diff --git a/erigon-lib/kv/memdb/memory_database.go b/erigon-lib/kv/memdb/memory_database.go index f9557aeb377..f4d82ab0bbd 100644 --- a/erigon-lib/kv/memdb/memory_database.go +++ b/erigon-lib/kv/memdb/memory_database.go @@ -61,16 +61,6 @@ func BeginRw(tb testing.TB, db kv.RwDB) kv.RwTx { return tx } -func BeginRo(tb testing.TB, db kv.RoDB) kv.Tx { - tb.Helper() - tx, err := db.BeginRo(context.Background()) //nolint:gocritic - if err != nil { - tb.Fatal(err) - } - tb.Cleanup(tx.Rollback) - return tx -} - func NewTestPoolDB(tb testing.TB) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() @@ -87,14 +77,6 @@ func NewTestDownloaderDB(tb testing.TB) kv.RwDB { return db } -func NewTestSentrylDB(tb testing.TB) kv.RwDB { - tb.Helper() - tmpDir := tb.TempDir() - db := New(tmpDir, kv.TxPoolDB) - tb.Cleanup(db.Close) - return db -} - func NewTestTx(tb testing.TB) (kv.RwDB, kv.RwTx) { tb.Helper() tmpDir := tb.TempDir() @@ -107,29 +89,3 @@ func NewTestTx(tb testing.TB) (kv.RwDB, kv.RwTx) { tb.Cleanup(tx.Rollback) return db, tx } - -func NewTestPoolTx(tb testing.TB) (kv.RwDB, kv.RwTx) { - tb.Helper() - db := NewTestPoolDB(tb) - tx, err := db.BeginRw(context.Background()) //nolint - if err != nil { - tb.Fatal(err) - } - if tb != nil { - tb.Cleanup(tx.Rollback) - } - return db, tx -} - -func NewTestSentryTx(tb testing.TB) (kv.RwDB, kv.RwTx) { - tb.Helper() - db := NewTestSentrylDB(tb) - tx, err := db.BeginRw(context.Background()) //nolint - if err != nil { - tb.Fatal(err) - } - if tb != nil { - tb.Cleanup(tx.Rollback) - } - return db, tx -} From ec7712694f6070c0fa5a303443a94ead3cf46c85 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 20 Oct 2024 12:55:04 +0700 Subject: [PATCH 25/27] save --- consensus/clique/clique_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 0f5cb32947b..e62f5180ac4 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -48,7 +48,7 @@ import ( func TestReimportMirroredState(t *testing.T) { // Initialize a Clique chain with a single signer var ( - cliqueDB = memdb.NewTestDB(t, kv.ChainDB) + cliqueDB = memdb.NewTestDB(t, kv.ConsensusDB) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) engine = clique.New(params.AllCliqueProtocolChanges, params.CliqueSnapshot, cliqueDB, log.New()) From 6f608b6cfed2f3129dc626fd6040c2be27d92ac2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 20 Oct 2024 12:57:04 +0700 Subject: [PATCH 26/27] save --- erigon-lib/kv/mdbx/kv_mdbx.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index a69fc2e5ec9..87c1b66d078 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -1203,7 +1203,6 @@ func (tx *MdbxTx) Put(table string, k, v []byte) error { func (tx *MdbxTx) Delete(table string, k []byte) error { err := tx.tx.Del(mdbx.DBI(tx.db.buckets[table].DBI), k, nil) - //TODO: revise the logic, why we should drop not found err? maybe we need another function for get with key error if mdbx.IsNotFound(err) { return nil } @@ -1212,10 +1211,12 @@ func (tx *MdbxTx) Delete(table string, k []byte) error { func (tx *MdbxTx) GetOne(bucket string, k []byte) ([]byte, error) { v, err := tx.tx.Get(mdbx.DBI(tx.db.buckets[bucket].DBI), k) - //TODO: revise the logic, why we should drop not found err? maybe we need another function for get with key error if mdbx.IsNotFound(err) { return nil, nil } + if err != nil { + return nil, fmt.Errorf("label: %s, table: %s, %w", tx.db.opts.label, bucket, err) + } return v, err } From c4d5129224da372dc5f384a55324339239ab7bb3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 20 Oct 2024 16:49:11 +0700 Subject: [PATCH 27/27] save --- erigon-lib/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index c9da468c0af..93bc2644848 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -41,7 +41,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/btree v1.6.0 - go.uber.org/mock v0.5.0 + go.uber.org/mock v0.5.0 golang.org/x/crypto v0.28.0 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b golang.org/x/sync v0.8.0