diff --git a/cl/antiquary/state_antiquary_test.go b/cl/antiquary/state_antiquary_test.go index 08e37c4fc6c..5f33094c994 100644 --- a/cl/antiquary/state_antiquary_test.go +++ b/cl/antiquary/state_antiquary_test.go @@ -23,6 +23,7 @@ import ( _ "embed" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" "github.com/erigontech/erigon-lib/common/datadir" @@ -36,7 +37,7 @@ import ( ) func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) reader := tests.LoadChain(blocks, postState, db, t) ctx := context.Background() diff --git a/cl/beacon/handler/utils_test.go b/cl/beacon/handler/utils_test.go index 5be6cfcd2a2..72297c77a13 100644 --- a/cl/beacon/handler/utils_test.go +++ b/cl/beacon/handler/utils_test.go @@ -67,8 +67,8 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge blocks, preState, postState = tests.GetCapellaRandom() } fcu = mock_services2.NewForkChoiceStorageMock(t) - db = memdb.NewTestDB(t) - blobDb := memdb.NewTestDB(t) + db = memdb.NewTestDB(t, kv.ChainDB) + blobDb := memdb.NewTestDB(t, kv.ChainDB) var reader *tests.MockBlockReader reader = tests.LoadChain(blocks, postState, db, t) firstBlockRoot, _ := blocks[0].Block.HashSSZ() diff --git a/cl/persistence/beacon_indicies/indicies_test.go b/cl/persistence/beacon_indicies/indicies_test.go index 249d4001687..38eb781f8c0 100644 --- a/cl/persistence/beacon_indicies/indicies_test.go +++ b/cl/persistence/beacon_indicies/indicies_test.go @@ -30,7 +30,7 @@ import ( func setupTestDB(t *testing.T) kv.RwDB { // Create an in-memory SQLite DB for testing purposes - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) return db } diff --git a/cl/persistence/blob_storage/blob_db_test.go b/cl/persistence/blob_storage/blob_db_test.go index 1481b65c12e..981f44af4c4 100644 --- a/cl/persistence/blob_storage/blob_db_test.go +++ b/cl/persistence/blob_storage/blob_db_test.go @@ -31,7 +31,7 @@ import ( ) func setupTestDB(t *testing.T) kv.RwDB { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) return db } diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go index 38151494504..df0299c1246 100644 --- a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go +++ b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" libcommon "github.com/erigontech/erigon-lib/common" @@ -36,7 +37,7 @@ import ( ) func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) reader := tests.LoadChain(blocks, postState, db, t) ctx := context.Background() diff --git a/cl/phase1/network/services/block_service_test.go b/cl/phase1/network/services/block_service_test.go index 772e8a85a57..c8fc80b9a8e 100644 --- a/cl/phase1/network/services/block_service_test.go +++ b/cl/phase1/network/services/block_service_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -34,7 +35,7 @@ import ( ) func setupBlockService(t *testing.T, ctrl *gomock.Controller) (BlockService, *synced_data.SyncedDataManager, *eth_clock.MockEthereumClock, *mock_services.ForkChoiceStorageMock) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) cfg := &clparams.MainnetBeaconConfig syncedDataManager := synced_data.NewSyncedDataManager(true, cfg) ethClock := eth_clock.NewMockEthereumClock(ctrl) diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index 18a6fc93095..20d6cd30190 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -26,7 +26,6 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon-lib/kv/memdb" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/cl/antiquary" "github.com/erigontech/erigon/cl/persistence/beacon_indicies" @@ -265,14 +264,6 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co } cfg.downloader.SetThrottle(cfg.backfillingThrottling) // throttle to 0.6 second for backfilling cfg.downloader.SetNeverSkip(false) - // If i do not give it a database, erigon lib starts to cry uncontrollably - db2 := memdb.New(cfg.tmpdir) - defer db2.Close() - tx2, err := db2.BeginRw(ctx) - if err != nil { - return err - } - defer tx2.Rollback() isBackfilling.Store(true) cfg.logger.Info("Ready to insert history, waiting for sync cycle to finish") diff --git a/cl/sentinel/handlers/blobs_test.go b/cl/sentinel/handlers/blobs_test.go index 12fee5be072..bf693892636 100644 --- a/cl/sentinel/handlers/blobs_test.go +++ b/cl/sentinel/handlers/blobs_test.go @@ -25,6 +25,7 @@ import ( "math" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/golang/snappy" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" @@ -88,7 +89,7 @@ func TestBlobsByRangeHandler(t *testing.T) { require.NoError(t, err) peersPool := peers.NewPool() - blobDb := memdb.NewTestDB(t) + blobDb := memdb.NewTestDB(t, kv.ChainDB) _, indiciesDB := setupStore(t) store := tests.NewMockBlockReader() @@ -209,7 +210,7 @@ func TestBlobsByIdentifiersHandler(t *testing.T) { require.NoError(t, err) peersPool := peers.NewPool() - blobDb := memdb.NewTestDB(t) + blobDb := memdb.NewTestDB(t, kv.ChainDB) _, indiciesDB := setupStore(t) store := tests.NewMockBlockReader() diff --git a/cl/sentinel/handlers/utils_test.go b/cl/sentinel/handlers/utils_test.go index 06798fd69ea..81a6c9e1d03 100644 --- a/cl/sentinel/handlers/utils_test.go +++ b/cl/sentinel/handlers/utils_test.go @@ -33,7 +33,7 @@ import ( ) func setupStore(t *testing.T) (freezeblocks.BeaconSnapshotReader, kv.RwDB) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) return tests.NewMockBlockReader(), db } diff --git a/cl/sentinel/sentinel_requests_test.go b/cl/sentinel/sentinel_requests_test.go index cfd0fa65cd4..2805d08db28 100644 --- a/cl/sentinel/sentinel_requests_test.go +++ b/cl/sentinel/sentinel_requests_test.go @@ -50,7 +50,7 @@ import ( func loadChain(t *testing.T) (db kv.RwDB, blocks []*cltypes.SignedBeaconBlock, f afero.Fs, preState, postState *state.CachingBeaconState, reader *tests.MockBlockReader) { blocks, preState, postState = tests.GetPhase0Random() - db = memdb.NewTestDB(t) + db = memdb.NewTestDB(t, kv.ChainDB) reader = tests.LoadChain(blocks, postState, db, t) ctx := context.Background() diff --git a/cl/spectest/consensus_tests/fork_choice.go b/cl/spectest/consensus_tests/fork_choice.go index 1872b32371b..895903a8069 100644 --- a/cl/spectest/consensus_tests/fork_choice.go +++ b/cl/spectest/consensus_tests/fork_choice.go @@ -23,6 +23,7 @@ import ( "math" "testing" + "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon/spectest" "github.com/spf13/afero" @@ -203,7 +204,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err emitters := beaconevents.NewEventEmitter() _, beaconConfig := clparams.GetConfigsByNetwork(clparams.MainnetNetwork) ethClock := eth_clock.NewEthereumClock(genesisState.GenesisTime(), genesisState.GenesisValidatorsRoot(), beaconConfig) - blobStorage := blob_storage.NewBlobStore(memdb.New("/tmp"), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock) + blobStorage := blob_storage.NewBlobStore(memdb.New("/tmp", kv.ChainDB), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock) validatorMonitor := monitor.NewValidatorMonitor(false, nil, nil, nil) forkStore, err := forkchoice.NewForkChoiceStore( diff --git a/cmd/devnet/services/polygon/proofgenerator_test.go b/cmd/devnet/services/polygon/proofgenerator_test.go index 54398a340a8..301fa095f25 100644 --- a/cmd/devnet/services/polygon/proofgenerator_test.go +++ b/cmd/devnet/services/polygon/proofgenerator_test.go @@ -67,7 +67,7 @@ type requestGenerator struct { } func newRequestGenerator(sentry *mock.MockSentry, chain *core.ChainPack) (*requestGenerator, error) { - db := memdb.New("") + db := memdb.New("", kv.ChainDB) if err := db.Update(context.Background(), func(tx kv.RwTx) error { if err := rawdb.WriteHeader(tx, chain.TopBlock.Header()); err != nil { return err diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index adec2e64675..907617ed815 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -8,8 +8,8 @@ import ( "math/big" "github.com/erigontech/erigon-lib/common" - common0 "github.com/erigontech/erigon/common" "github.com/erigontech/erigon-lib/common/math" + common0 "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/core/types" ) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 57c09bcceea..f36d6b37c9a 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -24,8 +24,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/erigontech/erigon-lib/config3" - "github.com/erigontech/erigon-lib/kv/temporal" "io" "math/big" "os" @@ -34,6 +32,10 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/config3" + "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/temporal" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/holiman/uint256" @@ -159,7 +161,7 @@ func runCmd(ctx *cli.Context) error { } else { debugLogger = logger.NewStructLogger(logconfig) } - db := memdb.New(os.TempDir()) + db := memdb.New(os.TempDir(), kv.ChainDB) defer db.Close() if ctx.String(GenesisFlag.Name) != "" { gen := readGenesis(ctx.String(GenesisFlag.Name)) diff --git a/cmd/integration/commands/root.go b/cmd/integration/commands/root.go index a3d1441a61f..dfef79b7f9a 100644 --- a/cmd/integration/commands/root.go +++ b/cmd/integration/commands/root.go @@ -23,6 +23,7 @@ import ( "path/filepath" "strings" + "github.com/erigontech/erigon/migrations" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" @@ -33,7 +34,6 @@ import ( kv2 "github.com/erigontech/erigon-lib/kv/mdbx" "github.com/erigontech/erigon/cmd/utils" - "github.com/erigontech/erigon/migrations" "github.com/erigontech/erigon/turbo/debug" "github.com/erigontech/erigon/turbo/logging" ) @@ -90,25 +90,25 @@ func dbCfg(label kv.Label, path string) kv2.MdbxOpts { func openDB(opts kv2.MdbxOpts, applyMigrations bool, logger log.Logger) (kv.RwDB, error) { db := opts.MustOpen() - if applyMigrations { - migrator := migrations.NewMigrator(opts.GetLabel()) - has, err := migrator.HasPendingMigrations(db) - if err != nil { - return nil, err - } - if has { - logger.Info("Re-Opening DB in exclusive mode to apply DB migrations") - db.Close() - db = opts.Exclusive().MustOpen() - if err := migrator.Apply(db, datadirCli, "", logger); err != nil { + if opts.GetLabel() == kv.ChainDB { + if applyMigrations { + migrator := migrations.NewMigrator(opts.GetLabel()) + has, err := migrator.HasPendingMigrations(db) + if err != nil { return nil, err } - db.Close() - db = opts.MustOpen() + if has { + logger.Info("Re-Opening DB in exclusive mode to apply DB migrations") + db.Close() + db = opts.Exclusive().MustOpen() + if err := migrator.Apply(db, datadirCli, "", logger); err != nil { + return nil, err + } + db.Close() + db = opts.MustOpen() + } } - } - if opts.GetLabel() == kv.ChainDB { _, _, agg, _, _, _ := allSnapshots(context.Background(), db, logger) tdb, err := temporal.New(db, agg) if err != nil { diff --git a/cmd/pics/state.go b/cmd/pics/state.go index 1d1d195e6d6..1f0e322b634 100644 --- a/cmd/pics/state.go +++ b/cmd/pics/state.go @@ -436,7 +436,7 @@ func initialState1() error { return err } - emptyKv := memdb.New("") + emptyKv := memdb.New("", kv.ChainDB) if err = stateDatabaseComparison(emptyKv, m.DB, 0); err != nil { return err } diff --git a/cmd/rpcdaemon/main.go b/cmd/rpcdaemon/main.go index 93253f908fa..c98e581ef80 100644 --- a/cmd/rpcdaemon/main.go +++ b/cmd/rpcdaemon/main.go @@ -39,7 +39,7 @@ func main() { rootCtx, rootCancel := common.RootContext() cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - logger := debug.SetupCobra(cmd, "sentry") + logger := debug.SetupCobra(cmd, "rpcdaemon") db, backend, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, heimdallReader, err := cli.RemoteServices(ctx, cfg, logger, rootCancel) if err != nil { if !errors.Is(err, context.Canceled) { diff --git a/consensus/aura/aura_test.go b/consensus/aura/aura_test.go index af080be3825..b26c59a0c49 100644 --- a/consensus/aura/aura_test.go +++ b/consensus/aura/aura_test.go @@ -18,6 +18,8 @@ package aura_test import ( "github.com/erigontech/erigon-lib/common/datadir" + "github.com/erigontech/erigon-lib/kv" + "math/big" "strings" "testing" @@ -48,7 +50,7 @@ func TestEmptyBlock(t *testing.T) { genesis.Config.TerminalTotalDifficultyPassed = false chainConfig := genesis.Config - auraDB := memdb.NewTestDB(t) + auraDB := memdb.NewTestDB(t, kv.ChainDB) engine, err := aura.NewAuRa(chainConfig.Aura, auraDB) require.NoError(err) checkStateRoot := true @@ -87,7 +89,7 @@ func TestAuRaSkipGasLimit(t *testing.T) { genesis.Config.Aura.BlockGasLimitContractTransitions = map[uint64]libcommon.Address{0: libcommon.HexToAddress("0x4000000000000000000000000000000000000001")} chainConfig := genesis.Config - auraDB := memdb.NewTestDB(t) + auraDB := memdb.NewTestDB(t, kv.ChainDB) engine, err := aura.NewAuRa(chainConfig.Aura, auraDB) require.NoError(err) checkStateRoot := true diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 4e155209b88..d7814a62375 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -48,7 +48,7 @@ import ( func TestReimportMirroredState(t *testing.T) { // Initialize a Clique chain with a single signer var ( - cliqueDB = memdb.NewTestDB(t) + cliqueDB = memdb.NewTestDB(t, kv.ConsensusDB) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) engine = clique.New(params.AllCliqueProtocolChanges, params.CliqueSnapshot, cliqueDB, log.New()) diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 1d81643f5d2..b7081ebf51c 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -427,7 +427,7 @@ func TestClique(t *testing.T) { Epoch: tt.epoch, } - cliqueDB := memdb.NewTestDB(t) + cliqueDB := memdb.NewTestDB(t, kv.ConsensusDB) engine := clique.New(&config, params.CliqueSnapshot, cliqueDB, log.New()) engine.FakeDiff = true diff --git a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go index e87d19897ca..bbc16f9f18f 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go @@ -21,7 +21,6 @@ package commitment import ( "bytes" "context" - "encoding/binary" "encoding/hex" "math" "math/rand" @@ -80,24 +79,15 @@ func Fuzz_ProcessUpdate(f *testing.F) { }) } -// go test -trimpath -v -fuzz=Fuzz_ProcessUpdates_ArbitraryUpdateCount -fuzztime=300s ./commitment +// go test -trimpath -v -fuzz=Fuzz_ProcessUpdates_ArbitraryUpdateCount2 -fuzztime=300s ./commitment -func Fuzz_ProcessUpdates_ArbitraryUpdateCount(f *testing.F) { - ha, _ := hex.DecodeString("0008852883b2850c7a48f4b0eea3ccc4c04e6cb6025e9e8f7db2589c7dae81517c514790cfd6f668903161349e") +func Fuzz_ProcessUpdates_ArbitraryUpdateCount2(f *testing.F) { + //ha, _ := hex.DecodeString("0008852883b2850c7a48f4b0eea3ccc4c04e6cb6025e9e8f7db2589c7dae81517c514790cfd6f668903161349e") ctx := context.Background() - f.Add(ha) + f.Add(uint16(10_000), uint32(1), uint32(2)) - f.Fuzz(func(t *testing.T, build []byte) { - if len(build) < 12 { - t.Skip() - } - i := 0 - keysCount := uint16(binary.BigEndian.Uint32(build[i : i+4])) - i += 4 - ks := binary.BigEndian.Uint32(build[i : i+4]) + f.Fuzz(func(t *testing.T, keysCount uint16, ks, us uint32) { keysSeed := rand.New(rand.NewSource(int64(ks))) - i += 4 - us := binary.BigEndian.Uint32(build[i : i+4]) updateSeed := rand.New(rand.NewSource(int64(us))) t.Logf("fuzzing %d keys keysSeed=%d updateSeed=%d", keysCount, ks, us) diff --git a/erigon-lib/crypto/bn256/cloudflare/lattice_test.go b/erigon-lib/crypto/bn256/cloudflare/lattice_test.go index 4d52ad9b27a..50e602cd018 100644 --- a/erigon-lib/crypto/bn256/cloudflare/lattice_test.go +++ b/erigon-lib/crypto/bn256/cloudflare/lattice_test.go @@ -2,7 +2,6 @@ package bn256 import ( "crypto/rand" - "testing" ) diff --git a/erigon-lib/crypto/bn256/cloudflare/main_test.go b/erigon-lib/crypto/bn256/cloudflare/main_test.go index f0610ed0694..65ba275cb93 100644 --- a/erigon-lib/crypto/bn256/cloudflare/main_test.go +++ b/erigon-lib/crypto/bn256/cloudflare/main_test.go @@ -1,9 +1,8 @@ package bn256 import ( - "testing" - "crypto/rand" + "testing" ) func TestRandomG2Marshal(t *testing.T) { diff --git a/erigon-lib/crypto/bn256/google/main_test.go b/erigon-lib/crypto/bn256/google/main_test.go index c0c85457bec..710f82a2a6d 100644 --- a/erigon-lib/crypto/bn256/google/main_test.go +++ b/erigon-lib/crypto/bn256/google/main_test.go @@ -1,9 +1,8 @@ package bn256 import ( - "testing" - "crypto/rand" + "testing" ) func TestRandomG2Marshal(t *testing.T) { diff --git a/erigon-lib/kv/mdbx/kv_abstract_test.go b/erigon-lib/kv/mdbx/kv_abstract_test.go index 3aaa4fad0f3..cd877302ae0 100644 --- a/erigon-lib/kv/mdbx/kv_abstract_test.go +++ b/erigon-lib/kv/mdbx/kv_abstract_test.go @@ -206,7 +206,7 @@ func TestRemoteKvRange(t *testing.T) { t.Skip("fix me on win please") } logger := log.New() - ctx, writeDB := context.Background(), memdb.NewTestDB(t) + ctx, writeDB := context.Background(), memdb.NewTestDB(t, kv.ChainDB) grpcServer, conn := grpc.NewServer(), bufconn.Listen(1024*1024) go func() { kvServer := remotedbserver.NewKvServer(ctx, writeDB, nil, nil, nil, logger) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index c7283200362..954c24ed3c3 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -419,7 +419,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) { MaxBatchDelay: DefaultMaxBatchDelay, } - customBuckets := opts.bucketsCfg(kv.ChaindataTablesCfg) + customBuckets := opts.bucketsCfg(kv.TablesCfgByLabel(opts.label)) for name, cfg := range customBuckets { // copy map to avoid changing global variable db.buckets[name] = cfg } @@ -1203,7 +1203,6 @@ func (tx *MdbxTx) Put(table string, k, v []byte) error { func (tx *MdbxTx) Delete(table string, k []byte) error { err := tx.tx.Del(mdbx.DBI(tx.db.buckets[table].DBI), k, nil) - //TODO: revise the logic, why we should drop not found err? maybe we need another function for get with key error if mdbx.IsNotFound(err) { return nil } @@ -1212,10 +1211,12 @@ func (tx *MdbxTx) Delete(table string, k []byte) error { func (tx *MdbxTx) GetOne(bucket string, k []byte) ([]byte, error) { v, err := tx.tx.Get(mdbx.DBI(tx.db.buckets[bucket].DBI), k) - //TODO: revise the logic, why we should drop not found err? maybe we need another function for get with key error if mdbx.IsNotFound(err) { return nil, nil } + if err != nil { + return nil, fmt.Errorf("label: %s, table: %s, %w", tx.db.opts.label, bucket, err) + } return v, err } diff --git a/erigon-lib/kv/membatch/database_test.go b/erigon-lib/kv/membatch/database_test.go index 9b02711f4c5..35919f62fcd 100644 --- a/erigon-lib/kv/membatch/database_test.go +++ b/erigon-lib/kv/membatch/database_test.go @@ -99,7 +99,7 @@ func TestPutGet(t *testing.T) { } func TestNoPanicAfterDbClosed(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) tx, err := db.BeginRo(context.Background()) require.NoError(t, err) defer tx.Rollback() @@ -136,7 +136,7 @@ func TestNoPanicAfterDbClosed(t *testing.T) { } func TestParallelPutGet(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) const n = 8 var pending sync.WaitGroup diff --git a/erigon-lib/kv/membatch/mapmutation_test.go b/erigon-lib/kv/membatch/mapmutation_test.go index c997ab63958..91aa5906ed2 100644 --- a/erigon-lib/kv/membatch/mapmutation_test.go +++ b/erigon-lib/kv/membatch/mapmutation_test.go @@ -30,7 +30,7 @@ import ( ) func TestMapmutation_Flush_Close(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) tx, err := db.BeginRw(context.Background()) require.NoError(t, err) diff --git a/erigon-lib/kv/memdb/memory_database.go b/erigon-lib/kv/memdb/memory_database.go index d33c870748b..f4d82ab0bbd 100644 --- a/erigon-lib/kv/memdb/memory_database.go +++ b/erigon-lib/kv/memdb/memory_database.go @@ -27,8 +27,8 @@ import ( "github.com/erigontech/erigon-lib/log/v3" ) -func New(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).MustOpen() +func New(tmpDir string, label kv.Label) kv.RwDB { + return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(label).MustOpen() } func NewStateDB(tmpDir string) kv.RwDB { @@ -38,21 +38,15 @@ func NewStateDB(tmpDir string) kv.RwDB { }).MustOpen() } -func NewPoolDB(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(kv.TxPoolDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.TxpoolTablesCfg }).MustOpen() -} -func NewDownloaderDB(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(kv.DownloaderDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.DownloaderTablesCfg }).MustOpen() -} -func NewSentryDB(tmpDir string) kv.RwDB { - return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(kv.SentryDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.SentryTablesCfg }).MustOpen() +func NewWithLabel(tmpDir string, label kv.Label) kv.RwDB { + return mdbx.NewMDBX(log.New()).InMem(tmpDir).Label(label).MustOpen() } -func NewTestDB(tb testing.TB) kv.RwDB { +func NewTestDB(tb testing.TB, label kv.Label) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() tb.Helper() - db := New(tmpDir) + db := New(tmpDir, label) tb.Cleanup(db.Close) return db } @@ -67,20 +61,10 @@ func BeginRw(tb testing.TB, db kv.RwDB) kv.RwTx { return tx } -func BeginRo(tb testing.TB, db kv.RoDB) kv.Tx { - tb.Helper() - tx, err := db.BeginRo(context.Background()) //nolint:gocritic - if err != nil { - tb.Fatal(err) - } - tb.Cleanup(tx.Rollback) - return tx -} - func NewTestPoolDB(tb testing.TB) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() - db := NewPoolDB(tmpDir) + db := New(tmpDir, kv.TxPoolDB) tb.Cleanup(db.Close) return db } @@ -88,15 +72,7 @@ func NewTestPoolDB(tb testing.TB) kv.RwDB { func NewTestDownloaderDB(tb testing.TB) kv.RwDB { tb.Helper() tmpDir := tb.TempDir() - db := NewDownloaderDB(tmpDir) - tb.Cleanup(db.Close) - return db -} - -func NewTestSentrylDB(tb testing.TB) kv.RwDB { - tb.Helper() - tmpDir := tb.TempDir() - db := NewPoolDB(tmpDir) + db := New(tmpDir, kv.DownloaderDB) tb.Cleanup(db.Close) return db } @@ -104,7 +80,7 @@ func NewTestSentrylDB(tb testing.TB) kv.RwDB { func NewTestTx(tb testing.TB) (kv.RwDB, kv.RwTx) { tb.Helper() tmpDir := tb.TempDir() - db := New(tmpDir) + db := New(tmpDir, kv.ChainDB) tb.Cleanup(db.Close) tx, err := db.BeginRw(context.Background()) //nolint:gocritic if err != nil { @@ -113,29 +89,3 @@ func NewTestTx(tb testing.TB) (kv.RwDB, kv.RwTx) { tb.Cleanup(tx.Rollback) return db, tx } - -func NewTestPoolTx(tb testing.TB) (kv.RwDB, kv.RwTx) { - tb.Helper() - db := NewTestPoolDB(tb) - tx, err := db.BeginRw(context.Background()) //nolint - if err != nil { - tb.Fatal(err) - } - if tb != nil { - tb.Cleanup(tx.Rollback) - } - return db, tx -} - -func NewTestSentryTx(tb testing.TB) (kv.RwDB, kv.RwTx) { - tb.Helper() - db := NewTestSentrylDB(tb) - tx, err := db.BeginRw(context.Background()) //nolint - if err != nil { - tb.Fatal(err) - } - if tb != nil { - tb.Cleanup(tx.Rollback) - } - return db, tx -} diff --git a/erigon-lib/kv/remotedbserver/remotedbserver_test.go b/erigon-lib/kv/remotedbserver/remotedbserver_test.go index e448363d37d..d1652da8b46 100644 --- a/erigon-lib/kv/remotedbserver/remotedbserver_test.go +++ b/erigon-lib/kv/remotedbserver/remotedbserver_test.go @@ -36,7 +36,7 @@ func TestKvServer_renew(t *testing.T) { t.Skip("fix me on win please") } - require, ctx, db := require.New(t), context.Background(), memdb.NewTestDB(t) + require, ctx, db := require.New(t), context.Background(), memdb.NewTestDB(t, kv.ChainDB) require.NoError(db.Update(ctx, func(tx kv.RwTx) error { wc, err := tx.RwCursorDupSort(kv.AccountChangeSet) require.NoError(err) diff --git a/erigon-lib/kv/stream/stream_test.go b/erigon-lib/kv/stream/stream_test.go index ff6e1f3e708..ffcf63ee441 100644 --- a/erigon-lib/kv/stream/stream_test.go +++ b/erigon-lib/kv/stream/stream_test.go @@ -79,7 +79,7 @@ func TestUnion(t *testing.T) { }) } func TestUnionPairs(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) ctx := context.Background() t.Run("simple", func(t *testing.T) { require := require.New(t) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 4391e16bb00..40cf03cf005 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -506,8 +506,6 @@ var ChaindataTables = []string{ ConfigTable, DatabaseInfo, IncarnationMap, - CliqueSeparate, - CliqueLastSnapshot, SyncStageProgress, PlainState, PlainContractCode, @@ -643,7 +641,16 @@ var TxPoolTables = []string{ PoolTransaction, PoolInfo, } -var SentryTables = []string{} +var SentryTables = []string{ + Inodes, + NodeRecords, +} +var ConsensusTables = []string{ + CliqueSeparate, + CliqueLastSnapshot, +} +var HeimdallTables = []string{} +var PolygonBridgeTables = []string{} var DownloaderTables = []string{ BittorrentCompletion, BittorrentInfo, @@ -762,8 +769,11 @@ var BorTablesCfg = TableCfg{ var TxpoolTablesCfg = TableCfg{} var SentryTablesCfg = TableCfg{} +var ConsensusTablesCfg = TableCfg{} var DownloaderTablesCfg = TableCfg{} var DiagnosticsTablesCfg = TableCfg{} +var HeimdallTablesCfg = TableCfg{} +var PolygonBridgeTablesCfg = TableCfg{} var ReconTablesCfg = TableCfg{ PlainStateD: {Flags: DupSort}, CodeD: {Flags: DupSort}, @@ -772,7 +782,7 @@ var ReconTablesCfg = TableCfg{ func TablesCfgByLabel(label Label) TableCfg { switch label { - case ChainDB: + case ChainDB, InMem: return ChaindataTablesCfg case TxPoolDB: return TxpoolTablesCfg @@ -782,6 +792,12 @@ func TablesCfgByLabel(label Label) TableCfg { return DownloaderTablesCfg case DiagnosticsDB: return DiagnosticsTablesCfg + case HeimdallDB: + return HeimdallTablesCfg + case PolygonBridgeDB: + return PolygonBridgeTablesCfg + case ConsensusDB: + return ConsensusTablesCfg default: panic(fmt.Sprintf("unexpected label: %s", label)) } @@ -830,6 +846,13 @@ func reinit() { } } + for _, name := range ConsensusTables { + _, ok := ConsensusTablesCfg[name] + if !ok { + ConsensusTablesCfg[name] = TableCfgItem{} + } + } + for _, name := range DownloaderTables { _, ok := DownloaderTablesCfg[name] if !ok { @@ -850,6 +873,19 @@ func reinit() { DiagnosticsTablesCfg[name] = TableCfgItem{} } } + + for _, name := range HeimdallTables { + _, ok := HeimdallTablesCfg[name] + if !ok { + HeimdallTablesCfg[name] = TableCfgItem{} + } + } + for _, name := range PolygonBridgeTables { + _, ok := PolygonBridgeTablesCfg[name] + if !ok { + PolygonBridgeTablesCfg[name] = TableCfgItem{} + } + } } // Temporal diff --git a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go index 0076d15daac..5c2df633b9e 100644 --- a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go +++ b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go @@ -36,9 +36,9 @@ func NewTestDB(tb testing.TB, dirs datadir.Dirs) (db kv.RwDB, agg *state.Aggrega } if tb != nil { - db = memdb.NewTestDB(tb) + db = memdb.NewTestDB(tb, kv.ChainDB) } else { - db = memdb.New(dirs.DataDir) + db = memdb.New(dirs.DataDir, kv.ChainDB) } var err error diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 4a8e2a505d2..bb6f19ed213 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -22,7 +22,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/erigontech/erigon-lib/commitment" "math" "math/rand" "os" @@ -33,6 +32,8 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/commitment" + "github.com/erigontech/erigon-lib/common/background" "github.com/c2h5oh/datasize" diff --git a/erigon-lib/txpool/fetch_test.go b/erigon-lib/txpool/fetch_test.go index a33308e56ef..ea3d7b4d947 100644 --- a/erigon-lib/txpool/fetch_test.go +++ b/erigon-lib/txpool/fetch_test.go @@ -230,7 +230,7 @@ func decodeHex(in string) []byte { func TestOnNewBlock(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - coreDB, db := memdb.NewTestDB(t), memdb.NewTestDB(t) + coreDB, db := memdb.NewTestDB(t, kv.ChainDB), memdb.NewTestDB(t, kv.TxPoolDB) ctrl := gomock.NewController(t) stream := remote.NewMockKV_StateChangesClient(ctrl) diff --git a/eth/stagedsync/stagedsynctest/harness.go b/eth/stagedsync/stagedsynctest/harness.go index bd958694086..e6efef2ab0f 100644 --- a/eth/stagedsync/stagedsynctest/harness.go +++ b/eth/stagedsync/stagedsynctest/harness.go @@ -61,7 +61,7 @@ func InitHarness(ctx context.Context, t *testing.T, cfg HarnessCfg) Harness { m := mock.MockWithGenesis(t, genesisInit.genesis, genesisInit.genesisAllocPrivateKey, false) chainDataDB := m.DB blockReader := m.BlockReader - borConsensusDB := memdb.NewTestDB(t) + borConsensusDB := memdb.NewTestDB(t, kv.ChainDB) ctrl := gomock.NewController(t) heimdallClient := heimdall.NewMockHeimdallClient(ctrl) miningState := stagedsync.NewMiningState(ðconfig.Defaults.Miner) diff --git a/migrations/migrations_test.go b/migrations/migrations_test.go index 5ff59dd8eb1..48bad188140 100644 --- a/migrations/migrations_test.go +++ b/migrations/migrations_test.go @@ -31,7 +31,7 @@ import ( ) func TestApplyWithInit(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { "one", @@ -96,7 +96,7 @@ func TestApplyWithInit(t *testing.T) { } func TestApplyWithoutInit(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { "one", @@ -161,7 +161,7 @@ func TestApplyWithoutInit(t *testing.T) { } func TestWhenNonFirstMigrationAlreadyApplied(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { "one", @@ -243,7 +243,7 @@ func TestMarshalStages(t *testing.T) { } func TestValidation(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { Name: "repeated_name", @@ -293,7 +293,7 @@ func TestValidation(t *testing.T) { } func TestCommitCallRequired(t *testing.T) { - require, db := require.New(t), memdb.NewTestDB(t) + require, db := require.New(t), memdb.NewTestDB(t, kv.ChainDB) m := []Migration{ { Name: "one", diff --git a/node/node.go b/node/node.go index 2116c91bec5..3e29ffc5b8a 100644 --- a/node/node.go +++ b/node/node.go @@ -31,13 +31,13 @@ import ( "github.com/c2h5oh/datasize" "github.com/erigontech/erigon-lib/common/dbg" + "github.com/erigontech/erigon/params" "golang.org/x/sync/semaphore" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon/cmd/utils" "github.com/erigontech/erigon/node/nodecfg" - "github.com/erigontech/erigon/params" "github.com/erigontech/erigon/turbo/debug" "github.com/gofrs/flock" @@ -316,7 +316,7 @@ func OpenDatabase(ctx context.Context, config *nodecfg.Config, label kv.Label, n var db kv.RwDB if config.Dirs.DataDir == "" { - db = memdb.New("") + db = memdb.New("", label) return db, nil } @@ -377,38 +377,38 @@ func OpenDatabase(ctx context.Context, config *nodecfg.Config, label kv.Label, n return nil, err } - migrator := migrations.NewMigrator(label) - if err := migrator.VerifyVersion(db, dbPath); err != nil { - return nil, err - } - - has, err := migrator.HasPendingMigrations(db) - if err != nil { - return nil, err - } - if has && !dbg.OnlyCreateDB { - logger.Info("Re-Opening DB in exclusive mode to apply migrations") - db.Close() - db, err = openFunc(true) - if err != nil { + if label == kv.ChainDB { + migrator := migrations.NewMigrator(label) + if err := migrator.VerifyVersion(db, dbPath); err != nil { return nil, err } - if err = migrator.Apply(db, config.Dirs.DataDir, dbPath, logger); err != nil { + has, err := migrator.HasPendingMigrations(db) + if err != nil { return nil, err } - db.Close() - db, err = openFunc(false) - if err != nil { + if has && !dbg.OnlyCreateDB { + logger.Info("Re-Opening DB in exclusive mode to apply migrations") + db.Close() + db, err = openFunc(true) + if err != nil { + return nil, err + } + if err = migrator.Apply(db, config.Dirs.DataDir, dbPath, logger); err != nil { + return nil, err + } + db.Close() + db, err = openFunc(false) + if err != nil { + return nil, err + } + } + if err := db.Update(context.Background(), func(tx kv.RwTx) (err error) { + return params.SetErigonVersion(tx, params.VersionKeyCreated) + }); err != nil { return nil, err } } - if err := db.Update(context.Background(), func(tx kv.RwTx) (err error) { - return params.SetErigonVersion(tx, params.VersionKeyCreated) - }); err != nil { - return nil, err - } - return db, nil } diff --git a/node/node_test.go b/node/node_test.go index eef9dd75dd3..25f51f820a2 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -156,14 +156,14 @@ func TestNodeCloseClosesDB(t *testing.T) { t.Fatal("can't open DB:", err) } if err = db.Update(context.Background(), func(tx kv.RwTx) error { - return tx.Put(kv.HashedAccounts, []byte("testK"), []byte{}) + return tx.Put(kv.Inodes, []byte("testK"), []byte{}) }); err != nil { t.Fatal("can't Put on open DB:", err) } stack.Close() //if err = db.Update(context.Background(), func(tx kv.RwTx) error { - // return tx.Put(kv.HashedAccounts, []byte("testK"), []byte{}) + // return tx.Put(kv.Inodes, []byte("testK"), []byte{}) //}); err == nil { // t.Fatal("Put succeeded after node is closed") //} diff --git a/polygon/bor/bor_test.go b/polygon/bor/bor_test.go index 3722318b90f..1ab01414bcf 100644 --- a/polygon/bor/bor_test.go +++ b/polygon/bor/bor_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -312,7 +313,7 @@ func newValidator(t *testing.T, heimdall *test_heimdall, blocks map[uint64]*type validatorAddress := crypto.PubkeyToAddress(validatorKey.PublicKey) bor := bor.New( heimdall.chainConfig, - memdb.New(""), + memdb.New("", kv.ChainDB), nil, /* blockReader */ &spanner{ ChainSpanner: bor.NewChainSpanner(borabi.ValidatorSetContractABI(), heimdall.chainConfig, false, logger), diff --git a/polygon/bor/finality/whitelist/service_test.go b/polygon/bor/finality/whitelist/service_test.go index 57012569b66..8b0dcc8fc44 100644 --- a/polygon/bor/finality/whitelist/service_test.go +++ b/polygon/bor/finality/whitelist/service_test.go @@ -65,7 +65,7 @@ func NewMockService(db kv.RwDB) *Service { func TestWhitelistedCheckpoint(t *testing.T) { t.Parallel() - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) //Creating the service for the whitelisting the checkpoints s := NewMockService(db) @@ -116,7 +116,7 @@ func TestWhitelistedCheckpoint(t *testing.T) { func TestMilestone(t *testing.T) { t.Parallel() - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) s := NewMockService(db) @@ -265,7 +265,7 @@ func TestMilestone(t *testing.T) { func TestIsValidChain(t *testing.T) { t.Parallel() - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) s := NewMockService(db) chainA := createMockChain(1, 20) // A1->A2...A19->A20 @@ -478,7 +478,7 @@ func TestIsValidChain(t *testing.T) { } func TestPropertyBasedTestingMilestone(t *testing.T) { - db := memdb.NewTestDB(t) + db := memdb.NewTestDB(t, kv.ChainDB) rapid.Check(t, func(t *rapid.T) { diff --git a/polygon/heimdall/entity_store_mock.go b/polygon/heimdall/entity_store_mock.go index 5ab8e8dc9e8..4ef01910dbd 100644 --- a/polygon/heimdall/entity_store_mock.go +++ b/polygon/heimdall/entity_store_mock.go @@ -456,7 +456,7 @@ func (c *MockEntityStoreDeleteToBlockNumCall[TEntity]) Return(arg0 int, arg1 err } // Do rewrite *gomock.Call.Do -func (c *MockEntityStoreDeleteToBlockNumCall[TEntity]) Do(f func(context.Context, uint64, int) (int,error)) *MockEntityStoreDeleteToBlockNumCall[TEntity] { +func (c *MockEntityStoreDeleteToBlockNumCall[TEntity]) Do(f func(context.Context, uint64, int) (int, error)) *MockEntityStoreDeleteToBlockNumCall[TEntity] { c.Call = c.Call.Do(f) return c } @@ -495,7 +495,7 @@ func (c *MockEntityStoreDeleteFromBlockNumCall[TEntity]) Return(arg0 int, arg1 e } // Do rewrite *gomock.Call.Do -func (c *MockEntityStoreDeleteFromBlockNumCall[TEntity]) Do(f func(context.Context, uint64) (int,error)) *MockEntityStoreDeleteFromBlockNumCall[TEntity] { +func (c *MockEntityStoreDeleteFromBlockNumCall[TEntity]) Do(f func(context.Context, uint64) (int, error)) *MockEntityStoreDeleteFromBlockNumCall[TEntity] { c.Call = c.Call.Do(f) return c } diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 7727767ff12..9c9eec4cd33 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -347,7 +347,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK if err != nil { tb.Fatal(err) } - mock.txPoolDB = memdb.NewPoolDB(tmpdir) + mock.txPoolDB = memdb.NewWithLabel(tmpdir, kv.TxPoolDB) stateChangesClient := direct.NewStateDiffClientDirect(erigonGrpcServeer)