Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip] mdbx: use lable to defautl table cfg detection #12338

Open
wants to merge 33 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 30 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
b4fc88d
save
AskAlexSharov Oct 16, 2024
3e03d7b
save
AskAlexSharov Oct 16, 2024
aba9881
save
AskAlexSharov Oct 16, 2024
d313b68
save
AskAlexSharov Oct 16, 2024
0c839a8
save
AskAlexSharov Oct 17, 2024
96e4580
save
AskAlexSharov Oct 17, 2024
084c150
save
AskAlexSharov Oct 17, 2024
52ad370
save
AskAlexSharov Oct 18, 2024
8234a29
save
AskAlexSharov Oct 18, 2024
2292b27
save
AskAlexSharov Oct 18, 2024
5c83d51
save
AskAlexSharov Oct 18, 2024
d6f51b4
save
AskAlexSharov Oct 18, 2024
c61717a
save
AskAlexSharov Oct 18, 2024
572b34c
save
AskAlexSharov Oct 18, 2024
ddade0c
save
AskAlexSharov Oct 18, 2024
8eb1bbf
save
AskAlexSharov Oct 18, 2024
db4cea4
save
AskAlexSharov Oct 18, 2024
7e0fbfb
save
AskAlexSharov Oct 18, 2024
8e9a4d8
save
AskAlexSharov Oct 18, 2024
7899a1f
save
AskAlexSharov Oct 18, 2024
13fc59c
save
AskAlexSharov Oct 18, 2024
a653f98
save
AskAlexSharov Oct 18, 2024
bc7ace3
save
AskAlexSharov Oct 19, 2024
331deac
save
AskAlexSharov Oct 19, 2024
ec77126
save
AskAlexSharov Oct 20, 2024
6f608b6
save
AskAlexSharov Oct 20, 2024
7888359
Merge branch 'main' into mdbx_use_lable_table_cfg
AskAlexSharov Oct 20, 2024
c4d5129
save
AskAlexSharov Oct 20, 2024
50eab09
Merge branch 'main' into mdbx_use_lable_table_cfg
AskAlexSharov Oct 21, 2024
7951c5c
Merge branch 'main' into mdbx_use_lable_table_cfg
AskAlexSharov Oct 22, 2024
90ab15d
save
AskAlexSharov Oct 24, 2024
1a01fc5
Merge branch 'main' into mdbx_use_lable_table_cfg
AskAlexSharov Oct 24, 2024
3701163
save
AskAlexSharov Oct 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cl/antiquary/state_antiquary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
_ "embed"
"testing"

"github.com/erigontech/erigon-lib/kv"
"github.com/stretchr/testify/require"

"github.com/erigontech/erigon-lib/common/datadir"
Expand All @@ -36,7 +37,7 @@ import (
)

func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) {
db := memdb.NewTestDB(t)
db := memdb.NewTestDB(t, kv.ChainDB)
reader := tests.LoadChain(blocks, postState, db, t)

ctx := context.Background()
Expand Down
4 changes: 2 additions & 2 deletions cl/beacon/handler/utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge
blocks, preState, postState = tests.GetCapellaRandom()
}
fcu = mock_services2.NewForkChoiceStorageMock(t)
db = memdb.NewTestDB(t)
blobDb := memdb.NewTestDB(t)
db = memdb.NewTestDB(t, kv.ChainDB)
blobDb := memdb.NewTestDB(t, kv.ChainDB)
var reader *tests.MockBlockReader
reader = tests.LoadChain(blocks, postState, db, t)
firstBlockRoot, _ := blocks[0].Block.HashSSZ()
Expand Down
2 changes: 1 addition & 1 deletion cl/persistence/beacon_indicies/indicies_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (

func setupTestDB(t *testing.T) kv.RwDB {
// Create an in-memory SQLite DB for testing purposes
db := memdb.NewTestDB(t)
db := memdb.NewTestDB(t, kv.ChainDB)
return db
}

Expand Down
2 changes: 1 addition & 1 deletion cl/persistence/blob_storage/blob_db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
)

func setupTestDB(t *testing.T) kv.RwDB {
db := memdb.NewTestDB(t)
db := memdb.NewTestDB(t, kv.ChainDB)
return db
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"testing"

"github.com/erigontech/erigon-lib/kv"
"github.com/stretchr/testify/require"

libcommon "github.com/erigontech/erigon-lib/common"
Expand All @@ -36,7 +37,7 @@ import (
)

func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) {
db := memdb.NewTestDB(t)
db := memdb.NewTestDB(t, kv.ChainDB)
reader := tests.LoadChain(blocks, postState, db, t)

ctx := context.Background()
Expand Down
3 changes: 2 additions & 1 deletion cl/phase1/network/services/block_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"testing"

"github.com/erigontech/erigon-lib/kv"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"

Expand All @@ -34,7 +35,7 @@ import (
)

func setupBlockService(t *testing.T, ctrl *gomock.Controller) (BlockService, *synced_data.SyncedDataManager, *eth_clock.MockEthereumClock, *mock_services.ForkChoiceStorageMock) {
db := memdb.NewTestDB(t)
db := memdb.NewTestDB(t, kv.ChainDB)
cfg := &clparams.MainnetBeaconConfig
syncedDataManager := synced_data.NewSyncedDataManager(true, cfg)
ethClock := eth_clock.NewMockEthereumClock(ctrl)
Expand Down
9 changes: 0 additions & 9 deletions cl/phase1/stages/stage_history_download.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (

libcommon "github.com/erigontech/erigon-lib/common"
"github.com/erigontech/erigon-lib/kv"
"github.com/erigontech/erigon-lib/kv/memdb"
"github.com/erigontech/erigon-lib/log/v3"
"github.com/erigontech/erigon/cl/antiquary"
"github.com/erigontech/erigon/cl/persistence/beacon_indicies"
Expand Down Expand Up @@ -265,14 +264,6 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co
}
cfg.downloader.SetThrottle(cfg.backfillingThrottling) // throttle to 0.6 second for backfilling
cfg.downloader.SetNeverSkip(false)
// If i do not give it a database, erigon lib starts to cry uncontrollably
db2 := memdb.New(cfg.tmpdir)
defer db2.Close()
tx2, err := db2.BeginRw(ctx)
if err != nil {
return err
}
defer tx2.Rollback()
isBackfilling.Store(true)

cfg.logger.Info("Ready to insert history, waiting for sync cycle to finish")
Expand Down
5 changes: 3 additions & 2 deletions cl/sentinel/handlers/blobs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"math"
"testing"

"github.com/erigontech/erigon-lib/kv"
"github.com/golang/snappy"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/peer"
Expand Down Expand Up @@ -88,7 +89,7 @@ func TestBlobsByRangeHandler(t *testing.T) {
require.NoError(t, err)

peersPool := peers.NewPool()
blobDb := memdb.NewTestDB(t)
blobDb := memdb.NewTestDB(t, kv.ChainDB)
_, indiciesDB := setupStore(t)
store := tests.NewMockBlockReader()

Expand Down Expand Up @@ -209,7 +210,7 @@ func TestBlobsByIdentifiersHandler(t *testing.T) {
require.NoError(t, err)

peersPool := peers.NewPool()
blobDb := memdb.NewTestDB(t)
blobDb := memdb.NewTestDB(t, kv.ChainDB)
_, indiciesDB := setupStore(t)
store := tests.NewMockBlockReader()

Expand Down
2 changes: 1 addition & 1 deletion cl/sentinel/handlers/utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import (
)

func setupStore(t *testing.T) (freezeblocks.BeaconSnapshotReader, kv.RwDB) {
db := memdb.NewTestDB(t)
db := memdb.NewTestDB(t, kv.ChainDB)
return tests.NewMockBlockReader(), db
}

Expand Down
2 changes: 1 addition & 1 deletion cl/sentinel/sentinel_requests_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ import (

func loadChain(t *testing.T) (db kv.RwDB, blocks []*cltypes.SignedBeaconBlock, f afero.Fs, preState, postState *state.CachingBeaconState, reader *tests.MockBlockReader) {
blocks, preState, postState = tests.GetPhase0Random()
db = memdb.NewTestDB(t)
db = memdb.NewTestDB(t, kv.ChainDB)
reader = tests.LoadChain(blocks, postState, db, t)

ctx := context.Background()
Expand Down
3 changes: 2 additions & 1 deletion cl/spectest/consensus_tests/fork_choice.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"math"
"testing"

"github.com/erigontech/erigon-lib/kv"
"github.com/erigontech/erigon/spectest"

"github.com/spf13/afero"
Expand Down Expand Up @@ -203,7 +204,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err
emitters := beaconevents.NewEventEmitter()
_, beaconConfig := clparams.GetConfigsByNetwork(clparams.MainnetNetwork)
ethClock := eth_clock.NewEthereumClock(genesisState.GenesisTime(), genesisState.GenesisValidatorsRoot(), beaconConfig)
blobStorage := blob_storage.NewBlobStore(memdb.New("/tmp"), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock)
blobStorage := blob_storage.NewBlobStore(memdb.New("/tmp", kv.ChainDB), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock)

validatorMonitor := monitor.NewValidatorMonitor(false, nil, nil, nil)
forkStore, err := forkchoice.NewForkChoiceStore(
Expand Down
2 changes: 1 addition & 1 deletion cmd/devnet/services/polygon/proofgenerator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ type requestGenerator struct {
}

func newRequestGenerator(sentry *mock.MockSentry, chain *core.ChainPack) (*requestGenerator, error) {
db := memdb.New("")
db := memdb.New("", kv.ChainDB)
if err := db.Update(context.Background(), func(tx kv.RwTx) error {
if err := rawdb.WriteHeader(tx, chain.TopBlock.Header()); err != nil {
return err
Expand Down
8 changes: 5 additions & 3 deletions cmd/evm/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/erigontech/erigon-lib/config3"
"github.com/erigontech/erigon-lib/kv/temporal"
"io"
"math/big"
"os"
Expand All @@ -34,6 +32,10 @@ import (
"testing"
"time"

"github.com/erigontech/erigon-lib/config3"
"github.com/erigontech/erigon-lib/kv"
"github.com/erigontech/erigon-lib/kv/temporal"

"github.com/erigontech/erigon-lib/common/datadir"

"github.com/holiman/uint256"
Expand Down Expand Up @@ -159,7 +161,7 @@ func runCmd(ctx *cli.Context) error {
} else {
debugLogger = logger.NewStructLogger(logconfig)
}
db := memdb.New(os.TempDir())
db := memdb.New(os.TempDir(), kv.ChainDB)
defer db.Close()
if ctx.String(GenesisFlag.Name) != "" {
gen := readGenesis(ctx.String(GenesisFlag.Name))
Expand Down
32 changes: 16 additions & 16 deletions cmd/integration/commands/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"path/filepath"
"strings"

"github.com/erigontech/erigon/migrations"
"github.com/spf13/cobra"
"golang.org/x/sync/semaphore"

Expand All @@ -33,7 +34,6 @@ import (
kv2 "github.com/erigontech/erigon-lib/kv/mdbx"

"github.com/erigontech/erigon/cmd/utils"
"github.com/erigontech/erigon/migrations"
"github.com/erigontech/erigon/turbo/debug"
"github.com/erigontech/erigon/turbo/logging"
)
Expand Down Expand Up @@ -90,25 +90,25 @@ func dbCfg(label kv.Label, path string) kv2.MdbxOpts {

func openDB(opts kv2.MdbxOpts, applyMigrations bool, logger log.Logger) (kv.RwDB, error) {
db := opts.MustOpen()
if applyMigrations {
migrator := migrations.NewMigrator(opts.GetLabel())
has, err := migrator.HasPendingMigrations(db)
if err != nil {
return nil, err
}
if has {
logger.Info("Re-Opening DB in exclusive mode to apply DB migrations")
db.Close()
db = opts.Exclusive().MustOpen()
if err := migrator.Apply(db, datadirCli, "", logger); err != nil {
if opts.GetLabel() == kv.ChainDB {
if applyMigrations {
migrator := migrations.NewMigrator(opts.GetLabel())
has, err := migrator.HasPendingMigrations(db)
if err != nil {
return nil, err
}
db.Close()
db = opts.MustOpen()
if has {
logger.Info("Re-Opening DB in exclusive mode to apply DB migrations")
db.Close()
db = opts.Exclusive().MustOpen()
if err := migrator.Apply(db, datadirCli, "", logger); err != nil {
return nil, err
}
db.Close()
db = opts.MustOpen()
}
}
}

if opts.GetLabel() == kv.ChainDB {
_, _, agg, _ := allSnapshots(context.Background(), db, logger)
tdb, err := temporal.New(db, agg)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/pics/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ func initialState1() error {
return err
}

emptyKv := memdb.New("")
emptyKv := memdb.New("", kv.ChainDB)
if err = stateDatabaseComparison(emptyKv, m.DB, 0); err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/rpcdaemon/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func main() {
rootCtx, rootCancel := common.RootContext()
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
logger := debug.SetupCobra(cmd, "sentry")
logger := debug.SetupCobra(cmd, "rpcdaemon")
db, backend, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, heimdallReader, err := cli.RemoteServices(ctx, cfg, logger, rootCancel)
if err != nil {
if !errors.Is(err, context.Canceled) {
Expand Down
6 changes: 4 additions & 2 deletions consensus/aura/aura_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package aura_test

import (
"github.com/erigontech/erigon-lib/common/datadir"
"github.com/erigontech/erigon-lib/kv"

"math/big"
"strings"
"testing"
Expand Down Expand Up @@ -48,7 +50,7 @@ func TestEmptyBlock(t *testing.T) {
genesis.Config.TerminalTotalDifficultyPassed = false

chainConfig := genesis.Config
auraDB := memdb.NewTestDB(t)
auraDB := memdb.NewTestDB(t, kv.ChainDB)
engine, err := aura.NewAuRa(chainConfig.Aura, auraDB)
require.NoError(err)
checkStateRoot := true
Expand Down Expand Up @@ -87,7 +89,7 @@ func TestAuRaSkipGasLimit(t *testing.T) {
genesis.Config.Aura.BlockGasLimitContractTransitions = map[uint64]libcommon.Address{0: libcommon.HexToAddress("0x4000000000000000000000000000000000000001")}

chainConfig := genesis.Config
auraDB := memdb.NewTestDB(t)
auraDB := memdb.NewTestDB(t, kv.ChainDB)
engine, err := aura.NewAuRa(chainConfig.Aura, auraDB)
require.NoError(err)
checkStateRoot := true
Expand Down
2 changes: 1 addition & 1 deletion consensus/clique/clique_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ import (
func TestReimportMirroredState(t *testing.T) {
// Initialize a Clique chain with a single signer
var (
cliqueDB = memdb.NewTestDB(t)
cliqueDB = memdb.NewTestDB(t, kv.ConsensusDB)
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
addr = crypto.PubkeyToAddress(key.PublicKey)
engine = clique.New(params.AllCliqueProtocolChanges, params.CliqueSnapshot, cliqueDB, log.New())
Expand Down
2 changes: 1 addition & 1 deletion consensus/clique/snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ func TestClique(t *testing.T) {
Epoch: tt.epoch,
}

cliqueDB := memdb.NewTestDB(t)
cliqueDB := memdb.NewTestDB(t, kv.ConsensusDB)

engine := clique.New(&config, params.CliqueSnapshot, cliqueDB, log.New())
engine.FakeDiff = true
Expand Down
22 changes: 6 additions & 16 deletions erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ package commitment
import (
"bytes"
"context"
"encoding/binary"
"encoding/hex"
"math"
"math/rand"
Expand Down Expand Up @@ -80,32 +79,23 @@ func Fuzz_ProcessUpdate(f *testing.F) {
})
}

// go test -trimpath -v -fuzz=Fuzz_ProcessUpdates_ArbitraryUpdateCount -fuzztime=300s ./commitment
// go test -trimpath -v -fuzz=Fuzz_ProcessUpdates_ArbitraryUpdateCount2 -fuzztime=300s ./commitment

func Fuzz_ProcessUpdates_ArbitraryUpdateCount(f *testing.F) {
ha, _ := hex.DecodeString("0008852883b2850c7a48f4b0eea3ccc4c04e6cb6025e9e8f7db2589c7dae81517c514790cfd6f668903161349e")
func Fuzz_ProcessUpdates_ArbitraryUpdateCount2(f *testing.F) {
//ha, _ := hex.DecodeString("0008852883b2850c7a48f4b0eea3ccc4c04e6cb6025e9e8f7db2589c7dae81517c514790cfd6f668903161349e")
ctx := context.Background()
f.Add(ha)
f.Add(uint16(10_000), uint32(1), uint32(2))

f.Fuzz(func(t *testing.T, build []byte) {
if len(build) < 12 {
t.Skip()
}
i := 0
keysCount := binary.BigEndian.Uint32(build[i : i+4])
i += 4
ks := binary.BigEndian.Uint32(build[i : i+4])
f.Fuzz(func(t *testing.T, keysCount uint16, ks, us uint32) {
keysSeed := rand.New(rand.NewSource(int64(ks)))
i += 4
us := binary.BigEndian.Uint32(build[i : i+4])
updateSeed := rand.New(rand.NewSource(int64(us)))

t.Logf("fuzzing %d keys keysSeed=%d updateSeed=%d", keysCount, ks, us)

plainKeys := make([][]byte, keysCount)
updates := make([]Update, keysCount)

for k := uint32(0); k < keysCount; k++ {
for k := uint16(0); k < keysCount; k++ {

aux := make([]byte, 32)

Expand Down
2 changes: 1 addition & 1 deletion erigon-lib/kv/mdbx/kv_abstract_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ func TestRemoteKvRange(t *testing.T) {
t.Skip("fix me on win please")
}
logger := log.New()
ctx, writeDB := context.Background(), memdb.NewTestDB(t)
ctx, writeDB := context.Background(), memdb.NewTestDB(t, kv.ChainDB)
grpcServer, conn := grpc.NewServer(), bufconn.Listen(1024*1024)
go func() {
kvServer := remotedbserver.NewKvServer(ctx, writeDB, nil, nil, nil, logger)
Expand Down
Loading
Loading