Merge pull request #6798 from spacemeshos/node-split-poc-merge-develop

merge develop to node-split-poc

Author: poszu

CHANGELOG.md

@@ -16,6 +16,21 @@ pointing to the running `node` service.
 
 * [#6625](https://github.com/spacemeshos/go-spacemesh/pull/6625) Add command for starting services
 
+## v1.8.1
+
+### Highlights
+
+This release will enable syncv2 fully to sync ATXs for all nodes. This should greatly reduce the time it takes to sync
+a node from genesis and the amount of data that is needed to be exchanged with the network to keep a node in sync. The
+improvements are enabled automatically after upgrading to this version and no user action is needed.
+
+### Improvements
+
+* [6792](https://github.com/spacemeshos/go-spacemesh/pull/6792) Fixed race condition in `FPTree` implementation (sync v2).
+
+* [6794](https://github.com/spacemeshos/go-spacemesh/pull/6794) Increased database connection idle timeout,
+  which should reduce occurances of `SQLITE_BUSY` errors.
+
 ## v1.8.0
 
 ### Highlights
@@ -34,6 +49,9 @@ in terms of data size and more easily extendable in the future.
 * [#6779](https://github.com/spacemeshos/go-spacemesh/pull/6779) Fixed the malfeasance publisher not emitting events.
   This caused the node to miss malfeasance proofs when streaming via the GRPC API.
 
+* [#6784](https://github.com/spacemeshos/go-spacemesh/pull/6784) Fixed possible corruptions of DB transactions
+  when a context is cancelled mid transaction (sync v2).
+
 ## v1.7.17
 
 ### Improvements

config/mainnet.go

@@ -96,7 +96,7 @@ func MainnetConfig() Config {
 			DatabaseConnections:     32,
 			DatabasePruneInterval:   30 * time.Minute,
 			DatabaseVacuumState:     21,
-			DatabaseConnIdleTimeout: 10 * time.Millisecond,
+			DatabaseConnIdleTimeout: 500 * time.Millisecond,
 			PruneActivesetsFrom:     12, // starting from epoch 13 activesets below 12 will be pruned
 			NetworkHRP:              "sm",
 
@@ -230,7 +230,7 @@ func MainnetConfig() Config {
 			MalSync:                  malsync.DefaultConfig(),
 			ReconcSync: syncer.ReconcSyncConfig{
 				Enable:            true,
-				EnableActiveSync:  false,
+				EnableActiveSync:  true,
 				OldAtxSyncCfg:     oldAtxSyncCfg,
 				NewAtxSyncCfg:     newAtxSyncCfg,
 				ParallelLoadLimit: 10,

config/presets/fastnet.go

@@ -23,6 +23,7 @@ func fastnet() config.Config {
 	conf.BaseConfig.OptFilterThreshold = 90
 	conf.BaseConfig.DatabasePruneInterval = time.Minute
 	conf.BaseConfig.DatabaseConnections = 16
+	conf.BaseConfig.DatabaseConnIdleTimeout = 500 * time.Millisecond
 
 	// set for systest TestEquivocation
 	conf.BaseConfig.MinerGoodAtxsPercent = 50

config/presets/standalone.go

@@ -26,6 +26,7 @@ func standalone() config.Config {
 	conf.BaseConfig.AtxVersions = activation.AtxVersions{
 		types.EpochID(2): types.AtxV2,
 	}
+	conf.BaseConfig.DatabaseConnIdleTimeout = 500 * time.Millisecond
 
 	conf.TIME.Peersync.Disable = true
 	conf.Standalone = true

config/presets/testnet.go

@@ -85,7 +85,7 @@ func testnet() config.Config {
 			DatabaseConnections:          32,
 			DatabaseSizeMeteringInterval: 10 * time.Minute,
 			DatabasePruneInterval:        30 * time.Minute,
-			DatabaseConnIdleTimeout:      10 * time.Millisecond,
+			DatabaseConnIdleTimeout:      500 * time.Millisecond,
 			NetworkHRP:                   "stest",
 
 			LayerDuration:  5 * time.Minute,

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a
 	github.com/prometheus/client_golang v1.21.1
 	github.com/prometheus/client_model v0.6.1
-	github.com/prometheus/common v0.62.0
+	github.com/prometheus/common v0.63.0
 	github.com/quic-go/quic-go v0.50.0
 	github.com/rqlite/sql v0.0.0-20241111133259-a4122fabb196
 	github.com/rs/cors v1.11.1

go.sum

@@ -547,8 +547,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
+github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k=
+github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=

sql/database.go

@@ -162,7 +162,7 @@ func WithForceMigrations(force bool) Opt {
 	}
 }
 
-// WithSchema specifies database schema script.
+// WithDatabaseSchema specifies database schema script.
 func WithDatabaseSchema(schema *Schema) Opt {
 	return func(c *conf) {
 		c.schema = schema

sync2/fptree/fptree.go

@@ -617,6 +617,8 @@ func (ft *FPTree) storeValues() bool {
 // If this function returns true and the tree doesn't store the values, the key may be
 // contained in the tree.
 func (ft *FPTree) CheckKey(k rangesync.KeyBytes) bool {
+	ft.np.lockRead()
+	defer ft.np.unlockRead()
 	// We're unlikely to be able to find a node with the full prefix, but if we can
 	// find a leaf node with matching partial prefix, that's good enough except
 	// that we also need to check the node's fingerprint.

sync2/sqlstore/syncedtable.go

@@ -163,7 +163,7 @@ func (st *SyncedTable) loadMaxRowID(db sql.Executor) (maxRowID int64, err error)
 	return maxRowID, err
 }
 
-// Snaptshot creates a snapshot of the table based on its current max rowid value.
+// Snapshot creates a snapshot of the table based on its current max rowid value.
 func (st *SyncedTable) Snapshot(db sql.Executor) (*SyncedTableSnapshot, error) {
 	maxRowID, err := st.loadMaxRowID(db)
 	if err != nil {