feat(dashmate): state sync configuration for tenderdash and drive snapshots

packages/dashmate/configs/defaults/getBaseConfigFactory.js

@@ -356,6 +356,13 @@ export default function getBaseConfigFactory() {
               txProcessingTimeLimit: null,
             },
             epochTime: 788400,
+            stateSync: {
+              snapshots: {
+                enabled: true,
+                frequencySeconds: 600,
+                maxCount: 6,
+              },
+            },
           },
           tenderdash: {
             mode: 'full',
@@ -460,6 +467,16 @@ export default function getBaseConfigFactory() {
               },
             },
             moniker: null,
+            // Serving snapshots to peers is always on in Tenderdash; `enabled`
+            // only makes a fresh node bootstrap from a snapshot, and Tenderdash
+            // ignores it once the node has local state, so it is safe on by
+            // default for existing nodes.
+            stateSync: {
+              enabled: true,
+              retries: 3,
+              chunkRequestTimeout: '15s',
+              fetchersCount: 4,
+            },
           },
         },
         sourcePath: null,

packages/dashmate/configs/defaults/getLocalConfigFactory.js

@@ -105,6 +105,11 @@ export default function getLocalConfigFactory(getBaseConfig) {
             metrics: {
               port: 46660,
             },
+            // A local network genesis starts every node from scratch at the
+            // same time, so there is no populated peer to state sync from.
+            stateSync: {
+              enabled: false,
+            },
           },
           abci: {
             tokioConsole: {
@@ -141,6 +146,11 @@ export default function getLocalConfigFactory(getBaseConfig) {
                 rotation: false,
               },
             },
+            stateSync: {
+              snapshots: {
+                enabled: false,
+              },
+            },
           },
         },
       },

packages/dashmate/configs/getConfigFileMigrationsFactory.js

@@ -1764,6 +1764,23 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs)
 
         return configFile;
       },
+      '4.2.0-dev.6': (configFile) => {
+        // State sync options are required by the schema now. Pulled from the
+        // default config matching each config's name or group, so the local
+        // preset gets its disables while everything else gets the base
+        // defaults (consume and serve snapshots).
+        Object.entries(configFile.configs)
+          .forEach(([name, options]) => {
+            const defaultConfig = getDefaultConfigByNameOrGroup(name, options.group);
+
+            options.platform.drive.tenderdash.stateSync = defaultConfig
+              .getStored('platform.drive.tenderdash.stateSync');
+            options.platform.drive.abci.stateSync = defaultConfig
+              .getStored('platform.drive.abci.stateSync');
+          });
+
+        return configFile;
+      },
     };
   }
 

packages/dashmate/docker-compose.yml

@@ -96,6 +96,11 @@ services:
       - GROVEDB_VISUALIZER_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_GROVEDB_VISUALIZER_PORT:?err}
       - PROPOSER_TX_PROCESSING_TIME_LIMIT=${PLATFORM_DRIVE_ABCI_PROPOSER_TX_PROCESSING_TIME_LIMIT}
       - NETWORK=${NETWORK:?err}
+      # Checkpoints live in the default CHECKPOINTS_PATH (DB_PATH/checkpoints),
+      # inside the drive_abci_data volume
+      - SNAPSHOTS_ENABLED=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_ENABLED:?err}
+      - SNAPSHOTS_FREQUENCY_SECONDS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_FREQUENCY_SECONDS:?err}
+      - MAX_NUM_SNAPSHOTS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_MAX_COUNT:?err}
     stop_grace_period: 30s
     expose:
       - 26658

packages/dashmate/docs/config/drive-abci.md

@@ -124,6 +124,18 @@ These settings control developer and debugging tools:
 - Tokio Console: A debugging tool for Rust's async runtime
 - GroveDB Visualizer: A visualization tool for the GroveDB database structure
 
+## State Sync Snapshots
+
+These settings control the serving side of state sync: Drive periodically takes GroveDB checkpoints and hands them to Tenderdash, which offers them to peers bootstrapping via state sync. The consuming side is configured on Tenderdash (see [Tenderdash configuration](./tenderdash.md#state-sync)).
+
+| Option | Description | Default | Example |
+|--------|-------------|---------|---------|
+| `platform.drive.abci.stateSync.snapshots.enabled` | Take and serve state sync snapshots | `true` | `false` |
+| `platform.drive.abci.stateSync.snapshots.frequencySeconds` | How often to take a snapshot, in seconds, at least 60 | `600` | `3600` |
+| `platform.drive.abci.stateSync.snapshots.maxCount` | Snapshots kept before pruning the oldest, at least 2 | `6` | `10` |
+
+Checkpoints are stored inside the Drive data volume under `db/checkpoints`. They hard-link unchanged data, so keeping several costs only a fraction of the database size. The local preset disables snapshots along with state sync consumption.
+
 ## Other options
 
 | Option | Description | Default | Example |

packages/dashmate/docs/config/tenderdash.md

@@ -80,6 +80,21 @@ The RPC interface is used for:
 - Submitting transactions
 - Fetching network status
 
+## State Sync
+
+State sync bootstraps a fresh node from a recent state snapshot fetched from peers instead of replaying every block. These settings control the consuming side; serving snapshots to peers is always on in Tenderdash and is fed by Drive's snapshots (see [Drive ABCI configuration](./drive-abci.md#state-sync-snapshots)).
+
+| Option | Description | Default | Example |
+|--------|-------------|---------|---------|
+| `platform.drive.tenderdash.stateSync.enabled` | Bootstrap a fresh node from a snapshot | `true` | `false` |
+| `platform.drive.tenderdash.stateSync.retries` | Retries before falling back to block sync, `0` disables retries | `3` | `5` |
+| `platform.drive.tenderdash.stateSync.chunkRequestTimeout` | Timeout before re-requesting a snapshot chunk, at least `5s` | `15s` | `30s` |
+| `platform.drive.tenderdash.stateSync.fetchersCount` | Concurrent chunk fetchers, 1 to 64 | `4` | `8` |
+
+- Enabling is safe for existing nodes: Tenderdash only attempts state sync when the node has no local state and disables it by itself otherwise. Only a freshly set up (or reset) node consumes a snapshot, and it ends up with a truncated block history starting at the snapshot height.
+- Snapshots are verified through the P2P layer. The alternative RPC state provider needs at least two reachable Tenderdash RPC servers, but dashmate publishes the Tenderdash RPC on loopback only, without TLS and unproxied by the gateway, so the rendered config hardcodes `use-p2p = true`.
+- The local preset disables state sync: a local network genesis starts every node from scratch, so there is no populated peer to sync from.
+
 ## Metrics and Profiling
 
 These settings control monitoring and profiling tools:

packages/dashmate/src/config/configJsonSchema.js

@@ -1087,9 +1087,37 @@ export default {
                   required: ['txProcessingTimeLimit'],
                   additionalProperties: false,
                 },
+                stateSync: {
+                  type: 'object',
+                  properties: {
+                    snapshots: {
+                      type: 'object',
+                      properties: {
+                        enabled: {
+                          type: 'boolean',
+                          description: 'Take state sync snapshots (GroveDB checkpoints) and serve them to peers',
+                        },
+                        frequencySeconds: {
+                          type: 'integer',
+                          minimum: 60,
+                          description: 'How often to take a snapshot, in seconds',
+                        },
+                        maxCount: {
+                          type: 'integer',
+                          minimum: 2,
+                          description: 'How many snapshots to keep before pruning the oldest',
+                        },
+                      },
+                      required: ['enabled', 'frequencySeconds', 'maxCount'],
+                      additionalProperties: false,
+                    },
+                  },
+                  required: ['snapshots'],
+                  additionalProperties: false,
+                },
               },
               additionalProperties: false,
-              required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime', 'metrics', 'grovedbVisualizer', 'proposer'],
+              required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime', 'metrics', 'grovedbVisualizer', 'proposer', 'stateSync'],
             },
             tenderdash: {
               type: 'object',
@@ -1337,8 +1365,45 @@ export default {
                 genesis: {
                   type: 'object',
                 },
+                stateSync: {
+                  type: 'object',
+                  properties: {
+                    enabled: {
+                      type: 'boolean',
+                      description: 'Bootstrap a fresh node from a state sync snapshot instead of replaying'
+                        + ' all blocks. Ignored once the node has local state',
+                    },
+                    retries: {
+                      type: 'integer',
+                      minimum: 0,
+                      description: 'How many times to retry state sync before falling back to block sync.'
+                        + ' 0 disables retries',
+                    },
+                    chunkRequestTimeout: {
+                      description: 'Timeout before re-requesting a snapshot chunk. Tenderdash requires at least 5s',
+                      allOf: [
+                        {
+                          $ref: '#/definitions/duration',
+                        },
+                        {
+                          type: 'string',
+                          // At least 5 seconds: 5s+, 5000ms+, or minutes/hours down to 0.1
+                          pattern: '^(([5-9]|[1-9][0-9]+)(\\.[0-9]+)?s|([5-9][0-9]{3}|[1-9][0-9]{4,})(\\.[0-9]+)?ms|([1-9][0-9]*(\\.[0-9]+)?|0\\.[1-9][0-9]*)[mh])$',
+                        },
+                      ],
+                    },
+                    fetchersCount: {
+                      type: 'integer',
+                      minimum: 1,
+                      maximum: 64,
+                      description: 'Number of concurrent snapshot chunk fetchers',
+                    },
+                  },
+                  required: ['enabled', 'retries', 'chunkRequestTimeout', 'fetchersCount'],
+                  additionalProperties: false,
+                },
               },
-              required: ['mode', 'docker', 'p2p', 'mempool', 'consensus', 'log', 'rpc', 'pprof', 'node', 'moniker', 'genesis', 'metrics'],
+              required: ['mode', 'docker', 'p2p', 'mempool', 'consensus', 'log', 'rpc', 'pprof', 'node', 'moniker', 'genesis', 'metrics', 'stateSync'],
               additionalProperties: false,
             },
           },

packages/dashmate/src/doctor/analyse/analyseSystemResourcesFactory.js

@@ -22,6 +22,14 @@ export default function analyseSystemResourcesFactory(verifySystemRequirements)
       diskIO,
     } = samples.getSystemInfo();
 
+    let stateSyncSnapshotsEnabled = false;
+    try {
+      stateSyncSnapshotsEnabled = samples.getDashmateConfig()
+        .get('platform.drive.abci.stateSync.snapshots.enabled') === true;
+    } catch {
+      // A config collected by an older dashmate has no state sync options
+    }
+
     // System requirements
     const problems = verifySystemRequirements(
       {
@@ -33,6 +41,7 @@ export default function analyseSystemResourcesFactory(verifySystemRequirements)
       samples.getDashmateConfig().get('platform.enable'),
       {
         diskSpace: 5,
+        stateSyncSnapshotsEnabled,
       },
     );
 

packages/dashmate/src/doctor/verifySystemRequirementsFactory.js

@@ -15,6 +15,7 @@ export default function verifySystemRequirementsFactory() {
    * @param {boolean} isHP
    * @param {Object} [overrideRequirements]
    * @param {Number} [overrideRequirements.diskSpace]
+   * @param {boolean} [overrideRequirements.stateSyncSnapshotsEnabled]
    * @returns {Problem[]}
    */
   function verifySystemRequirements(
@@ -30,7 +31,12 @@ export default function verifySystemRequirementsFactory() {
     const MINIMUM_CPU_CORES = isHP ? 4 : 2;
     const MINIMUM_CPU_FREQUENCY = 2.4; // GHz
     const MINIMUM_RAM = isHP ? 7.3 : 3.6; // GB
-    const MINIMUM_DISK_SPACE = overrideRequirements.diskSpace ?? (isHP ? 200 : 100); // GB
+
+    // State sync snapshots are GroveDB checkpoints stored next to the database.
+    // They share unchanged data with it, so a small fixed headroom is enough.
+    const SNAPSHOTS_DISK_HEADROOM = overrideRequirements.stateSyncSnapshotsEnabled ? 10 : 0; // GB
+    const BASE_MINIMUM_DISK_SPACE = overrideRequirements.diskSpace ?? (isHP ? 200 : 100); // GB
+    const MINIMUM_DISK_SPACE = BASE_MINIMUM_DISK_SPACE + SNAPSHOTS_DISK_HEADROOM; // GB
 
     const problems = [];
 
@@ -112,11 +118,17 @@ for required network services and avoid Proof-of-Service bans`,
       const availableDiskSpace = diskSpace.available / (1024 ** 3); // Convert to GB
 
       if (availableDiskSpace < MINIMUM_DISK_SPACE) {
+        const headroomNote = SNAPSHOTS_DISK_HEADROOM > 0
+          ? ` (including ${SNAPSHOTS_DISK_HEADROOM}GB headroom for state sync snapshots)`
+          : '';
+
         const problem = new Problem(
-          `${availableDiskSpace.toFixed(2)}GB of available disk space detected. At least ${MINIMUM_DISK_SPACE}GB is required`,
+          `${availableDiskSpace.toFixed(2)}GB of available disk space detected. At least ${MINIMUM_DISK_SPACE}GB is required${headroomNote}`,
           `Consider increasing disk space to make sure the node can provide timely responses
 for required network services and avoid Proof-of-Service bans`,
-          MINIMUM_DISK_SPACE - availableDiskSpace < 5 ? SEVERITY.HIGH : SEVERITY.MEDIUM,
+          // Judged against the base minimum so that enabling snapshots can
+          // widen when a problem is raised but never downgrade its severity
+          BASE_MINIMUM_DISK_SPACE - availableDiskSpace < 5 ? SEVERITY.HIGH : SEVERITY.MEDIUM,
         );
 
         problems.push(problem);

packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot

@@ -81,7 +81,7 @@ filter-peers = false
 # Example for routed multi-app setup:
 #   abci = "routed"
 #   address = "Info:socket:unix:///tmp/socket.1,Info:socket:unix:///tmp/socket.2,CheckTx:socket:unix:///tmp/socket.1,*:socket:unix:///tmp/socket.3"
-address = "CheckTx:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658"
+address = "CheckTx:grpc:drive_abci:26670,ListSnapshots:grpc:drive_abci:26670,LoadSnapshotChunk:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658"
 # Transport mechanism to connect to the ABCI application: socket | grpc | routed
 transport = "routed"
 # Maximum number of simultaneous connections to the ABCI application
@@ -97,6 +97,10 @@ transport = "routed"
 #]
 grpc-concurrency = [
   { "check_tx" = {{= it.platform.drive.tenderdash.mempool.maxConcurrentCheckTx }} },
+  # Snapshot serving: discovery is one request per peer, chunk downloads run
+  # several concurrent fetchers per syncing peer.
+  { "list_snapshots" = 10 },
+  { "load_snapshot_chunk" = 100 },
 ]
 
 
@@ -418,40 +422,38 @@ ttl-num-blocks = {{=it.platform.drive.tenderdash.mempool.ttlNumBlocks}}
 # the network to take and serve state machine snapshots. State sync is not attempted if the node
 # has any local state (LastBlockHeight > 0). The node will have a truncated block history,
 # starting from the height of the snapshot.
-enable = false
+enable = {{? it.platform.drive.tenderdash.stateSync.enabled }}true{{??}}false{{?}}
 
 # State sync uses light client verification to verify state. This can be done either through the
-# P2P layer or RPC layer. Set this to true to use the P2P layer. If false (default), RPC layer
-# will be used.
-use-p2p = false
+# P2P layer or RPC layer. Set this to true to use the P2P layer.
+# Hardcoded to P2P: the RPC mode needs at least two reachable RPC servers, but dashmate
+# publishes the Tenderdash RPC on loopback only and does not proxy it through the gateway
+# (no TLS or auth), so the RPC state provider is not viable here.
+use-p2p = true
 
 # If using RPC, at least two addresses need to be provided. They should be compatible with net.Dial,
 # for example: "host.example.com:2125"
 rpc-servers = ""
 
-# The hash and height of a trusted block. Must be within the trust-period.
-trust-height = 0
-trust-hash = ""
-
-# The trust period should be set so that Tendermint can detect and gossip misbehavior before
-# it is considered expired. For chains based on the Cosmos SDK, one day less than the unbonding
-# period should suffice.
-trust-period = "168h0m0s"
-
 # Time to spend discovering snapshots before initiating a restore.
 discovery-time = "15s"
 
+# The number of times to retry state sync. When retries are exhausted, the node falls back
+# to block sync. Set to 0 to disable retries. In the pessimistic case it takes at least
+# discovery-time * retries before falling back.
+retries = {{= it.platform.drive.tenderdash.stateSync.retries }}
+
 # Temporary directory for state sync snapshot chunks, defaults to os.TempDir().
 # The synchronizer will create a new, randomly named directory within this directory
 # and remove it when the sync is complete.
 temp-dir = ""
 
 # The timeout duration before re-requesting a chunk, possibly from a different
 # peer (default: 15 seconds).
-chunk-request-timeout = "15s"
+chunk-request-timeout = "{{= it.platform.drive.tenderdash.stateSync.chunkRequestTimeout }}"
 
 # The number of concurrent chunk and block fetchers to run (default: 4).
-fetchers = "4"
+fetchers = "{{= it.platform.drive.tenderdash.stateSync.fetchersCount }}"
 
 #######################################################
 ###         Consensus Configuration Options         ###