Validator - support partitioning of the slot id space (#890)

* Adds validatorPartitionSize and validatorPartitionIndex config options * adds partitioning options to the validation type * adds partitioning logic to the validator * ignores partitionIndex when partitionSize is either 0 or 1 * clips the partition index to <<partitionIndex mod partitionSize>> * handles negative values for the validation partition index * updates long description of the new validator cli options * makes default partitionSize to be 0 for better backward compatibility * Improving formatting on validator CLI * reactors validation params into a separate type and simplifies validation of validation params * removes suspected duplication * fixes typo in validator CLI help * updates README * Applies review comments - using optionals and range types to handle validation params * Adds initializer to the configFactory for validatorMaxSlots * [Review] update validator CLI description and README * [Review]: renaming validationParams to validationConfig (config) * [Review]: move validationconfig.nim to a higher level (next to validation.nim) * changes backing type of MaxSlots to be int and makes sure slots are validated without limit when maxSlots is set to 0 * adds more end-to-end test for the validator and the groups * fixes typo in README and conf.nim * makes `maxSlotsConstraintRespected` and `shouldValidateSlot` private + updates the tests * fixes public address of the signer account in the marketplace tutorial * applies review comments - removes two tests
codex-storage · Oct 2, 2024 · 5ace105 · 5ace105
1 parent e1a02c8
commit 5ace105
Show file tree

Hide file tree

Showing 7 changed files with 268 additions and 58 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,8 @@
 # Codex Decentralized Durability Engine
 
-> The Codex project aims to create a decentralized durability engine that allows persisting data in p2p networks. In other words, it allows storing files and data with predictable durability guarantees for later retrieval.
+> The Codex project aims to create a decentralized durability engine that
+> allows persisting data in p2p networks. In other words, it allows storing 
+> files and data with predictable durability guarantees for later retrieval.
 
 > WARNING: This project is under active development and is considered pre-alpha.
 
@@ -24,7 +26,8 @@ To build the project, clone it and run:
 make update && make
 ```
 
-The executable will be placed under the `build` directory under the project root.
+The executable will be placed under the `build` directory under the project
+root.
 
 Run the client with:
 
@@ -38,33 +41,42 @@ It is possible to configure a Codex node in several ways:
  2. Env. variable
  3. Config
 
-The order of priority is the same as above: Cli arguments > Env variables > Config file values.
+The order of priority is the same as above:
+Cli arguments > Env variables > Config file values.
 
 ### Environment variables
 
-In order to set a configuration option using environment variables, first find the desired CLI option
-and then transform it in the following way:
+In order to set a configuration option using environment variables,
+first find the desired CLI option and then transform it in the following way:
 
  1. prepend it with `CODEX_`
  2. make it uppercase
  3. replace `-` with `_`
 
-For example, to configure `--log-level`, use `CODEX_LOG_LEVEL` as the environment variable name.
+For example, to configure `--log-level`, use `CODEX_LOG_LEVEL` as the
+environment variable name.
 
 ### Configuration file
 
-A [TOML](https://toml.io/en/) configuration file can also be used to set configuration values. Configuration option names and corresponding values are placed in the file, separated by `=`. Configuration option names can be obtained from the `codex --help` command, and should not include the `--` prefix. For example, a node's log level (`--log-level`) can be configured using TOML as follows:
+A [TOML](https://toml.io/en/) configuration file can also be used to set
+configuration values. Configuration option names and corresponding values are
+placed in the file, separated by `=`. Configuration option names can be
+obtained from the `codex --help` command, and should not include
+the `--` prefix. For example, a node's log level (`--log-level`) can be
+configured using TOML as follows:
 
 ```toml
 log-level = "trace"
 ```
 
-The Codex node can then read the configuration from this file using the `--config-file` CLI parameter, like `codex --config-file=/path/to/your/config.toml`.
+The Codex node can then read the configuration from this file using
+the `--config-file` CLI parameter, like
+`codex --config-file=/path/to/your/config.toml`.
 
 ### CLI Options
 
 ```
-build/codex --help
+$ build/codex --help
 Usage:
 
 codex [OPTIONS]... command
@@ -87,8 +99,11 @@ The following options are available:
      --agent-string         Node agent string which is used as identifier in network [=Codex].
      --api-bindaddr         The REST API bind address [=127.0.0.1].
  -p, --api-port             The REST Api port [=8080].
-     --repo-kind            Backend for main repo store (fs, sqlite) [=fs].
- -q, --storage-quota        The size of the total storage quota dedicated to the node [=8589934592].
+     --api-cors-origin      The REST Api CORS allowed origin for downloading data. '*' will allow all
+                            origins, '' will allow none. [=Disallow all cross origin requests to download
+                            data].
+     --repo-kind            Backend for main repo store (fs, sqlite, leveldb) [=fs].
+ -q, --storage-quota        The size of the total storage quota dedicated to the node [=$DefaultQuotaBytes].
  -t, --block-ttl            Default block timeout in seconds - 0 disables the ttl [=$DefaultBlockTtl].
      --block-mi             Time interval in seconds - determines frequency of block maintenance cycle: how
                             often blocks are checked for expiration and cleanup
@@ -109,6 +124,18 @@ The following options are available:
      --marketplace-address  Address of deployed Marketplace contract.
      --validator            Enables validator, requires an Ethereum node [=false].
      --validator-max-slots  Maximum number of slots that the validator monitors [=1000].
+                            If set to 0, the validator will not limit the maximum number of slots it
+                            monitors.
+     --validator-groups     Slot validation groups [=ValidationGroups.none].
+                            A number indicating total number of groups into which the whole slot id space
+                            will be divided. The value must be in the range [2, 65535]. If not provided, the
+                            validator will observe the whole slot id space and the value of the
+                            --validator-group-index parameter will be ignored. Powers of twos are advised
+                            for even distribution.
+     --validator-group-index  Slot validation group index [=0].
+                            The value provided must be in the range [0, validatorGroups). Ignored when
+                            --validator-groups is not provided. Only slot ids satisfying condition [(slotId
+                            mod validationGroups) == groupIndex] will be observed by the validator.
 
 Available sub-commands:
 
@@ -129,19 +156,27 @@ The following options are available:
 
 #### Logging
 
-Codex uses [Chronicles](https://github.com/status-im/nim-chronicles) logging library, which allows great flexibility in working with logs.
-Chronicles has the concept of topics, which categorize log entries into semantic groups.
+Codex uses [Chronicles](https://github.com/status-im/nim-chronicles) logging
+library, which allows great flexibility in working with logs.
+Chronicles has the concept of topics, which categorize log entries into
+semantic groups.
 
-Using the `log-level` parameter, you can set the top-level log level like `--log-level="trace"`, but more importantly,
-you can set log levels for specific topics like `--log-level="info; trace: marketplace,node; error: blockexchange"`,
-which sets the top-level log level to `info` and then for topics `marketplace` and `node` sets the level to `trace` and so on.
+Using the `log-level` parameter, you can set the top-level log level like
+`--log-level="trace"`, but more importantly, you can set log levels for
+specific topics like `--log-level="info; trace: marketplace,node; error: blockexchange"`,
+which sets the top-level log level to `info` and then for topics
+`marketplace` and `node` sets the level to `trace` and so on.
 
 ### Guides
 
 To get acquainted with Codex, consider:
-* running the simple [Codex Two-Client Test](docs/TwoClientTest.md) for a start, and;
-* if you are feeling more adventurous, try [Running a Local Codex Network with Marketplace Support](docs/Marketplace.md) using a local blockchain as well.
+* running the simple [Codex Two-Client Test](docs/TwoClientTest.md) for
+  a start, and;
+* if you are feeling more adventurous, try
+  [Running a Local Codex Network with Marketplace Support](docs/Marketplace.md)
+  using a local blockchain as well.
 
 ## API
 
-The client exposes a REST API that can be used to interact with the clients. Overview of the API can be found on [api.codex.storage](https://api.codex.storage).
+The client exposes a REST API that can be used to interact with the clients.
+Overview of the API can be found on [api.codex.storage](https://api.codex.storage).
diff --git a/codex/codex.nim b/codex/codex.nim
@@ -122,25 +122,30 @@ proc bootstrapInteractions(
     else:
       s.codexNode.clock = SystemClock()
 
-    if config.persistence:
-      # This is used for simulation purposes. Normal nodes won't be compiled with this flag
-      # and hence the proof failure will always be 0.
-      when codex_enable_proof_failures:
-        let proofFailures = config.simulateProofFailures
-        if proofFailures > 0:
-          warn "Enabling proof failure simulation!"
-      else:
-        let proofFailures = 0
-        if config.simulateProofFailures > 0:
-          warn "Proof failure simulation is not enabled for this build! Configuration ignored"
-
-      let purchasing = Purchasing.new(market, clock)
-      let sales = Sales.new(market, clock, repo, proofFailures)
-      client = some ClientInteractions.new(clock, purchasing)
-      host = some HostInteractions.new(clock, sales)
+    # This is used for simulation purposes. Normal nodes won't be compiled with this flag
+    # and hence the proof failure will always be 0.
+    when codex_enable_proof_failures:
+      let proofFailures = config.simulateProofFailures
+      if proofFailures > 0:
+        warn "Enabling proof failure simulation!"
+    else:
+      let proofFailures = 0
+      if config.simulateProofFailures > 0:
+        warn "Proof failure simulation is not enabled for this build! Configuration ignored"
+
+    let purchasing = Purchasing.new(market, clock)
+    let sales = Sales.new(market, clock, repo, proofFailures)
+    client = some ClientInteractions.new(clock, purchasing)
+    host = some HostInteractions.new(clock, sales)
 
     if config.validator:
-      let validation = Validation.new(clock, market, config.validatorMaxSlots)
+      without validationConfig =? ValidationConfig.init(
+        config.validatorMaxSlots,
+        config.validatorGroups,
+        config.validatorGroupIndex), err:
+          error "Invalid validation parameters", err = err.msg
+          quit QuitFailure
+      let validation = Validation.new(clock, market, validationConfig)
       validator = some ValidatorInteractions.new(clock, validation)
 
     s.codexNode.contracts = (client, host, validator)

diff --git a/codex/conf.nim b/codex/conf.nim
@@ -37,8 +37,10 @@ import ./logutils
 import ./stores
 import ./units
 import ./utils
+from ./validationconfig import MaxSlots, ValidationGroups
 
 export units, net, codextypes, logutils
+export ValidationGroups, MaxSlots
 
 export
   DefaultQuotaBytes,
@@ -99,7 +101,8 @@ type
 
     logFormat* {.
       hidden
-      desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
+      desc: "Specifies what kind of logs should be written to stdout (auto, " &
+        "colors, nocolors, json)"
       defaultValueDesc: "auto"
       defaultValue: LogKind.Auto
       name: "log-format" }: LogKind
@@ -164,7 +167,8 @@ type
       name: "net-privkey" }: string
 
     bootstrapNodes* {.
-      desc: "Specifies one or more bootstrap nodes to use when connecting to the network"
+      desc: "Specifies one or more bootstrap nodes to use when " &
+        "connecting to the network"
       abbr: "b"
       name: "bootstrap-node" }: seq[SignedPeerRecord]
 
@@ -192,7 +196,8 @@ type
       abbr: "p" }: Port
 
     apiCorsAllowedOrigin* {.
-      desc: "The REST Api CORS allowed origin for downloading data. '*' will allow all origins, '' will allow none.",
+      desc: "The REST Api CORS allowed origin for downloading data. " &
+        "'*' will allow all origins, '' will allow none.",
       defaultValue: string.none
       defaultValueDesc: "Disallow all cross origin requests to download data"
       name: "api-cors-origin" }: Option[string]
@@ -218,7 +223,9 @@ type
       abbr: "t" }: Duration
 
     blockMaintenanceInterval* {.
-      desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup"
+      desc: "Time interval in seconds - determines frequency of block " &
+        "maintenance cycle: how often blocks are checked " &
+        "for expiration and cleanup"
       defaultValue: DefaultBlockMaintenanceInterval
       defaultValueDesc: $DefaultBlockMaintenanceInterval
       name: "block-mi" }: Duration
@@ -230,7 +237,8 @@ type
       name: "block-mn" }: int
 
     cacheSize* {.
-      desc: "The size of the block cache, 0 disables the cache - might help on slow hardrives"
+      desc: "The size of the block cache, 0 disables the cache - " &
+        "might help on slow hardrives"
       defaultValue: 0
       defaultValueDesc: "0"
       name: "cache-size"
@@ -290,9 +298,35 @@ type
 
       validatorMaxSlots* {.
         desc: "Maximum number of slots that the validator monitors"
+        longDesc: "If set to 0, the validator will not limit " &
+          "the maximum number of slots it monitors"
         defaultValue: 1000
         name: "validator-max-slots"
-      .}: int
+      .}: MaxSlots
+
+      validatorGroups* {.
+        desc: "Slot validation groups"
+        longDesc: "A number indicating total number of groups into " &
+          "which the whole slot id space will be divided. " &
+          "The value must be in the range [2, 65535]. " &
+          "If not provided, the validator will observe " &
+          "the whole slot id space and the value of " &
+          "the --validator-group-index parameter will be ignored. " &
+          "Powers of twos are advised for even distribution"
+        defaultValue: ValidationGroups.none
+        name: "validator-groups"
+      .}: Option[ValidationGroups]
+
+      validatorGroupIndex* {.
+        desc: "Slot validation group index"
+        longDesc: "The value provided must be in the range " &
+          "[0, validatorGroups). Ignored when --validator-groups " &
+          "is not provided. Only slot ids satisfying condition " &
+          "[(slotId mod validationGroups) == groupIndex] will be " &
+          "observed by the validator"
+        defaultValue: 0
+        name: "validator-group-index"
+      .}: uint16
 
       rewardRecipient* {.
         desc: "Address to send payouts to (eg rewards and refunds)"
@@ -546,7 +580,10 @@ proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} =
   try:
     setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii))
   except ValueError:
-    raise (ref ValueError)(msg: "Please specify one of: trace, debug, info, notice, warn, error or fatal")
+    raise (ref ValueError)(
+      msg: "Please specify one of: trace, debug, " &
+        "info, notice, warn, error or fatal"
+    )
 
   if directives.len > 1:
     for topicName, settings in parseTopicDirectives(directives[1..^1]):

diff --git a/codex/validation.nim b/codex/validation.nim
@@ -1,35 +1,38 @@
 import std/sets
 import std/sequtils
 import pkg/chronos
+import pkg/questionable/results
+
+import ./validationconfig
 import ./market
 import ./clock
 import ./logutils
 
 export market
 export sets
+export validationconfig
 
 type
   Validation* = ref object
     slots: HashSet[SlotId]
-    maxSlots: int
     clock: Clock
     market: Market
     subscriptions: seq[Subscription]
     running: Future[void]
     periodicity: Periodicity
     proofTimeout: UInt256
+    config: ValidationConfig
 
 logScope:
   topics = "codex validator"
 
 proc new*(
-    _: type Validation,
-    clock: Clock,
-    market: Market,
-    maxSlots: int
+  _: type Validation,
+  clock: Clock,
+  market: Market,
+  config: ValidationConfig
 ): Validation =
-  ## Create a new Validation instance
-  Validation(clock: clock, market: market, maxSlots: maxSlots)
+  Validation(clock: clock, market: market, config: config)
 
 proc slots*(validation: Validation): seq[SlotId] =
   validation.slots.toSeq
@@ -43,13 +46,29 @@ proc waitUntilNextPeriod(validation: Validation) {.async.} =
   trace "Waiting until next period", currentPeriod = period
   await validation.clock.waitUntil(periodEnd.truncate(int64) + 1)
 
+func groupIndexForSlotId*(slotId: SlotId,
+                          validationGroups: ValidationGroups): uint16 =
+  let slotIdUInt256 = UInt256.fromBytesBE(slotId.toArray)
+  (slotIdUInt256 mod validationGroups.u256).truncate(uint16)
+
+func maxSlotsConstraintRespected(validation: Validation): bool =
+  validation.config.maxSlots == 0 or
+    validation.slots.len < validation.config.maxSlots
+
+func shouldValidateSlot(validation: Validation, slotId: SlotId): bool =
+  if (validationGroups =? validation.config.groups):
+    (groupIndexForSlotId(slotId, validationGroups) ==
+    validation.config.groupIndex) and
+    validation.maxSlotsConstraintRespected
+  else:
+    validation.maxSlotsConstraintRespected
+
 proc subscribeSlotFilled(validation: Validation) {.async.} =
   proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) =
     let slotId = slotId(requestId, slotIndex)
-    if slotId notin validation.slots:
-      if validation.slots.len < validation.maxSlots:
-        trace "Adding slot", slotId
-        validation.slots.incl(slotId)
+    if validation.shouldValidateSlot(slotId):
+      trace "Adding slot", slotId
+      validation.slots.incl(slotId)
   let subscription = await validation.market.subscribeSlotFilled(onSlotFilled)
   validation.subscriptions.add(subscription)