Improve chain priority logic (#690)

* Improve chain priority logic

* Update scenarios/test_codegen/test/lib_tests/FetchState_test.res

* Update performance in the readme

* Improve hasFullBatch logic

* Fix Hyperliquid chain id

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

Author: DZakh

.cursor/rules/navigation.mdc

@@ -31,3 +31,9 @@ Tips for navigation:
 - Config parsing & codegen lives in Rust. When tracking how a value reaches templates, follow `human_config.rs` → `system_config.rs` → `codegen_templates.rs`.
 - Prefer reading ReScript `.res` modules directly; compiled `.js` artifacts can be ignored.
 - When suggesting file edits to contributors, reference modules with `.res` extension but drop the long path (e.g., "edit `ChainFetcher.res`").
+
+## Testing
+
+Prefer Public module API for testing.
+
+Verify that tests pass by running a compiler `pnpm rescript` and tests `pnpm mocha`. Use `_only` to specify which tests to run.

codegenerator/cli/README.md

@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 5,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
 - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-compatible blockchain and Fuel (simultaneously)
 - **Real-time indexing** – Instantly track blockchain events

codegenerator/cli/npm/envio/src/FetchState.res

@@ -1235,14 +1235,34 @@ let filterAndSortForUnorderedBatch = {
     }
   }
 
-  let compareUnorderedBatchChainPriority = (a: t, b: t) => {
-    // Use unsafe since we filtered out all queues without batch items
-    (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+  let hasFullBatch = ({queue, latestFullyFetchedBlock}: t, ~maxBatchSize) => {
+    // Queue is ordered from latest to earliest, so the earliest eligible
+    // item for a full batch of size B is at index (length - B).
+    // Do NOT subtract an extra 1 here; when length === B we should still
+    // classify the queue as full and probe index 0.
+    let targetBlockIdx = queue->Array.length - maxBatchSize
+    if targetBlockIdx < 0 {
+      false
+    } else {
+      // Unsafe can fail when maxBatchSize is 0,
+      // but we ignore the case
+      (queue->Js.Array2.unsafe_get(targetBlockIdx)).blockNumber <=
+      latestFullyFetchedBlock.blockNumber
+    }
   }
 
-  (fetchStates: array<t>) => {
+  (fetchStates: array<t>, ~maxBatchSize: int) => {
     fetchStates
     ->Array.keepU(hasBatchItem)
-    ->Js.Array2.sortInPlaceWith(compareUnorderedBatchChainPriority)
+    ->Js.Array2.sortInPlaceWith((a: t, b: t) => {
+      switch (a->hasFullBatch(~maxBatchSize), b->hasFullBatch(~maxBatchSize)) {
+      | (true, true)
+      | (false, false) =>
+        // Use unsafe since we filtered out all queues without batch items
+        (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+      | (true, false) => -1
+      | (false, true) => 1
+      }
+    })
   }
 }

codegenerator/cli/src/config_parsing/chain_helpers.rs

@@ -194,7 +194,7 @@ pub enum Network {
     Holesky = 17000,
 
     #[subenum(HypersyncNetwork)]
-    Hyperliquid = 645749,
+    Hyperliquid = 999,
 
     IncoGentryTestnet = 9090,
 

codegenerator/cli/templates/static/codegen/src/eventFetching/ChainManager.res

@@ -185,7 +185,7 @@ let createUnorderedBatch = (
   let preparedFetchStates =
     fetchStates
     ->ChainMap.values
-    ->FetchState.filterAndSortForUnorderedBatch
+    ->FetchState.filterAndSortForUnorderedBatch(~maxBatchSize)
 
   let idx = ref(0)
   let preparedNumber = preparedFetchStates->Array.length

scenarios/test_codegen/test/lib_tests/FetchState_test.res

@@ -1818,12 +1818,10 @@ describe("FetchState.getNextQuery & integration", () => {
 
 describe("FetchState unit tests for specific cases", () => {
   it("Should merge events in correct order on merging", () => {
-    let normalSelection: FetchState.selection = {
-      dependsOnAddresses: true,
-      eventConfigs: [],
-    }
-    let fetchState: FetchState.t = {
-      partitions: [
+    let base = makeInitial()
+    let normalSelection = base.normalSelection
+    let fetchState = base->FetchState.updateInternal(
+      ~partitions=[
         {
           id: "0",
           status: {fetchingStateId: None},
@@ -1845,32 +1843,15 @@ describe("FetchState unit tests for specific cases", () => {
           addressesByContractName: Js.Dict.empty(),
         },
       ],
-      nextPartitionIndex: 2,
-      isFetchingAtHead: false,
-      maxAddrInPartition: 2,
-      latestFullyFetchedBlock: {
-        blockNumber: 1,
-        blockTimestamp: 0,
-      },
-      queue: [
+      ~nextPartitionIndex=2,
+      ~queue=[
         mockEvent(~blockNumber=4, ~logIndex=2),
         mockEvent(~blockNumber=4),
         mockEvent(~blockNumber=3),
         mockEvent(~blockNumber=2),
         mockEvent(~blockNumber=1),
       ],
-      firstEventBlockNumber: Some(1),
-      endBlock: undefined,
-      normalSelection,
-      chainId,
-      indexingContracts: Js.Dict.empty(),
-      contractConfigs: Js.Dict.fromArray([
-        ("Gravatar", {FetchState.filterByAddresses: false}),
-        ("NftFactory", {FetchState.filterByAddresses: false}),
-      ]),
-      dcsToStore: None,
-      blockLag: 0,
-    }
+    )
 
     let updatedFetchState =
       fetchState
@@ -2139,16 +2120,10 @@ describe("FetchState unit tests for specific cases", () => {
 
   it("Get earliest event", () => {
     let latestFetchedBlock = getBlockData(~blockNumber=500)
-
-    let normalSelection: FetchState.selection = {
-      dependsOnAddresses: true,
-      eventConfigs: [
-        (Mock.evmEventConfig(~id="0", ~contractName="ContractA") :> Internal.eventConfig),
-      ],
-    }
-
-    let fetchState: FetchState.t = {
-      partitions: [
+    let base = makeInitial()
+    let normalSelection = base.normalSelection
+    let fetchState = base->FetchState.updateInternal(
+      ~partitions=[
         {
           id: "0",
           status: {fetchingStateId: None},
@@ -2164,27 +2139,13 @@ describe("FetchState unit tests for specific cases", () => {
           addressesByContractName: Js.Dict.empty(),
         },
       ],
-      nextPartitionIndex: 2,
-      isFetchingAtHead: false,
-      maxAddrInPartition: 2,
-      latestFullyFetchedBlock: latestFetchedBlock,
-      queue: [
+      ~nextPartitionIndex=2,
+      ~queue=[
         mockEvent(~blockNumber=6, ~logIndex=1),
         mockEvent(~blockNumber=5),
         mockEvent(~blockNumber=2, ~logIndex=1),
       ],
-      firstEventBlockNumber: Some(1),
-      endBlock: undefined,
-      normalSelection,
-      chainId,
-      indexingContracts: Js.Dict.empty(),
-      contractConfigs: Js.Dict.fromArray([
-        ("Gravatar", {FetchState.filterByAddresses: false}),
-        ("NftFactory", {FetchState.filterByAddresses: false}),
-      ]),
-      dcsToStore: None,
-      blockLag: 0,
-    }
+    )
 
     Assert.deepEqual(
       fetchState->FetchState.getEarliestEvent->getItem,
@@ -2667,7 +2628,55 @@ describe("Test queue item", () => {
 })
 
 describe("FetchState.filterAndSortForUnorderedBatch", () => {
-  it("Filters out states without eligible items and sorts by earliest timestamp (public API)", () => {
+  it(
+    "Filters out states without eligible items and sorts by earliest timestamp (public API)",
+    () => {
+      let mk = () => makeInitial()
+      let mkQuery = (fetchState: FetchState.t) => {
+        {
+          FetchState.partitionId: "0",
+          target: Head,
+          selection: fetchState.normalSelection,
+          addressesByContractName: Js.Dict.empty(),
+          fromBlock: 0,
+          indexingContracts: fetchState.indexingContracts,
+        }
+      }
+
+      // Helper: create a fetch state with desired latestFetchedBlock and queue items via public API
+      let makeFsWith = (~latestBlock: int, ~queueBlocks: array<int>): FetchState.t => {
+        let fs0 = mk()
+        let query = mkQuery(fs0)
+        fs0
+        ->FetchState.handleQueryResult(
+          ~query,
+          ~latestFetchedBlock={blockNumber: latestBlock, blockTimestamp: latestBlock},
+          ~reversedNewItems=queueBlocks->Array.map(b => mockEvent(~blockNumber=b)),
+          ~currentBlockHeight=latestBlock,
+        )
+        ->Result.getExn
+      }
+
+      // Included: last queue item at block 1, latestFullyFetchedBlock = 10
+      let fsEarly = makeFsWith(~latestBlock=10, ~queueBlocks=[2, 1])
+      // Included: last queue item at block 5, latestFullyFetchedBlock = 10
+      let fsLate = makeFsWith(~latestBlock=10, ~queueBlocks=[5])
+      // Excluded: last queue item at block 11 (> latestFullyFetchedBlock = 10)
+      let fsExcluded = makeFsWith(~latestBlock=10, ~queueBlocks=[11])
+
+      let prepared = FetchState.filterAndSortForUnorderedBatch(
+        [fsLate, fsExcluded, fsEarly],
+        ~maxBatchSize=3,
+      )
+
+      Assert.deepEqual(
+        prepared->Array.map(fs => (fs.queue->Utils.Array.last->Option.getUnsafe).blockNumber),
+        [1, 5],
+      )
+    },
+  )
+
+  it("Prioritizes full batches over half full ones", () => {
     let mk = () => makeInitial()
     let mkQuery = (fetchState: FetchState.t) => {
       {
@@ -2680,7 +2689,6 @@ describe("FetchState.filterAndSortForUnorderedBatch", () => {
       }
     }
 
-    // Helper: create a fetch state with desired latestFetchedBlock and queue items via public API
     let makeFsWith = (~latestBlock: int, ~queueBlocks: array<int>): FetchState.t => {
       let fs0 = mk()
       let query = mkQuery(fs0)
@@ -2694,18 +2702,62 @@ describe("FetchState.filterAndSortForUnorderedBatch", () => {
       ->Result.getExn
     }
 
-    // Included: last queue item at block 1, latestFullyFetchedBlock = 10
-    let fsEarly = makeFsWith(~latestBlock=10, ~queueBlocks=[2, 1])
-    // Included: last queue item at block 5, latestFullyFetchedBlock = 10
-    let fsLate = makeFsWith(~latestBlock=10, ~queueBlocks=[5])
-    // Excluded: last queue item at block 11 (> latestFullyFetchedBlock = 10)
-    let fsExcluded = makeFsWith(~latestBlock=10, ~queueBlocks=[11])
+    // Full batch (>= maxBatchSize items). Make it later (earliest item at block 7)
+    let fsFullLater = makeFsWith(~latestBlock=10, ~queueBlocks=[9, 8, 7])
+    // Half-full batch (1 item) but earlier earliest item (block 1)
+    let fsHalfEarlier = makeFsWith(~latestBlock=10, ~queueBlocks=[1])
 
-    let prepared = [fsLate, fsExcluded, fsEarly]->FetchState.filterAndSortForUnorderedBatch
+    let prepared = FetchState.filterAndSortForUnorderedBatch(
+      [fsHalfEarlier, fsFullLater],
+      ~maxBatchSize=2,
+    )
+
+    Assert.deepEqual(
+      prepared->Array.map(fs => (fs.queue->Utils.Array.last->Option.getUnsafe).blockNumber),
+      [7, 1],
+    )
+  })
+
+  it("Treats exactly-full batches as full", () => {
+    let mk = () => makeInitial()
+    let mkQuery = (fetchState: FetchState.t) => {
+      {
+        FetchState.partitionId: "0",
+        target: Head,
+        selection: fetchState.normalSelection,
+        addressesByContractName: Js.Dict.empty(),
+        fromBlock: 0,
+        indexingContracts: fetchState.indexingContracts,
+      }
+    }
+
+    let makeFsWith = (~latestBlock: int, ~queueBlocks: array<int>): FetchState.t => {
+      let fs0 = mk()
+      let query = mkQuery(fs0)
+      fs0
+      ->FetchState.handleQueryResult(
+        ~query,
+        ~latestFetchedBlock={blockNumber: latestBlock, blockTimestamp: latestBlock},
+        ~reversedNewItems=queueBlocks->Array.map(b => mockEvent(~blockNumber=b)),
+        ~currentBlockHeight=latestBlock,
+      )
+      ->Result.getExn
+    }
+
+    // Exactly full (== maxBatchSize items)
+    let fsExactFull = makeFsWith(~latestBlock=10, ~queueBlocks=[3, 2])
+    // Half-full (1 item) but earlier earliest item
+    let fsHalfEarlier = makeFsWith(~latestBlock=10, ~queueBlocks=[1])
+
+    let prepared = FetchState.filterAndSortForUnorderedBatch(
+      [fsHalfEarlier, fsExactFull],
+      ~maxBatchSize=2,
+    )
 
+    // Full batch should take priority regardless of earlier timestamp of half batch
     Assert.deepEqual(
       prepared->Array.map(fs => (fs.queue->Utils.Array.last->Option.getUnsafe).blockNumber),
-      [1, 5],
+      [2, 1],
     )
   })
 })