feat: indexing fees / dips (wip)

Author: pcarranzav

packages/indexer-agent/jest.config.js

@@ -11,6 +11,13 @@ module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
   testPathIgnorePatterns: ['/node_modules/', '/dist/', '.yalc'],
+  transformIgnorePatterns: [
+    // Transform @graphprotocol packages but ignore other node_modules
+    '/node_modules/(?!(@graphprotocol)/)'
+  ],
+  transform: {
+    '^.+\\.ts$': ['ts-jest']  // Only transform .ts files
+  },
   globals: {
     __DATABASE__: {
       host:

packages/indexer-agent/src/__tests__/indexer.ts

@@ -146,6 +146,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     networkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-agent/src/agent.ts

@@ -223,6 +223,16 @@ export class Agent {
         { logger, milliseconds: requestIntervalSmall },
         async () => {
           const { network, operator } = this.networkAndOperator
+          if (network.specification.indexerOptions.enableDips) {
+            // There should be a DipsManager in the operator
+            if (!operator.dipsManager) {
+              throw new Error('DipsManager is not available')
+            }
+            logger.trace('Ensuring indexing rules for DIPS', {
+              protocolNetwork: network.specification.networkIdentifier,
+            })
+            await operator.dipsManager.ensureAgreementRules()
+          }
           logger.trace('Fetching indexing rules', {
             protocolNetwork: network.specification.networkIdentifier,
           })

packages/indexer-agent/src/commands/start.ts

@@ -307,6 +307,26 @@ export const start = {
         default: 1,
         group: 'Indexer Infrastructure',
       })
+      .option('enable-dips', {
+        description: 'Whether to enable Indexing Fees (DIPs)',
+        type: 'boolean',
+        default: false,
+        group: 'Indexing Fees ("DIPs")',
+      })
+      .option('dipper-endpoint', {
+        description: 'Gateway endpoint for DIPs receipts',
+        type: 'string',
+        array: false,
+        required: false,
+        group: 'Indexing Fees ("DIPs")',
+      })
+      .option('dips-allocation-amount', {
+        description: 'Amount of GRT to allocate for DIPs',
+        type: 'number',
+        default: 1,
+        required: false,
+        group: 'Indexing Fees ("DIPs")',
+      })
       .check(argv => {
         if (
           !argv['network-subgraph-endpoint'] &&
@@ -334,6 +354,9 @@ export const start = {
         ) {
           return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.'
         }
+        if (argv['enable-dips'] && !argv['dipper-endpoint']) {
+          return 'Invalid --dipper-endpoint provided. Must be provided when --enable-dips is true.'
+        }
         return true
       })
   },
@@ -369,6 +392,10 @@ export async function createNetworkSpecification(
     allocateOnNetworkSubgraph: argv.allocateOnNetworkSubgraph,
     register: argv.register,
     finalityTime: argv.chainFinalizeTime,
+    enableDips: argv.enableDips,
+    dipperEndpoint: argv.dipperEndpoint,
+    dipsAllocationAmount: argv.dipsAllocationAmount,
+    dipsEpochsMargin: argv.dipsEpochsMargin,
   }
 
   const transactionMonitoring = {
@@ -585,6 +612,7 @@ export async function run(
   const network = await Network.create(
     logger,
     networkSpecification,
+    managementModels,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-cli/jest.config.js

@@ -13,6 +13,13 @@ module.exports = {
   // The glob patterns Jest uses to detect test files
   testMatch: ['**/?(*.)+(spec|test).ts?(x)'],
   testPathIgnorePatterns: ['/node_modules/', '/dist/', '.yalc'],
+  transformIgnorePatterns: [
+    // Transform @graphprotocol packages but ignore other node_modules
+    '/node_modules/(?!(@graphprotocol)/)'
+  ],
+  transform: {
+    '^.+\\.ts$': ['ts-jest']  // Only transform .ts files
+  },
   testTimeout: 60000,
   globals: {
     __DATABASE__: {

packages/indexer-cli/src/__tests__/util.ts

@@ -87,6 +87,7 @@ export const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/jest.config.js

@@ -11,7 +11,13 @@ module.exports = {
   forceExit: true,
   testEnvironment: 'node',
   testPathIgnorePatterns: ['/node_modules/', '/dist/', '/.yalc', 'util.ts'],
-  transformIgnorePatterns: ['!node_modules/'],
+  transformIgnorePatterns: [
+    // Transform @graphprotocol packages but ignore other node_modules
+    '/node_modules/(?!(@graphprotocol)/)'
+  ],
+  transform: {
+    '^.+\\.ts$': ['ts-jest']  // Only transform .ts files
+  },
   globals: {
     __DATABASE__: {
       host: process.env.POSTGRES_TEST_HOST || bail('POSTGRES_TEST_HOST is not defined'),

packages/indexer-common/package.json

@@ -22,8 +22,11 @@
     "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo"
   },
   "dependencies": {
+    "@bufbuild/protobuf": "2.2.3",
     "@graphprotocol/common-ts": "2.0.11",
     "@graphprotocol/cost-model": "0.1.18",
+    "@graphprotocol/dips-proto": "0.2.1",
+    "@grpc/grpc-js": "^1.12.6",
     "@semiotic-labs/tap-contracts-bindings": "^1.2.1",
     "@thi.ng/heaps": "1.2.38",
     "@types/lodash.clonedeep": "^4.5.7",

packages/indexer-common/src/allocations/__tests__/tap.test.ts

@@ -7,6 +7,7 @@ import {
   TapSubgraphResponse,
   TapCollector,
   Allocation,
+  defineIndexerManagementModels,
 } from '@graphprotocol/indexer-common'
 import {
   Address,
@@ -43,6 +44,7 @@ const setup = async () => {
   // Clearing the registry prevents duplicate metric registration in the default registry.
   metrics.registry.clear()
   sequelize = await connectDatabase(__DATABASE__)
+  const models = defineIndexerManagementModels(sequelize)
   queryFeeModels = defineQueryFeeModels(sequelize)
   sequelize = await sequelize.sync({ force: true })
 
@@ -56,6 +58,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts

@@ -1,4 +1,5 @@
 import {
+  defineIndexerManagementModels,
   defineQueryFeeModels,
   GraphNode,
   Network,
@@ -36,6 +37,7 @@ const setup = async () => {
   // Clearing the registry prevents duplicate metric registration in the default registry.
   metrics.registry.clear()
   sequelize = await connectDatabase(__DATABASE__)
+  const models = defineIndexerManagementModels(sequelize)
   queryFeeModels = defineQueryFeeModels(sequelize)
   sequelize = await sequelize.sync({ force: true })
 
@@ -49,6 +51,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,