Merge pull request #188 from securesecrets/batch-query-validators-data

feat: change from individual to batch query of validators

.changeset/rotten-paws-judge.md

@@ -0,0 +1,5 @@
+---
+"@shadeprotocol/shadejs": patch
+---
+
+change from individual validator queries to batch data

src/client/services/clientServices.ts

@@ -74,16 +74,35 @@ const secretClientTokenSupplyQuery$ = (
   })),
 ));
 
-// This function queries the total supply of the a token
+// This function queries the individual validator data
 const secretClientValidatorQuery$ = (
   client: SecretNetworkClient,
   validatorAddress: string,
 ) => createFetchClient(defer(
   () => from(client.query.staking.validator({ validator_addr: validatorAddress })),
 ));
 
+// This function queries a single page of multiple validators data
+const secretClientValidatorsQuery$ = ({
+  client,
+  offset,
+  limit,
+}:{
+  client: SecretNetworkClient,
+  offset?: number,
+  limit?: number,
+}) => createFetchClient(defer(
+  () => from(client.query.staking.validators({
+    pagination: {
+      offset: offset ? offset.toString() : undefined,
+      limit: limit ? limit.toString() : undefined,
+    },
+  })),
+));
+
 export {
   sendSecretClientContractQuery$,
   secretClientTokenSupplyQuery$,
   secretClientValidatorQuery$,
+  secretClientValidatorsQuery$,
 };

src/lib/apy/derivativeScrt.test.ts

@@ -23,7 +23,7 @@ beforeAll(() => {
     ...(await importOriginal()),
     secretChainQueries$: vi.fn(() => of(chainQueryParsedResponse)),
     queryScrtTotalSupply$: vi.fn(() => of(292470737038201)),
-    queryValidatorsCommission$: vi.fn(() => of(mockValidatorsCommissions)),
+    queryAllValidatorsCommissions$: vi.fn(() => of(mockValidatorsCommissions)),
   }));
 
   vi.mock('~/contracts/services/derivativeScrt', () => ({

src/lib/apy/derivativeScrt.ts

@@ -1,11 +1,12 @@
 import {
   SecretChainDataQueryModel,
+  ValidatorRate,
 } from '~/types/apy';
 import {
+  first,
   forkJoin,
   lastValueFrom,
   map,
-  switchMap,
 } from 'rxjs';
 import {
   DerivativeScrtInfo,
@@ -14,7 +15,7 @@ import { convertCoinFromUDenom } from '~/lib/utils';
 import { queryDerivativeScrtInfo$ } from '~/contracts/services/derivativeScrt';
 import {
   queryScrtTotalSupply$,
-  queryValidatorsCommission$,
+  queryAllValidatorsCommissions$,
   secretChainQueries$,
   SecretQueryOptions,
 } from '~/lib/apy/secretQueries';
@@ -47,6 +48,11 @@ function calculateDerivativeScrtApy$({
   return forkJoin({
     chainParameters: secretChainQueries$<SecretChainDataQueryModel>(lcdEndpoint, queries),
     scrtTotalSupplyRaw: queryScrtTotalSupply$(lcdEndpoint, chainId),
+    validatorCommissions: queryAllValidatorsCommissions$({
+      lcdEndpoint,
+      chainId,
+      limit: undefined, // no limit on the batch size, let secretjs handle this for us
+    }),
     derivativeInfo: queryDerivativeScrtInfo$({
       queryRouterContractAddress,
       queryRouterCodeHash,
@@ -56,36 +62,30 @@ function calculateDerivativeScrtApy$({
       chainId,
     }),
   }).pipe(
-    switchMap((response: {
+    map((response: {
       chainParameters: SecretChainDataQueryModel,
       scrtTotalSupplyRaw: number,
+      validatorCommissions: ValidatorRate[],
       derivativeInfo: DerivativeScrtInfo,
-    }) => queryValidatorsCommission$({
-      lcdEndpoint,
-      chainId,
-      validatorAddresses: response.derivativeInfo.validators.map((
-        validator,
-      ) => validator.validatorAddress),
-    }).pipe(
-      map((validatorCommissions) => {
-        const apr = calcAggregateAPR({
-          networkValidatorList: validatorCommissions,
-          validatorSet: response.derivativeInfo.validators,
-          inflationRate: response.chainParameters.secretInflationPercent,
-          totalScrtStaked: convertCoinFromUDenom(
-            response.chainParameters.secretTotalStakedRaw,
-            SECRET_DECIMALS,
-          ).toNumber(),
-          totalScrtSupply: convertCoinFromUDenom(
-            response.scrtTotalSupplyRaw,
-            SECRET_DECIMALS,
-          ).toNumber(),
-          foundationTax: response.chainParameters.secretTaxes!.foundationTaxPercent,
-          communityTax: response.chainParameters.secretTaxes!.communityTaxPercent,
-        });
-        return calcAPY(365, apr);
-      }),
-    )),
+    }) => {
+      const apr = calcAggregateAPR({
+        networkValidatorList: response.validatorCommissions,
+        validatorSet: response.derivativeInfo.validators,
+        inflationRate: response.chainParameters.secretInflationPercent,
+        totalScrtStaked: convertCoinFromUDenom(
+          response.chainParameters.secretTotalStakedRaw,
+          SECRET_DECIMALS,
+        ).toNumber(),
+        totalScrtSupply: convertCoinFromUDenom(
+          response.scrtTotalSupplyRaw,
+          SECRET_DECIMALS,
+        ).toNumber(),
+        foundationTax: response.chainParameters.secretTaxes!.foundationTaxPercent,
+        communityTax: response.chainParameters.secretTaxes!.communityTaxPercent,
+      });
+      return calcAPY(365, apr);
+    }),
+    first(),
   );
 }
 

src/lib/apy/secretQueries.ts

@@ -5,9 +5,10 @@ import {
   lastValueFrom,
   map,
   first, switchMap,
-  from,
-  mergeMap,
-  toArray,
+  expand,
+  of,
+  takeWhile,
+  reduce,
 } from 'rxjs';
 import { fromFetch } from 'rxjs/fetch';
 import { createFetch } from '~/client/services/createFetch';
@@ -16,8 +17,9 @@ import { getActiveQueryClient$ } from '~/client';
 import {
   secretClientTokenSupplyQuery$,
   secretClientValidatorQuery$,
+  secretClientValidatorsQuery$,
 } from '~/client/services/clientServices';
-import { QueryValidatorResponse } from 'secretjs/dist/grpc_gateway/cosmos/staking/v1beta1/query.pb';
+import { QueryValidatorResponse, QueryValidatorsResponse } from 'secretjs/dist/grpc_gateway/cosmos/staking/v1beta1/query.pb';
 import { ValidatorRate } from '~/types/apy';
 
 enum SecretQueryOptions {
@@ -157,6 +159,29 @@ const parseValidatorResponse = (response: QueryValidatorResponse): ValidatorRate
   };
 };
 
+const parseValidatorsResponse = (response: QueryValidatorsResponse): ValidatorRate[] => {
+  if (response === undefined
+    || response.validators === undefined
+  ) {
+    throw new Error('Validators commissions not found');
+  }
+
+  return response.validators.map((validator) => {
+    if (validator === undefined
+      || validator.commission === undefined
+      || validator.commission.commission_rates === undefined
+      || validator.operator_address === undefined
+      || validator.commission.commission_rates.rate === undefined
+    ) {
+      throw new Error('Validator commission not found');
+    }
+    return {
+      validatorAddress: validator.operator_address,
+      ratePercent: Number(validator.commission.commission_rates.rate),
+    };
+  });
+};
+
 /**
  * query the SCRT token total supply
  */
@@ -187,22 +212,58 @@ const queryValidatorCommission$ = ({
 );
 
 /**
- * query the commission for multiple validators
+ * query the commission for all validators
  */
-const queryValidatorsCommission$ = ({
+const queryAllValidatorsCommissions$ = ({
   lcdEndpoint,
   chainId,
-  validatorAddresses,
-}: {
-  lcdEndpoint?: string;
-  chainId?: string;
-  validatorAddresses: string[];
-}) => from(validatorAddresses).pipe(
-  mergeMap((
-    validatorAddress,
-  ) => queryValidatorCommission$({ lcdEndpoint, chainId, validatorAddress })),
-  toArray(),
+  limit,
+}:{
+  lcdEndpoint?: string,
+  chainId?: string,
+  limit?: number,
+}) => getActiveQueryClient$(lcdEndpoint, chainId).pipe(
+  switchMap(({ client }) => {
+    const initialOffset = 0;
+    return of({
+      validators: [],
+      offset: initialOffset,
+      totalItems: undefined,
+    }).pipe(
+      expand(({ offset }) => secretClientValidatorsQuery$({
+        client,
+        offset,
+        limit,
+      }).pipe(
+        map((response) => {
+          const validators = response.validators === undefined
+            ? []
+            : parseValidatorsResponse(response);
+
+          const newOffset = offset + validators.length;
+          return {
+            validators,
+            offset: newOffset,
+            // non-null assertation used because the parser would
+            // have already caught issues with undefined
+            totalItems: Number(response.pagination!.total),
+          };
+        }),
+      )),
+      takeWhile(
+        ({
+          offset, totalItems,
+        }) => offset < totalItems || totalItems === undefined,
+        true, // include the last value
+      ),
+      reduce((
+        allValidators,
+        { validators },
+      ) => allValidators.concat(validators), [] as ValidatorRate[]),
+    );
+  }),
 );
+
 export {
   SecretQueryOptions,
   parseSecretQueryResponse,
@@ -212,5 +273,5 @@ export {
   secretChainQueries,
   queryScrtTotalSupply$,
   queryValidatorCommission$,
-  queryValidatorsCommission$,
+  queryAllValidatorsCommissions$,
 };