From faeec268fd0529128965760358da4e1d80e303b4 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 5 Aug 2024 14:34:29 -0300 Subject: [PATCH] get avail slots/timestamps from light client instead of rpc --- .../src/funnels/avail/baseFunnel.ts | 2 +- .../src/funnels/avail/parallelFunnel.ts | 39 +++--- .../paima-funnel/src/funnels/avail/utils.ts | 119 ++++++++++++------ packages/engine/paima-funnel/src/index.ts | 2 +- 4 files changed, 103 insertions(+), 59 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/avail/baseFunnel.ts b/packages/engine/paima-funnel/src/funnels/avail/baseFunnel.ts index 59679e15a..a3df2c85d 100644 --- a/packages/engine/paima-funnel/src/funnels/avail/baseFunnel.ts +++ b/packages/engine/paima-funnel/src/funnels/avail/baseFunnel.ts @@ -99,7 +99,7 @@ export class AvailBlockFunnel extends BaseFunnel implements ChainFunnel { ); const parallelHeaders = await timeout( - getMultipleHeaderData(this.api, numbers), + getMultipleHeaderData(this.api, this.config.lightClient, numbers), GET_DATA_TIMEOUT ); diff --git a/packages/engine/paima-funnel/src/funnels/avail/parallelFunnel.ts b/packages/engine/paima-funnel/src/funnels/avail/parallelFunnel.ts index c5346b54c..d29da6a38 100644 --- a/packages/engine/paima-funnel/src/funnels/avail/parallelFunnel.ts +++ b/packages/engine/paima-funnel/src/funnels/avail/parallelFunnel.ts @@ -8,18 +8,17 @@ import type { PoolClient } from 'pg'; import { FUNNEL_PRESYNC_FINISHED } from '@paima/utils'; import { createApi } from './createApi.js'; import { getLatestProcessedCdeBlockheight } from '@paima/db'; -import type { Header } from './utils.js'; import { getDAData, - getLatestBlockNumber, getMultipleHeaderData, - getSlotFromHeader, getTimestampForBlockAt, slotToTimestamp, timestampToSlot, GET_DATA_TIMEOUT, getLatestAvailableBlockNumberFromLightClient, + getBlockHeaderDataFromLightClient, } from './utils.js'; +import type { HeaderData } from './utils.js'; import { addInternalCheckpointingEvent, buildParallelBlockMappings, @@ -137,10 +136,16 @@ export class AvailParallelFunnel extends BaseFunnel implements ChainFunnel { } // get only headers for block that have data - parallelHeaders = await timeout(getMultipleHeaderData(this.api, numbers), GET_DATA_TIMEOUT); + parallelHeaders = await timeout( + getMultipleHeaderData(this.api, this.config.lightClient, numbers), + GET_DATA_TIMEOUT + ); } else { // unless the range is empty - parallelHeaders = await timeout(getMultipleHeaderData(this.api, [to]), GET_DATA_TIMEOUT); + parallelHeaders = await timeout( + getMultipleHeaderData(this.api, this.config.lightClient, [to]), + GET_DATA_TIMEOUT + ); } for (const blockData of roundParallelData) { @@ -290,10 +295,11 @@ export class AvailParallelFunnel extends BaseFunnel implements ChainFunnel { const mappedStartingBlockHeight = await findBlockByTimestamp( // the genesis doesn't have a slot to extract a timestamp from 1, - await getLatestBlockNumber(api), + await getLatestAvailableBlockNumberFromLightClient(config.lightClient), applyDelay(config, Number(startingBlock.timestamp)), chainName, - async (blockNumber: number) => await getTimestampForBlockAt(api, blockNumber) + async (blockNumber: number) => + await getTimestampForBlockAt(config.lightClient, api, blockNumber) ); availFunnelCacheEntry.initialize(mappedStartingBlockHeight); @@ -324,26 +330,27 @@ export class AvailParallelFunnel extends BaseFunnel implements ChainFunnel { const config = this.config; const latestHeader = await timeout( - (async (): Promise
=> { + (async (): Promise => { const latestNumber = await getLatestAvailableBlockNumberFromLightClient(config.lightClient); - const latestBlockHash = await this.api.rpc.chain.getBlockHash(latestNumber); - const latestHeader = await this.api.rpc.chain.getHeader(latestBlockHash); + const latestHeader = await getBlockHeaderDataFromLightClient( + config.lightClient, + latestNumber, + this.api + ); - return latestHeader as unknown as Header; + return latestHeader; })(), LATEST_BLOCK_UPDATE_TIMEOUT ); - const slot = getSlotFromHeader(latestHeader, this.api); - this.sharedData.cacheManager.cacheEntries[AvailFunnelCacheEntry.SYMBOL]?.updateLatestBlock({ - number: latestHeader.number.toNumber(), + number: latestHeader.number, hash: latestHeader.hash.toString(), - slot: slot, + slot: latestHeader.slot, }); - return latestHeader.number.toNumber(); + return latestHeader.number; } private getCacheEntry(): AvailFunnelCacheEntry { diff --git a/packages/engine/paima-funnel/src/funnels/avail/utils.ts b/packages/engine/paima-funnel/src/funnels/avail/utils.ts index f9dc9fe05..591d08626 100644 --- a/packages/engine/paima-funnel/src/funnels/avail/utils.ts +++ b/packages/engine/paima-funnel/src/funnels/avail/utils.ts @@ -1,40 +1,24 @@ import type { ApiPromise } from 'avail-js-sdk'; -import type { Header as PolkadotHeader } from '@polkadot/types/interfaces/types'; +import type { Header as PolkadotHeader, DigestItem } from '@polkadot/types/interfaces/types'; +import { Bytes } from '@polkadot/types-codec'; import type { SubmittedData } from '@paima/sm'; import { base64Decode } from '@polkadot/util-crypto'; import { BaseFunnelSharedApi } from '../BaseFunnel.js'; import { createApi } from './createApi.js'; +import { GenericConsensusEngineId } from '@polkadot/types/generic/ConsensusEngineId'; export const GET_DATA_TIMEOUT = 10000; export type Header = PolkadotHeader; -export function getSlotFromHeader(header: Header, api: ApiPromise): number { - const preRuntime = header.digest.logs.find(log => log.isPreRuntime)!.asPreRuntime; - - const rawBabeDigest = api.createType('RawBabePreDigest', preRuntime[1]); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const babeDigest = rawBabeDigest.toPrimitive() as unknown as any; - - // the object is an enumeration, but all the variants have a slotNumber field - const slot = babeDigest[Object.getOwnPropertyNames(babeDigest)[0]].slotNumber; - return slot; -} - -export async function getLatestBlockNumber(api: ApiPromise): Promise { - let highHash = await api.rpc.chain.getFinalizedHead(); - let high = (await api.rpc.chain.getHeader(highHash)).number.toNumber(); - return high; -} - -export async function getTimestampForBlockAt(api: ApiPromise, mid: number): Promise { - const hash = await api.rpc.chain.getBlockHash(mid); - // FIXME: why is the conversion needed? - const header = (await api.rpc.chain.getHeader(hash)) as unknown as Header; +export async function getTimestampForBlockAt( + lc: string, + api: ApiPromise, + bn: number +): Promise { + const header = await getBlockHeaderDataFromLightClient(lc, bn, api); - const slot = getSlotFromHeader(header, api); - return slotToTimestamp(slot, api); + return slotToTimestamp(header.slot, api); } export function slotToTimestamp(slot: number, api: ApiPromise): number { @@ -55,32 +39,82 @@ export function timestampToSlot(timestamp: number, api: ApiPromise): number { return timestamp / slotDuration; } -type HeaderData = { number: number; hash: string; slot: number }; +export type HeaderData = { number: number; hash: string; slot: number }; export async function getMultipleHeaderData( api: ApiPromise, + lc: string, blockNumbers: number[] ): Promise { const results = [] as HeaderData[]; for (const bn of blockNumbers) { - // NOTE: the light client allows getting header directly from block number, - // but it doesn't provide the babe data for the slot - const hash = await api.rpc.chain.getBlockHash(bn); - const header = (await api.rpc.chain.getHeader(hash)) as unknown as Header; - - const slot = getSlotFromHeader(header, api); - - results.push({ - number: header.number.toNumber(), - hash: header.hash.toString(), - slot: slot, - }); + results.push(await getBlockHeaderDataFromLightClient(lc, bn, api)); } return results; } +export async function getBlockHeaderDataFromLightClient( + lc: string, + bn: number, + api: ApiPromise +): Promise<{ number: number; hash: string; slot: number }> { + const responseRaw = await fetch(`${lc}/v2/blocks/${bn}/header`); + + if (responseRaw.status !== 200) { + // we don't want to accidentally skip blocks if there is something wrong + // with the light client. We only fetch blocks in range, so a not found + // here it's a logic error. + throw new Error( + `Unexpected error encountered when fetching headers from Avail's light client. Error: ${responseRaw.status}` + ); + } + + const response = (await responseRaw.json()) as unknown as { + hash: string; + number: number; + digest: { + logs: { + [key in DigestItem['type']]: [number[], number[]]; + }[]; + }; + }; + + const preRuntimeJson = response.digest.logs.find(log => log.PreRuntime)?.PreRuntime; + + if (!preRuntimeJson) { + throw new Error("Couldn't find preruntime digest"); + } + + // using ts-expect-error because for some reason the types of the registry are + // different, but avail-js-sdk doesn't seem to re-export @polkadot/types in + // order to access these constructors directly. + const preRuntime = [ + // this is not used, but we parse it just in case it fails. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + new GenericConsensusEngineId(api.registry, preRuntimeJson[0]), + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + new Bytes(api.registry, preRuntimeJson[1]), + ]; + + const rawBabeDigest = api.createType('RawBabePreDigest', preRuntime[1]); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const babeDigest = rawBabeDigest.toPrimitive() as unknown as any; + + // the object is an enumeration, but all the variants have a slotNumber field + const slot = babeDigest[Object.getOwnPropertyNames(babeDigest)[0]].slotNumber; + + return { + number: response.number, + hash: response.hash, + slot: slot, + }; +} + export async function getDAData( api: ApiPromise, lc: string, @@ -157,7 +191,10 @@ export async function getLatestAvailableBlockNumberFromLightClient(lc: string): } export class AvailSharedApi extends BaseFunnelSharedApi { - public constructor(private rpc: string) { + public constructor( + private rpc: string, + private lightClient: string + ) { super(); this.getBlock.bind(this); } @@ -167,7 +204,7 @@ export class AvailSharedApi extends BaseFunnelSharedApi { ): Promise<{ timestamp: number | string } | undefined> { const api = await createApi(this.rpc); - const headerData = await getMultipleHeaderData(api, [height]); + const headerData = await getMultipleHeaderData(api, this.lightClient, [height]); const timestamp = slotToTimestamp(headerData[0].slot, api); diff --git a/packages/engine/paima-funnel/src/index.ts b/packages/engine/paima-funnel/src/index.ts index 1e6db2eee..16365ded0 100644 --- a/packages/engine/paima-funnel/src/index.ts +++ b/packages/engine/paima-funnel/src/index.ts @@ -52,7 +52,7 @@ export class FunnelFactory implements IFunnelFactory { } if (mainConfig.type === ConfigNetworkType.AVAIL_MAIN) { - mainNetworkApi = new AvailSharedApi(mainConfig.rpc); + mainNetworkApi = new AvailSharedApi(mainConfig.rpc, mainConfig.lightClient); } const web3s = await Promise.all(