From 66c17646917a1681b8451c2e29ff2c8667b3eaf4 Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Wed, 16 Aug 2023 10:45:55 -0600 Subject: [PATCH 1/7] feat(hasura): add table to persists historical stats --- .../default/tables/evm_historical_stats.yaml | 13 +++++++++++++ .../metadata/databases/default/tables/tables.yaml | 1 + .../down.sql | 5 +++++ .../up.sql | 5 +++++ .../down.sql | 1 + .../up.sql | 2 ++ 6 files changed, 27 insertions(+) create mode 100644 hasura/metadata/databases/default/tables/evm_historical_stats.yaml create mode 100644 hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql create mode 100644 hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql create mode 100644 hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql create mode 100644 hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql diff --git a/hasura/metadata/databases/default/tables/evm_historical_stats.yaml b/hasura/metadata/databases/default/tables/evm_historical_stats.yaml new file mode 100644 index 00000000..db99c245 --- /dev/null +++ b/hasura/metadata/databases/default/tables/evm_historical_stats.yaml @@ -0,0 +1,13 @@ +table: + name: historical_stats + schema: evm +select_permissions: + - role: guest + permission: + columns: + - id + - total_transactions + - tps_all_time_high + - total_incoming_token + - total_outgoing_token + filter: {} diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml index c741a2a0..daa73701 100644 --- a/hasura/metadata/databases/default/tables/tables.yaml +++ b/hasura/metadata/databases/default/tables/tables.yaml @@ -1,4 +1,5 @@ - "!include evm_block.yaml" +- "!include evm_historical_stats.yaml" - "!include evm_hyperion_state.yaml" - "!include evm_param.yaml" - "!include evm_stats.yaml" diff --git a/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql new file mode 100644 index 00000000..922df2f9 --- /dev/null +++ b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql @@ -0,0 +1,5 @@ +alter table "evm"."transaction" drop constraint "transaction_block_hash_block_number_fkey", + add constraint "transaction_block_number_block_hash_fkey" + foreign key ("block_number", "block_hash") + references "evm"."block" + ("number", "hash") on update restrict on delete restrict; diff --git a/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql new file mode 100644 index 00000000..0266f6a7 --- /dev/null +++ b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql @@ -0,0 +1,5 @@ +alter table "evm"."transaction" drop constraint "transaction_block_number_block_hash_fkey", + add constraint "transaction_block_hash_block_number_fkey" + foreign key ("block_hash", "block_number") + references "evm"."block" + ("hash", "number") on update restrict on delete cascade; diff --git a/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql new file mode 100644 index 00000000..8ab1f4c2 --- /dev/null +++ b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql @@ -0,0 +1 @@ +DROP TABLE "evm"."historical_stats"; diff --git a/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql new file mode 100644 index 00000000..49bc7329 --- /dev/null +++ b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql @@ -0,0 +1,2 @@ +CREATE TABLE "evm"."historical_stats" ("id" uuid NOT NULL DEFAULT gen_random_uuid(), "total_transactions" integer NOT NULL DEFAULT 0, "tps_all_time_high" jsonb NOT NULL DEFAULT jsonb_build_object(), "total_incoming_token" numeric NOT NULL DEFAULT 0, "total_outgoing_token" numeric NOT NULL DEFAULT 0, PRIMARY KEY ("id") , UNIQUE ("id")); +CREATE EXTENSION IF NOT EXISTS pgcrypto; From 64b6573b07f7cebfa18aa9b76fb523469d877dfb Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Wed, 16 Aug 2023 16:26:54 -0600 Subject: [PATCH 2/7] feat(hapi-evm): add logic to save historical stats --- hapi-evm/src/models/block/queries.ts | 37 ++++- hapi-evm/src/models/historical-stats/index.ts | 2 + .../src/models/historical-stats/interfaces.ts | 17 +++ .../src/models/historical-stats/queries.ts | 138 ++++++++++++++++++ hapi-evm/src/models/index.ts | 2 + hapi-evm/src/models/stats/index.ts | 2 + hapi-evm/src/models/stats/interfaces.ts | 5 + hapi-evm/src/models/stats/queries.ts | 24 +++ hapi-evm/src/models/transfer/queries.ts | 31 ++++ hapi-evm/src/services/block.service.ts | 65 ++++++++- hapi-evm/src/services/hyperion/index.ts | 2 +- hapi-evm/src/services/transfer.service.ts | 15 ++ hapi-evm/src/services/worker/index.ts | 4 + 13 files changed, 334 insertions(+), 10 deletions(-) create mode 100644 hapi-evm/src/models/historical-stats/index.ts create mode 100644 hapi-evm/src/models/historical-stats/interfaces.ts create mode 100644 hapi-evm/src/models/historical-stats/queries.ts create mode 100644 hapi-evm/src/models/stats/index.ts create mode 100644 hapi-evm/src/models/stats/interfaces.ts create mode 100644 hapi-evm/src/models/stats/queries.ts create mode 100644 hapi-evm/src/services/transfer.service.ts diff --git a/hapi-evm/src/models/block/queries.ts b/hapi-evm/src/models/block/queries.ts index 883ece5d..8110a972 100644 --- a/hapi-evm/src/models/block/queries.ts +++ b/hapi-evm/src/models/block/queries.ts @@ -1,3 +1,4 @@ +import moment from 'moment' import { gql } from 'graphql-request' import { coreUtil } from '../../utils' @@ -22,6 +23,12 @@ interface BlockInsertOneResponse { } } +interface BlockDeleteResponse { + delete_evm_block: { + affected_rows: number + } +} + const internal_get = async ( type: OperationType = Operation.query, table: TableType, @@ -104,12 +111,32 @@ export const add_or_modify = async (block: CappedBlock) => { } } ` - const { insert_evm_block_one: data } = - await coreUtil.hasura.default.request(mutation, { - evm_block: block - }) - return data + try { + const { insert_evm_block_one: data } = + await coreUtil.hasura.default.request(mutation, { + evm_block: block + }) + + return data + } catch (error) { + //console.log('ERROR', block.number) + //throw error + } +} + +export const deleteOldBlocks = async () => { + const mutation = gql` + mutation ($date: timestamptz) { + delete_evm_block(where: { timestamp: { _lt: $date } }) { + affected_rows + } + } + ` + + await coreUtil.hasura.default.request(mutation, { + date: moment().subtract(1, 'years').format('YYYY-MM-DD') + }) } export default { diff --git a/hapi-evm/src/models/historical-stats/index.ts b/hapi-evm/src/models/historical-stats/index.ts new file mode 100644 index 00000000..5e08e0f1 --- /dev/null +++ b/hapi-evm/src/models/historical-stats/index.ts @@ -0,0 +1,2 @@ +export * as interfaces from './interfaces' +export * as queries from './queries' diff --git a/hapi-evm/src/models/historical-stats/interfaces.ts b/hapi-evm/src/models/historical-stats/interfaces.ts new file mode 100644 index 00000000..69998a62 --- /dev/null +++ b/hapi-evm/src/models/historical-stats/interfaces.ts @@ -0,0 +1,17 @@ +export interface HistoricalStats { + id?: string + total_transactions?: number + total_incoming_token?: number + total_outgoing_token?: number + tps_all_time_high?: { + blocks: string[] + transactions_count: number + gas_used: number + } +} + +export interface HistoricalStatsIncInput { + total_transactions?: number + total_incoming_token?: number + total_outgoing_token?: number +} diff --git a/hapi-evm/src/models/historical-stats/queries.ts b/hapi-evm/src/models/historical-stats/queries.ts new file mode 100644 index 00000000..875c0220 --- /dev/null +++ b/hapi-evm/src/models/historical-stats/queries.ts @@ -0,0 +1,138 @@ +import { gql } from 'graphql-request' + +import { coreUtil } from '../../utils' +import { HistoricalStats, HistoricalStatsIncInput } from './interfaces' + +interface HistoricalStatsResponse { + evm_historical_stats: HistoricalStats[] +} + +interface HistoricalStatsOneResponse { + insert_evm_historical_stats_one: { + id: string + } +} + +const defaultHistoricalStats = { + id: '00000000-0000-0000-0000-000000000000', + total_transactions: 0, + total_incoming_token: 0, + total_outgoing_token: 0, + tps_all_time_high: { + blocks: [], + transactions_count: 0, + gas_used: 0 + } +} + +const save = async (payload: HistoricalStats) => { + const mutation = gql` + mutation ($payload: evm_historical_stats_insert_input!) { + insert_evm_historical_stats_one(object: $payload) { + id + } + } + ` + + const data = + await coreUtil.hasura.default.request( + mutation, + { + payload + } + ) + + return data.insert_evm_historical_stats_one +} + +const update = async ( + id: string, + inc: HistoricalStatsIncInput, + payload: HistoricalStats +) => { + const mutation = gql` + mutation ( + $id: uuid! + $inc: evm_historical_stats_inc_input + $payload: evm_historical_stats_set_input + ) { + update_evm_historical_stats_by_pk( + _inc: $inc + pk_columns: { id: $id } + _set: $payload + ) { + id + } + } + ` + + await coreUtil.hasura.default.request(mutation, { + id, + inc, + payload + }) +} + +export const getState = async () => { + const query = gql` + query { + evm_historical_stats( + where: { id: { _neq: "00000000-0000-0000-0000-000000000000" } } + limit: 1 + ) { + id + total_transactions + total_incoming_token + total_outgoing_token + tps_all_time_high + } + } + ` + const data = await coreUtil.hasura.default.request( + query + ) + + if (!data.evm_historical_stats.length) { + return defaultHistoricalStats + } + + const state = data.evm_historical_stats[0] + + return { + id: state.id || defaultHistoricalStats.id, + total_transactions: + state.total_transactions || defaultHistoricalStats.total_transactions, + total_incoming_token: + state.total_incoming_token || defaultHistoricalStats.total_incoming_token, + total_outgoing_token: + state.total_outgoing_token || defaultHistoricalStats.total_outgoing_token, + tps_all_time_high: + state.tps_all_time_high || defaultHistoricalStats.tps_all_time_high + } +} + +export const saveOrUpdate = async (payload: HistoricalStats): Promise => { + const currentState = await getState() + + if (currentState === defaultHistoricalStats) { + await save(payload) + + return + } + + await update(currentState.id, {}, payload) +} + +export const saveOrIncrement = async ( + payload: HistoricalStatsIncInput +): Promise => { + const currentState = await getState() + + if (currentState === defaultHistoricalStats) { + await save(payload) + + return + } + + await update(currentState.id, payload, {}) +} diff --git a/hapi-evm/src/models/index.ts b/hapi-evm/src/models/index.ts index a6491da5..28294e7c 100644 --- a/hapi-evm/src/models/index.ts +++ b/hapi-evm/src/models/index.ts @@ -5,3 +5,5 @@ export * as hyperionStateModel from './hyperion-state' export * as transferModel from './transfer' export * as paramModel from './param' export * as historyPayloadModel from './history-payload' +export * as historicalStatsModel from './historical-stats' +export * as StatsModel from './stats' diff --git a/hapi-evm/src/models/stats/index.ts b/hapi-evm/src/models/stats/index.ts new file mode 100644 index 00000000..5e08e0f1 --- /dev/null +++ b/hapi-evm/src/models/stats/index.ts @@ -0,0 +1,2 @@ +export * as interfaces from './interfaces' +export * as queries from './queries' diff --git a/hapi-evm/src/models/stats/interfaces.ts b/hapi-evm/src/models/stats/interfaces.ts new file mode 100644 index 00000000..ab41d856 --- /dev/null +++ b/hapi-evm/src/models/stats/interfaces.ts @@ -0,0 +1,5 @@ +export interface Stats { + ath_blocks: string + ath_transactions_count: number + ath_gas_used: number +} diff --git a/hapi-evm/src/models/stats/queries.ts b/hapi-evm/src/models/stats/queries.ts new file mode 100644 index 00000000..18ca4935 --- /dev/null +++ b/hapi-evm/src/models/stats/queries.ts @@ -0,0 +1,24 @@ +import { gql } from 'graphql-request' + +import { coreUtil } from '../../utils' +import { Stats } from './interfaces' + +interface StatsResponse { + evm_stats: Stats[] +} + +export const getPartialATH = async () => { + const query = gql` + query { + evm_stats(limit: 1) { + ath_blocks + ath_transactions_count + ath_gas_used + } + } + ` + const data = await coreUtil.hasura.default.request(query) + const state = data.evm_stats[0] + + return state +} diff --git a/hapi-evm/src/models/transfer/queries.ts b/hapi-evm/src/models/transfer/queries.ts index aa2c9157..ace45bde 100644 --- a/hapi-evm/src/models/transfer/queries.ts +++ b/hapi-evm/src/models/transfer/queries.ts @@ -1,7 +1,9 @@ +import moment from 'moment' import { gql } from 'graphql-request' import { coreUtil } from '../../utils' import { Transfer } from './interfaces' +import { historicalStatsModel } from '..' // interface TransferResponse { // evm_transfer: Transfer[] @@ -13,6 +15,12 @@ interface TransferInsertOneResponse { } } +interface TransferDeleteResponse { + delete_evm_transfer: { + affected_rows: number + } +} + export const save = async (payload: Transfer) => { const mutation = gql` mutation ($payload: evm_transfer_insert_input!) { @@ -22,6 +30,15 @@ export const save = async (payload: Transfer) => { } ` + await historicalStatsModel.queries.saveOrIncrement({ + total_incoming_token: Number(payload.type === 'incoming'), + total_outgoing_token: Number(payload.type === 'outgoing') + }) + + if (moment(payload.timestamp).isBefore(moment().subtract(1, 'years'))) { + return + } + const data = await coreUtil.hasura.default.request( mutation, { @@ -31,3 +48,17 @@ export const save = async (payload: Transfer) => { return data.insert_evm_transfer_one } + +export const deleteOldTransfers = async () => { + const mutation = gql` + mutation ($date: timestamptz) { + delete_evm_transfer(where: { timestamp: { _lt: $date } }) { + affected_rows + } + } + ` + + await coreUtil.hasura.default.request(mutation, { + date: moment().subtract(1, 'years').format('YYYY-MM-DD') + }) +} diff --git a/hapi-evm/src/services/block.service.ts b/hapi-evm/src/services/block.service.ts index 7f8a2f61..1e18f946 100644 --- a/hapi-evm/src/services/block.service.ts +++ b/hapi-evm/src/services/block.service.ts @@ -5,9 +5,12 @@ import { defaultModel, blockModel, transactionModel, - paramModel + paramModel, + historicalStatsModel, + StatsModel } from '../models' import { networkConfig } from '../config' +import moment from 'moment' const httpProvider = new Web3.providers.HttpProvider(networkConfig.evmEndpoint) const web3 = new Web3(httpProvider) @@ -26,7 +29,6 @@ const web3 = new Web3(httpProvider) // test() // TODO: syncronize passed blocks - const syncFullBlock = async (blockNumber: number | bigint) => { const block: Block = await web3.eth.getBlock(blockNumber) @@ -38,12 +40,27 @@ const syncFullBlock = async (blockNumber: number | bigint) => { if (blockExist) return + const transactionsCount = block.transactions?.length + + if (transactionsCount) { + await historicalStatsModel.queries.saveOrIncrement({ + total_transactions: transactionsCount + }) + } + + const blockTimestamp = new Date(Number(block.timestamp) * 1000) + const isBefore = moment(blockTimestamp).isBefore( + moment().subtract(1, 'years') + ) + + if (isBefore) return + const cappedBlock = { hash: block.hash.toString(), gas_used: Number(block.gasUsed), transactions: (block.transactions || []) as TransactionHash[], number: Number(block.number), - timestamp: new Date(Number(block.timestamp) * 1000) + timestamp: blockTimestamp } await blockModel.queries.add_or_modify(cappedBlock) @@ -124,6 +141,28 @@ const blockWorker = async () => { getBlock() } +const cleanOldBlocks = async () => { + await blockModel.queries.deleteOldBlocks() +} + +const syncATH = async () => { + const currentState = await historicalStatsModel.queries.getState() + const partialATH = await StatsModel.queries.getPartialATH() + + if ( + currentState.tps_all_time_high.transactions_count || + 0 < partialATH.ath_transactions_count + ) { + await historicalStatsModel.queries.saveOrUpdate({ + tps_all_time_high: { + blocks: partialATH.ath_blocks.split(','), + transactions_count: partialATH.ath_transactions_count, + gas_used: partialATH.ath_gas_used + } + }) + } +} + const syncBlockWorker = (): defaultModel.Worker => { return { name: 'SYNC BLOCK WORKER', @@ -140,7 +179,25 @@ const syncOldBlockWorker = (): defaultModel.Worker => { } } +const syncATHWorker = (): defaultModel.Worker => { + return { + name: 'SYNC ATH WORKER', + intervalSec: 60, + action: syncATH + } +} + +const cleanOldBlocksWorker = (): defaultModel.Worker => { + return { + name: 'CLEAN UP OLD BLOCKS WORKER', + intervalSec: 86400, + action: cleanOldBlocks + } +} + export default { syncBlockWorker, - syncOldBlockWorker + syncOldBlockWorker, + cleanOldBlocksWorker, + syncATHWorker } diff --git a/hapi-evm/src/services/hyperion/index.ts b/hapi-evm/src/services/hyperion/index.ts index 59c18092..aa3d95a9 100644 --- a/hapi-evm/src/services/hyperion/index.ts +++ b/hapi-evm/src/services/hyperion/index.ts @@ -2,7 +2,7 @@ import moment, { DurationInputArg2 } from 'moment' import { hyperionConfig } from '../../config' import { coreUtil, timeUtil } from '../../utils' -import { hyperionStateModel } from '../../models' +import { historicalStatsModel, hyperionStateModel } from '../../models' import updaters from './updaters' diff --git a/hapi-evm/src/services/transfer.service.ts b/hapi-evm/src/services/transfer.service.ts new file mode 100644 index 00000000..ab8faa81 --- /dev/null +++ b/hapi-evm/src/services/transfer.service.ts @@ -0,0 +1,15 @@ +import { defaultModel, transferModel } from '../models' + +const cleanOldTransfersWorker = (): defaultModel.Worker => { + return { + name: 'CLEAN UP OLD TRANSFER WORKER', + intervalSec: 86400, + action: async () => { + await transferModel.queries.deleteOldTransfers() + } + } +} + +export default { + cleanOldTransfersWorker +} diff --git a/hapi-evm/src/services/worker/index.ts b/hapi-evm/src/services/worker/index.ts index 043e7955..8da10122 100644 --- a/hapi-evm/src/services/worker/index.ts +++ b/hapi-evm/src/services/worker/index.ts @@ -1,6 +1,7 @@ import { timeUtil, coreUtil } from '../../utils' import { defaultModel } from '../../models' import blockService from '../block.service' +import transferService from '../transfer.service' import hyperionService from '../hyperion' const run = async (worker: defaultModel.Worker) => { @@ -23,6 +24,9 @@ const init = async () => { run(blockService.syncBlockWorker()) run(blockService.syncOldBlockWorker()) + run(blockService.syncATHWorker()) + run(blockService.cleanOldBlocksWorker()) + run(transferService.cleanOldTransfersWorker()) run(hyperionService.syncWorker()) } From 445d7afd9ab4ed9056e9450d33e215fcd2de8e1a Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Thu, 17 Aug 2023 09:37:59 -0600 Subject: [PATCH 3/7] chore(hasura): remove unused fields from evm stats view and add partial ath calculation --- .../databases/default/tables/evm_stats.yaml | 5 +--- .../down.sql | 30 +++++++++++++++++++ .../up.sql | 28 +++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql create mode 100644 hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql diff --git a/hasura/metadata/databases/default/tables/evm_stats.yaml b/hasura/metadata/databases/default/tables/evm_stats.yaml index 0ab926b1..ef1561bd 100644 --- a/hasura/metadata/databases/default/tables/evm_stats.yaml +++ b/hasura/metadata/databases/default/tables/evm_stats.yaml @@ -5,9 +5,6 @@ select_permissions: - role: guest permission: columns: - - ath_transaction_sum - - daily_transaction_count - - incoming_tlos_count - - outgoing_tlos_count - block_gas_avg + - daily_transaction_count filter: {} diff --git a/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql new file mode 100644 index 00000000..839dc42d --- /dev/null +++ b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql @@ -0,0 +1,30 @@ +-- Could not auto-generate a down migration. +-- Please write an appropriate down migration for the SQL below: +-- DROP VIEW IF EXISTS "evm"."stats"; +-- CREATE OR REPLACE VIEW "evm"."stats" AS +-- SELECT COALESCE(evm_block.avg_gas_used, (0)::numeric) AS block_gas_avg, +-- COALESCE(daily_transactions.total_transaction_count, (0)::bigint) AS daily_transaction_count, +-- COALESCE(max_transaction_sum, (0)::numeric) AS ath_transactions_count, +-- COALESCE(gas_used_sum, (0)::numeric) AS ath_gas_used, +-- blocks AS ath_blocks +-- FROM (((( SELECT avg(subquery_alias.gas_used) AS avg_gas_used +-- FROM ( SELECT block.gas_used, block."timestamp" +-- FROM evm.block +-- ORDER BY block."timestamp" DESC +-- LIMIT 100) subquery_alias) evm_block +-- CROSS JOIN LATERAL ( +-- SELECT sum(jsonb_array_length(block.transactions)) AS total_transaction_count +-- FROM evm.block +-- WHERE (block."timestamp" >= (now() - '24:00:00'::interval))) daily_transactions) +-- CROSS JOIN LATERAL ( +-- WITH subquery AS( +-- SELECT array_to_string(array_agg(evm.block.number), ',') AS blocks , +-- sum(jsonb_array_length(evm.block.transactions)) AS total_transaction_count, +-- sum(evm.block.gas_used) AS gas_used_sum +-- FROM evm.block +-- GROUP BY timestamp) +-- SELECT blocks, max_transaction_sum, gas_used_sum +-- FROM ( SELECT max(subquery.total_transaction_count) AS max_transaction_sum +-- FROM subquery) q1 +-- INNER JOIN subquery q2 ON q1.max_transaction_sum = q2.total_transaction_count +-- LIMIT 1) ath_last_year)); diff --git a/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql new file mode 100644 index 00000000..91993b63 --- /dev/null +++ b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql @@ -0,0 +1,28 @@ +DROP VIEW IF EXISTS "evm"."stats"; +CREATE OR REPLACE VIEW "evm"."stats" AS + SELECT COALESCE(evm_block.avg_gas_used, (0)::numeric) AS block_gas_avg, + COALESCE(daily_transactions.total_transaction_count, (0)::bigint) AS daily_transaction_count, + COALESCE(max_transaction_sum, (0)::numeric) AS ath_transactions_count, + COALESCE(gas_used_sum, (0)::numeric) AS ath_gas_used, + blocks AS ath_blocks + FROM (((( SELECT avg(subquery_alias.gas_used) AS avg_gas_used + FROM ( SELECT block.gas_used, block."timestamp" + FROM evm.block + ORDER BY block."timestamp" DESC + LIMIT 100) subquery_alias) evm_block + CROSS JOIN LATERAL ( + SELECT sum(jsonb_array_length(block.transactions)) AS total_transaction_count + FROM evm.block + WHERE (block."timestamp" >= (now() - '24:00:00'::interval))) daily_transactions) + CROSS JOIN LATERAL ( + WITH subquery AS( + SELECT array_to_string(array_agg(evm.block.number), ',') AS blocks , + sum(jsonb_array_length(evm.block.transactions)) AS total_transaction_count, + sum(evm.block.gas_used) AS gas_used_sum + FROM evm.block + GROUP BY timestamp) + SELECT blocks, max_transaction_sum, gas_used_sum + FROM ( SELECT max(subquery.total_transaction_count) AS max_transaction_sum + FROM subquery) q1 + INNER JOIN subquery q2 ON q1.max_transaction_sum = q2.total_transaction_count + LIMIT 1) ath_last_year)); From 3e84eda08306fef904e391fa72ee7177407919e0 Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Thu, 17 Aug 2023 09:45:13 -0600 Subject: [PATCH 4/7] chore(hapi-evm): remove unused code and add conditional when partialATH is undefined --- hapi-evm/src/models/block/queries.ts | 16 +++++----------- hapi-evm/src/services/block.service.ts | 2 ++ hapi-evm/src/services/hyperion/index.ts | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/hapi-evm/src/models/block/queries.ts b/hapi-evm/src/models/block/queries.ts index 8110a972..983a7c97 100644 --- a/hapi-evm/src/models/block/queries.ts +++ b/hapi-evm/src/models/block/queries.ts @@ -111,18 +111,12 @@ export const add_or_modify = async (block: CappedBlock) => { } } ` + const { insert_evm_block_one: data } = + await coreUtil.hasura.default.request(mutation, { + evm_block: block + }) - try { - const { insert_evm_block_one: data } = - await coreUtil.hasura.default.request(mutation, { - evm_block: block - }) - - return data - } catch (error) { - //console.log('ERROR', block.number) - //throw error - } + return data } export const deleteOldBlocks = async () => { diff --git a/hapi-evm/src/services/block.service.ts b/hapi-evm/src/services/block.service.ts index 1e18f946..3621c3ad 100644 --- a/hapi-evm/src/services/block.service.ts +++ b/hapi-evm/src/services/block.service.ts @@ -149,6 +149,8 @@ const syncATH = async () => { const currentState = await historicalStatsModel.queries.getState() const partialATH = await StatsModel.queries.getPartialATH() + if (!partialATH) return + if ( currentState.tps_all_time_high.transactions_count || 0 < partialATH.ath_transactions_count diff --git a/hapi-evm/src/services/hyperion/index.ts b/hapi-evm/src/services/hyperion/index.ts index aa3d95a9..59c18092 100644 --- a/hapi-evm/src/services/hyperion/index.ts +++ b/hapi-evm/src/services/hyperion/index.ts @@ -2,7 +2,7 @@ import moment, { DurationInputArg2 } from 'moment' import { hyperionConfig } from '../../config' import { coreUtil, timeUtil } from '../../utils' -import { historicalStatsModel, hyperionStateModel } from '../../models' +import { hyperionStateModel } from '../../models' import updaters from './updaters' From 36bcf545a295b8ab1d0a0352b01a042cc17f1cb6 Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Thu, 17 Aug 2023 14:23:22 -0600 Subject: [PATCH 5/7] fix(hapi-evm): use the DB to get the last block to avoid missing blocks --- hapi-evm/src/models/block/queries.ts | 20 +++++++++++++++----- hapi-evm/src/services/block.service.ts | 12 +++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/hapi-evm/src/models/block/queries.ts b/hapi-evm/src/models/block/queries.ts index 983a7c97..0982b0a5 100644 --- a/hapi-evm/src/models/block/queries.ts +++ b/hapi-evm/src/models/block/queries.ts @@ -36,19 +36,23 @@ const internal_get = async ( // TODO: not only accept where but also additional content // such as limit, order, etc where: object, + order: object | null, attributes: string, operation?: string ): Promise => { const query = gql` ${type} (${parameters}) { - ${table}${operation ? `_${operation}` : ''}(where: $where) { + ${table}${ + operation ? `_${operation}` : '' + }(where: $where, order_by: $order) { ${attributes} } } ` return await coreUtil.hasura.default.request(query, { - where + where, + order }) } @@ -56,12 +60,13 @@ export const exist = async (hashOrNumber: string | number) => { const result = await internal_get( 'query', 'evm_block', - '$where: evm_block_bool_exp!', + '$where: evm_block_bool_exp!, $order: [evm_block_order_by!]', { [typeof hashOrNumber === 'string' ? 'hash' : 'number']: { _eq: hashOrNumber } }, + null, 'aggregate { count }', 'aggregate' ) @@ -69,12 +74,17 @@ export const exist = async (hashOrNumber: string | number) => { return result.evm_block_aggregate.aggregate.count > 0 } -const get = async (where: object, many = false) => { +const get = async ( + where: object, + order: object | null = null, + many = false +) => { const result = await internal_get( 'query', 'evm_block', - '$where: evm_block_bool_exp!', + '$where: evm_block_bool_exp!, $order: [evm_block_order_by!]', where, + order, 'hash, gas_used, transactions, number, timestamp' ) diff --git a/hapi-evm/src/services/block.service.ts b/hapi-evm/src/services/block.service.ts index 3621c3ad..458563ab 100644 --- a/hapi-evm/src/services/block.service.ts +++ b/hapi-evm/src/services/block.service.ts @@ -103,7 +103,17 @@ const syncFullBlock = async (blockNumber: number | bigint) => { } const getBlock = async () => { - const blockNumber = await web3.eth.getBlockNumber() + let blockNumber: bigint + const lastBlockInDB = (await blockModel.queries.default.get( + { timestamp: { _gt: moment().subtract(30, 'minutes') } }, + { number: 'desc' } + )) as blockModel.interfaces.CappedBlock + + if (!lastBlockInDB) { + blockNumber = await web3.eth.getBlockNumber() + } else { + blockNumber = BigInt(lastBlockInDB.number + 1) + } await syncFullBlock(blockNumber) } From 305531b9266af907fa6ee107b5d24661109455e0 Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Thu, 17 Aug 2023 14:33:19 -0600 Subject: [PATCH 6/7] refactor(hapi-evm): use Type to compare the type of transfers --- hapi-evm/src/models/transfer/queries.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hapi-evm/src/models/transfer/queries.ts b/hapi-evm/src/models/transfer/queries.ts index ace45bde..e763791e 100644 --- a/hapi-evm/src/models/transfer/queries.ts +++ b/hapi-evm/src/models/transfer/queries.ts @@ -2,7 +2,7 @@ import moment from 'moment' import { gql } from 'graphql-request' import { coreUtil } from '../../utils' -import { Transfer } from './interfaces' +import { Transfer, Type } from './interfaces' import { historicalStatsModel } from '..' // interface TransferResponse { @@ -31,8 +31,8 @@ export const save = async (payload: Transfer) => { ` await historicalStatsModel.queries.saveOrIncrement({ - total_incoming_token: Number(payload.type === 'incoming'), - total_outgoing_token: Number(payload.type === 'outgoing') + total_incoming_token: Number(payload.type === Type.incoming), + total_outgoing_token: Number(payload.type === Type.outgoing) }) if (moment(payload.timestamp).isBefore(moment().subtract(1, 'years'))) { From 5a1f787a0231bd9cd796c36cda1d6826f217e3b3 Mon Sep 17 00:00:00 2001 From: Torresmorah Date: Tue, 22 Aug 2023 13:53:39 -0600 Subject: [PATCH 7/7] fix(hapi-evm): fix token history query --- hapi-evm/src/services/token-history.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hapi-evm/src/services/token-history.service.ts b/hapi-evm/src/services/token-history.service.ts index 7c9ba67c..b22ea549 100644 --- a/hapi-evm/src/services/token-history.service.ts +++ b/hapi-evm/src/services/token-history.service.ts @@ -15,8 +15,8 @@ export const getTokenHistory = async (range: string) => { SELECT interval.value as datetime, - sum(CASE WHEN transfer.type = 'incoming' THEN transfer.amount END)::numeric as incoming, - sum(CASE WHEN transfer.type = 'outgoing' THEN transfer.amount END)::numeric as outgoing + COUNT(CASE WHEN transfer.type = 'incoming' THEN 1 END) as incoming, + COUNT(CASE WHEN transfer.type = 'outgoing' THEN 1 END) as outgoing FROM interval LEFT JOIN