diff --git a/hapi-evm/src/models/block/queries.ts b/hapi-evm/src/models/block/queries.ts index 883ece5d..0982b0a5 100644 --- a/hapi-evm/src/models/block/queries.ts +++ b/hapi-evm/src/models/block/queries.ts @@ -1,3 +1,4 @@ +import moment from 'moment' import { gql } from 'graphql-request' import { coreUtil } from '../../utils' @@ -22,6 +23,12 @@ interface BlockInsertOneResponse { } } +interface BlockDeleteResponse { + delete_evm_block: { + affected_rows: number + } +} + const internal_get = async ( type: OperationType = Operation.query, table: TableType, @@ -29,19 +36,23 @@ const internal_get = async ( // TODO: not only accept where but also additional content // such as limit, order, etc where: object, + order: object | null, attributes: string, operation?: string ): Promise => { const query = gql` ${type} (${parameters}) { - ${table}${operation ? `_${operation}` : ''}(where: $where) { + ${table}${ + operation ? `_${operation}` : '' + }(where: $where, order_by: $order) { ${attributes} } } ` return await coreUtil.hasura.default.request(query, { - where + where, + order }) } @@ -49,12 +60,13 @@ export const exist = async (hashOrNumber: string | number) => { const result = await internal_get( 'query', 'evm_block', - '$where: evm_block_bool_exp!', + '$where: evm_block_bool_exp!, $order: [evm_block_order_by!]', { [typeof hashOrNumber === 'string' ? 'hash' : 'number']: { _eq: hashOrNumber } }, + null, 'aggregate { count }', 'aggregate' ) @@ -62,12 +74,17 @@ export const exist = async (hashOrNumber: string | number) => { return result.evm_block_aggregate.aggregate.count > 0 } -const get = async (where: object, many = false) => { +const get = async ( + where: object, + order: object | null = null, + many = false +) => { const result = await internal_get( 'query', 'evm_block', - '$where: evm_block_bool_exp!', + '$where: evm_block_bool_exp!, $order: [evm_block_order_by!]', where, + order, 'hash, gas_used, transactions, number, timestamp' ) @@ -112,6 +129,20 @@ export const add_or_modify = async (block: CappedBlock) => { return data } +export const deleteOldBlocks = async () => { + const mutation = gql` + mutation ($date: timestamptz) { + delete_evm_block(where: { timestamp: { _lt: $date } }) { + affected_rows + } + } + ` + + await coreUtil.hasura.default.request(mutation, { + date: moment().subtract(1, 'years').format('YYYY-MM-DD') + }) +} + export default { exist, get, diff --git a/hapi-evm/src/models/historical-stats/index.ts b/hapi-evm/src/models/historical-stats/index.ts new file mode 100644 index 00000000..5e08e0f1 --- /dev/null +++ b/hapi-evm/src/models/historical-stats/index.ts @@ -0,0 +1,2 @@ +export * as interfaces from './interfaces' +export * as queries from './queries' diff --git a/hapi-evm/src/models/historical-stats/interfaces.ts b/hapi-evm/src/models/historical-stats/interfaces.ts new file mode 100644 index 00000000..69998a62 --- /dev/null +++ b/hapi-evm/src/models/historical-stats/interfaces.ts @@ -0,0 +1,17 @@ +export interface HistoricalStats { + id?: string + total_transactions?: number + total_incoming_token?: number + total_outgoing_token?: number + tps_all_time_high?: { + blocks: string[] + transactions_count: number + gas_used: number + } +} + +export interface HistoricalStatsIncInput { + total_transactions?: number + total_incoming_token?: number + total_outgoing_token?: number +} diff --git a/hapi-evm/src/models/historical-stats/queries.ts b/hapi-evm/src/models/historical-stats/queries.ts new file mode 100644 index 00000000..875c0220 --- /dev/null +++ b/hapi-evm/src/models/historical-stats/queries.ts @@ -0,0 +1,138 @@ +import { gql } from 'graphql-request' + +import { coreUtil } from '../../utils' +import { HistoricalStats, HistoricalStatsIncInput } from './interfaces' + +interface HistoricalStatsResponse { + evm_historical_stats: HistoricalStats[] +} + +interface HistoricalStatsOneResponse { + insert_evm_historical_stats_one: { + id: string + } +} + +const defaultHistoricalStats = { + id: '00000000-0000-0000-0000-000000000000', + total_transactions: 0, + total_incoming_token: 0, + total_outgoing_token: 0, + tps_all_time_high: { + blocks: [], + transactions_count: 0, + gas_used: 0 + } +} + +const save = async (payload: HistoricalStats) => { + const mutation = gql` + mutation ($payload: evm_historical_stats_insert_input!) { + insert_evm_historical_stats_one(object: $payload) { + id + } + } + ` + + const data = + await coreUtil.hasura.default.request( + mutation, + { + payload + } + ) + + return data.insert_evm_historical_stats_one +} + +const update = async ( + id: string, + inc: HistoricalStatsIncInput, + payload: HistoricalStats +) => { + const mutation = gql` + mutation ( + $id: uuid! + $inc: evm_historical_stats_inc_input + $payload: evm_historical_stats_set_input + ) { + update_evm_historical_stats_by_pk( + _inc: $inc + pk_columns: { id: $id } + _set: $payload + ) { + id + } + } + ` + + await coreUtil.hasura.default.request(mutation, { + id, + inc, + payload + }) +} + +export const getState = async () => { + const query = gql` + query { + evm_historical_stats( + where: { id: { _neq: "00000000-0000-0000-0000-000000000000" } } + limit: 1 + ) { + id + total_transactions + total_incoming_token + total_outgoing_token + tps_all_time_high + } + } + ` + const data = await coreUtil.hasura.default.request( + query + ) + + if (!data.evm_historical_stats.length) { + return defaultHistoricalStats + } + + const state = data.evm_historical_stats[0] + + return { + id: state.id || defaultHistoricalStats.id, + total_transactions: + state.total_transactions || defaultHistoricalStats.total_transactions, + total_incoming_token: + state.total_incoming_token || defaultHistoricalStats.total_incoming_token, + total_outgoing_token: + state.total_outgoing_token || defaultHistoricalStats.total_outgoing_token, + tps_all_time_high: + state.tps_all_time_high || defaultHistoricalStats.tps_all_time_high + } +} + +export const saveOrUpdate = async (payload: HistoricalStats): Promise => { + const currentState = await getState() + + if (currentState === defaultHistoricalStats) { + await save(payload) + + return + } + + await update(currentState.id, {}, payload) +} + +export const saveOrIncrement = async ( + payload: HistoricalStatsIncInput +): Promise => { + const currentState = await getState() + + if (currentState === defaultHistoricalStats) { + await save(payload) + + return + } + + await update(currentState.id, payload, {}) +} diff --git a/hapi-evm/src/models/index.ts b/hapi-evm/src/models/index.ts index a6491da5..28294e7c 100644 --- a/hapi-evm/src/models/index.ts +++ b/hapi-evm/src/models/index.ts @@ -5,3 +5,5 @@ export * as hyperionStateModel from './hyperion-state' export * as transferModel from './transfer' export * as paramModel from './param' export * as historyPayloadModel from './history-payload' +export * as historicalStatsModel from './historical-stats' +export * as StatsModel from './stats' diff --git a/hapi-evm/src/models/stats/index.ts b/hapi-evm/src/models/stats/index.ts new file mode 100644 index 00000000..5e08e0f1 --- /dev/null +++ b/hapi-evm/src/models/stats/index.ts @@ -0,0 +1,2 @@ +export * as interfaces from './interfaces' +export * as queries from './queries' diff --git a/hapi-evm/src/models/stats/interfaces.ts b/hapi-evm/src/models/stats/interfaces.ts new file mode 100644 index 00000000..ab41d856 --- /dev/null +++ b/hapi-evm/src/models/stats/interfaces.ts @@ -0,0 +1,5 @@ +export interface Stats { + ath_blocks: string + ath_transactions_count: number + ath_gas_used: number +} diff --git a/hapi-evm/src/models/stats/queries.ts b/hapi-evm/src/models/stats/queries.ts new file mode 100644 index 00000000..18ca4935 --- /dev/null +++ b/hapi-evm/src/models/stats/queries.ts @@ -0,0 +1,24 @@ +import { gql } from 'graphql-request' + +import { coreUtil } from '../../utils' +import { Stats } from './interfaces' + +interface StatsResponse { + evm_stats: Stats[] +} + +export const getPartialATH = async () => { + const query = gql` + query { + evm_stats(limit: 1) { + ath_blocks + ath_transactions_count + ath_gas_used + } + } + ` + const data = await coreUtil.hasura.default.request(query) + const state = data.evm_stats[0] + + return state +} diff --git a/hapi-evm/src/models/transfer/queries.ts b/hapi-evm/src/models/transfer/queries.ts index aa2c9157..e763791e 100644 --- a/hapi-evm/src/models/transfer/queries.ts +++ b/hapi-evm/src/models/transfer/queries.ts @@ -1,7 +1,9 @@ +import moment from 'moment' import { gql } from 'graphql-request' import { coreUtil } from '../../utils' -import { Transfer } from './interfaces' +import { Transfer, Type } from './interfaces' +import { historicalStatsModel } from '..' // interface TransferResponse { // evm_transfer: Transfer[] @@ -13,6 +15,12 @@ interface TransferInsertOneResponse { } } +interface TransferDeleteResponse { + delete_evm_transfer: { + affected_rows: number + } +} + export const save = async (payload: Transfer) => { const mutation = gql` mutation ($payload: evm_transfer_insert_input!) { @@ -22,6 +30,15 @@ export const save = async (payload: Transfer) => { } ` + await historicalStatsModel.queries.saveOrIncrement({ + total_incoming_token: Number(payload.type === Type.incoming), + total_outgoing_token: Number(payload.type === Type.outgoing) + }) + + if (moment(payload.timestamp).isBefore(moment().subtract(1, 'years'))) { + return + } + const data = await coreUtil.hasura.default.request( mutation, { @@ -31,3 +48,17 @@ export const save = async (payload: Transfer) => { return data.insert_evm_transfer_one } + +export const deleteOldTransfers = async () => { + const mutation = gql` + mutation ($date: timestamptz) { + delete_evm_transfer(where: { timestamp: { _lt: $date } }) { + affected_rows + } + } + ` + + await coreUtil.hasura.default.request(mutation, { + date: moment().subtract(1, 'years').format('YYYY-MM-DD') + }) +} diff --git a/hapi-evm/src/services/block.service.ts b/hapi-evm/src/services/block.service.ts index 7f8a2f61..458563ab 100644 --- a/hapi-evm/src/services/block.service.ts +++ b/hapi-evm/src/services/block.service.ts @@ -5,9 +5,12 @@ import { defaultModel, blockModel, transactionModel, - paramModel + paramModel, + historicalStatsModel, + StatsModel } from '../models' import { networkConfig } from '../config' +import moment from 'moment' const httpProvider = new Web3.providers.HttpProvider(networkConfig.evmEndpoint) const web3 = new Web3(httpProvider) @@ -26,7 +29,6 @@ const web3 = new Web3(httpProvider) // test() // TODO: syncronize passed blocks - const syncFullBlock = async (blockNumber: number | bigint) => { const block: Block = await web3.eth.getBlock(blockNumber) @@ -38,12 +40,27 @@ const syncFullBlock = async (blockNumber: number | bigint) => { if (blockExist) return + const transactionsCount = block.transactions?.length + + if (transactionsCount) { + await historicalStatsModel.queries.saveOrIncrement({ + total_transactions: transactionsCount + }) + } + + const blockTimestamp = new Date(Number(block.timestamp) * 1000) + const isBefore = moment(blockTimestamp).isBefore( + moment().subtract(1, 'years') + ) + + if (isBefore) return + const cappedBlock = { hash: block.hash.toString(), gas_used: Number(block.gasUsed), transactions: (block.transactions || []) as TransactionHash[], number: Number(block.number), - timestamp: new Date(Number(block.timestamp) * 1000) + timestamp: blockTimestamp } await blockModel.queries.add_or_modify(cappedBlock) @@ -86,7 +103,17 @@ const syncFullBlock = async (blockNumber: number | bigint) => { } const getBlock = async () => { - const blockNumber = await web3.eth.getBlockNumber() + let blockNumber: bigint + const lastBlockInDB = (await blockModel.queries.default.get( + { timestamp: { _gt: moment().subtract(30, 'minutes') } }, + { number: 'desc' } + )) as blockModel.interfaces.CappedBlock + + if (!lastBlockInDB) { + blockNumber = await web3.eth.getBlockNumber() + } else { + blockNumber = BigInt(lastBlockInDB.number + 1) + } await syncFullBlock(blockNumber) } @@ -124,6 +151,30 @@ const blockWorker = async () => { getBlock() } +const cleanOldBlocks = async () => { + await blockModel.queries.deleteOldBlocks() +} + +const syncATH = async () => { + const currentState = await historicalStatsModel.queries.getState() + const partialATH = await StatsModel.queries.getPartialATH() + + if (!partialATH) return + + if ( + currentState.tps_all_time_high.transactions_count || + 0 < partialATH.ath_transactions_count + ) { + await historicalStatsModel.queries.saveOrUpdate({ + tps_all_time_high: { + blocks: partialATH.ath_blocks.split(','), + transactions_count: partialATH.ath_transactions_count, + gas_used: partialATH.ath_gas_used + } + }) + } +} + const syncBlockWorker = (): defaultModel.Worker => { return { name: 'SYNC BLOCK WORKER', @@ -140,7 +191,25 @@ const syncOldBlockWorker = (): defaultModel.Worker => { } } +const syncATHWorker = (): defaultModel.Worker => { + return { + name: 'SYNC ATH WORKER', + intervalSec: 60, + action: syncATH + } +} + +const cleanOldBlocksWorker = (): defaultModel.Worker => { + return { + name: 'CLEAN UP OLD BLOCKS WORKER', + intervalSec: 86400, + action: cleanOldBlocks + } +} + export default { syncBlockWorker, - syncOldBlockWorker + syncOldBlockWorker, + cleanOldBlocksWorker, + syncATHWorker } diff --git a/hapi-evm/src/services/token-history.service.ts b/hapi-evm/src/services/token-history.service.ts index 7c9ba67c..b22ea549 100644 --- a/hapi-evm/src/services/token-history.service.ts +++ b/hapi-evm/src/services/token-history.service.ts @@ -15,8 +15,8 @@ export const getTokenHistory = async (range: string) => { SELECT interval.value as datetime, - sum(CASE WHEN transfer.type = 'incoming' THEN transfer.amount END)::numeric as incoming, - sum(CASE WHEN transfer.type = 'outgoing' THEN transfer.amount END)::numeric as outgoing + COUNT(CASE WHEN transfer.type = 'incoming' THEN 1 END) as incoming, + COUNT(CASE WHEN transfer.type = 'outgoing' THEN 1 END) as outgoing FROM interval LEFT JOIN diff --git a/hapi-evm/src/services/transfer.service.ts b/hapi-evm/src/services/transfer.service.ts new file mode 100644 index 00000000..ab8faa81 --- /dev/null +++ b/hapi-evm/src/services/transfer.service.ts @@ -0,0 +1,15 @@ +import { defaultModel, transferModel } from '../models' + +const cleanOldTransfersWorker = (): defaultModel.Worker => { + return { + name: 'CLEAN UP OLD TRANSFER WORKER', + intervalSec: 86400, + action: async () => { + await transferModel.queries.deleteOldTransfers() + } + } +} + +export default { + cleanOldTransfersWorker +} diff --git a/hapi-evm/src/services/worker/index.ts b/hapi-evm/src/services/worker/index.ts index 043e7955..8da10122 100644 --- a/hapi-evm/src/services/worker/index.ts +++ b/hapi-evm/src/services/worker/index.ts @@ -1,6 +1,7 @@ import { timeUtil, coreUtil } from '../../utils' import { defaultModel } from '../../models' import blockService from '../block.service' +import transferService from '../transfer.service' import hyperionService from '../hyperion' const run = async (worker: defaultModel.Worker) => { @@ -23,6 +24,9 @@ const init = async () => { run(blockService.syncBlockWorker()) run(blockService.syncOldBlockWorker()) + run(blockService.syncATHWorker()) + run(blockService.cleanOldBlocksWorker()) + run(transferService.cleanOldTransfersWorker()) run(hyperionService.syncWorker()) } diff --git a/hasura/metadata/databases/default/tables/evm_historical_stats.yaml b/hasura/metadata/databases/default/tables/evm_historical_stats.yaml new file mode 100644 index 00000000..db99c245 --- /dev/null +++ b/hasura/metadata/databases/default/tables/evm_historical_stats.yaml @@ -0,0 +1,13 @@ +table: + name: historical_stats + schema: evm +select_permissions: + - role: guest + permission: + columns: + - id + - total_transactions + - tps_all_time_high + - total_incoming_token + - total_outgoing_token + filter: {} diff --git a/hasura/metadata/databases/default/tables/evm_stats.yaml b/hasura/metadata/databases/default/tables/evm_stats.yaml index 0ab926b1..ef1561bd 100644 --- a/hasura/metadata/databases/default/tables/evm_stats.yaml +++ b/hasura/metadata/databases/default/tables/evm_stats.yaml @@ -5,9 +5,6 @@ select_permissions: - role: guest permission: columns: - - ath_transaction_sum - - daily_transaction_count - - incoming_tlos_count - - outgoing_tlos_count - block_gas_avg + - daily_transaction_count filter: {} diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml index c741a2a0..daa73701 100644 --- a/hasura/metadata/databases/default/tables/tables.yaml +++ b/hasura/metadata/databases/default/tables/tables.yaml @@ -1,4 +1,5 @@ - "!include evm_block.yaml" +- "!include evm_historical_stats.yaml" - "!include evm_hyperion_state.yaml" - "!include evm_param.yaml" - "!include evm_stats.yaml" diff --git a/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql new file mode 100644 index 00000000..922df2f9 --- /dev/null +++ b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/down.sql @@ -0,0 +1,5 @@ +alter table "evm"."transaction" drop constraint "transaction_block_hash_block_number_fkey", + add constraint "transaction_block_number_block_hash_fkey" + foreign key ("block_number", "block_hash") + references "evm"."block" + ("number", "hash") on update restrict on delete restrict; diff --git a/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql new file mode 100644 index 00000000..0266f6a7 --- /dev/null +++ b/hasura/migrations/default/1692117604338_set_fk_evm_transaction_block_hash_block_number/up.sql @@ -0,0 +1,5 @@ +alter table "evm"."transaction" drop constraint "transaction_block_number_block_hash_fkey", + add constraint "transaction_block_hash_block_number_fkey" + foreign key ("block_hash", "block_number") + references "evm"."block" + ("hash", "number") on update restrict on delete cascade; diff --git a/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql new file mode 100644 index 00000000..8ab1f4c2 --- /dev/null +++ b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/down.sql @@ -0,0 +1 @@ +DROP TABLE "evm"."historical_stats"; diff --git a/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql new file mode 100644 index 00000000..49bc7329 --- /dev/null +++ b/hasura/migrations/default/1692132948056_create_table_evm_historical_stats/up.sql @@ -0,0 +1,2 @@ +CREATE TABLE "evm"."historical_stats" ("id" uuid NOT NULL DEFAULT gen_random_uuid(), "total_transactions" integer NOT NULL DEFAULT 0, "tps_all_time_high" jsonb NOT NULL DEFAULT jsonb_build_object(), "total_incoming_token" numeric NOT NULL DEFAULT 0, "total_outgoing_token" numeric NOT NULL DEFAULT 0, PRIMARY KEY ("id") , UNIQUE ("id")); +CREATE EXTENSION IF NOT EXISTS pgcrypto; diff --git a/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql new file mode 100644 index 00000000..839dc42d --- /dev/null +++ b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/down.sql @@ -0,0 +1,30 @@ +-- Could not auto-generate a down migration. +-- Please write an appropriate down migration for the SQL below: +-- DROP VIEW IF EXISTS "evm"."stats"; +-- CREATE OR REPLACE VIEW "evm"."stats" AS +-- SELECT COALESCE(evm_block.avg_gas_used, (0)::numeric) AS block_gas_avg, +-- COALESCE(daily_transactions.total_transaction_count, (0)::bigint) AS daily_transaction_count, +-- COALESCE(max_transaction_sum, (0)::numeric) AS ath_transactions_count, +-- COALESCE(gas_used_sum, (0)::numeric) AS ath_gas_used, +-- blocks AS ath_blocks +-- FROM (((( SELECT avg(subquery_alias.gas_used) AS avg_gas_used +-- FROM ( SELECT block.gas_used, block."timestamp" +-- FROM evm.block +-- ORDER BY block."timestamp" DESC +-- LIMIT 100) subquery_alias) evm_block +-- CROSS JOIN LATERAL ( +-- SELECT sum(jsonb_array_length(block.transactions)) AS total_transaction_count +-- FROM evm.block +-- WHERE (block."timestamp" >= (now() - '24:00:00'::interval))) daily_transactions) +-- CROSS JOIN LATERAL ( +-- WITH subquery AS( +-- SELECT array_to_string(array_agg(evm.block.number), ',') AS blocks , +-- sum(jsonb_array_length(evm.block.transactions)) AS total_transaction_count, +-- sum(evm.block.gas_used) AS gas_used_sum +-- FROM evm.block +-- GROUP BY timestamp) +-- SELECT blocks, max_transaction_sum, gas_used_sum +-- FROM ( SELECT max(subquery.total_transaction_count) AS max_transaction_sum +-- FROM subquery) q1 +-- INNER JOIN subquery q2 ON q1.max_transaction_sum = q2.total_transaction_count +-- LIMIT 1) ath_last_year)); diff --git a/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql new file mode 100644 index 00000000..91993b63 --- /dev/null +++ b/hasura/migrations/default/1692286077976_create_evm_stats_view_v4/up.sql @@ -0,0 +1,28 @@ +DROP VIEW IF EXISTS "evm"."stats"; +CREATE OR REPLACE VIEW "evm"."stats" AS + SELECT COALESCE(evm_block.avg_gas_used, (0)::numeric) AS block_gas_avg, + COALESCE(daily_transactions.total_transaction_count, (0)::bigint) AS daily_transaction_count, + COALESCE(max_transaction_sum, (0)::numeric) AS ath_transactions_count, + COALESCE(gas_used_sum, (0)::numeric) AS ath_gas_used, + blocks AS ath_blocks + FROM (((( SELECT avg(subquery_alias.gas_used) AS avg_gas_used + FROM ( SELECT block.gas_used, block."timestamp" + FROM evm.block + ORDER BY block."timestamp" DESC + LIMIT 100) subquery_alias) evm_block + CROSS JOIN LATERAL ( + SELECT sum(jsonb_array_length(block.transactions)) AS total_transaction_count + FROM evm.block + WHERE (block."timestamp" >= (now() - '24:00:00'::interval))) daily_transactions) + CROSS JOIN LATERAL ( + WITH subquery AS( + SELECT array_to_string(array_agg(evm.block.number), ',') AS blocks , + sum(jsonb_array_length(evm.block.transactions)) AS total_transaction_count, + sum(evm.block.gas_used) AS gas_used_sum + FROM evm.block + GROUP BY timestamp) + SELECT blocks, max_transaction_sum, gas_used_sum + FROM ( SELECT max(subquery.total_transaction_count) AS max_transaction_sum + FROM subquery) q1 + INNER JOIN subquery q2 ON q1.max_transaction_sum = q2.total_transaction_count + LIMIT 1) ath_last_year));