Skip to content

Commit 530acef

Browse files
kaladinlightkaladinlight
authored
feat: make unchained great again (#908)
1 parent 3e633ce commit 530acef

File tree

31 files changed

+159
-152
lines changed

31 files changed

+159
-152
lines changed

‎.circleci/config.yml

+20-6
Original file line numberDiff line numberDiff line change
@@ -609,12 +609,26 @@ aliases:
609609
api-memory-limit: 1Gi
610610
api-memory-request: 500Mi
611611
stateful-service-replicas: 2
612-
service-name-1: indexer
613-
service-image-1: shapeshiftdao/unchained-blockbook:559cfbc
614-
service-cpu-limit-1: "4"
615-
service-cpu-request-1: "2"
616-
service-memory-limit-1: 24Gi
617-
service-storage-size-1: 750Gi
612+
service-name-1: daemon
613+
service-image-1: 0xpolygon/bor:1.2.0
614+
service-cpu-limit-1: "8"
615+
service-cpu-request-1: "4"
616+
service-memory-limit-1: 48Gi
617+
service-storage-size-1: 4000Gi
618+
service-storage-iops-1: "6000"
619+
service-storage-throughput-1: "300"
620+
service-name-2: heimdall
621+
service-image-2: 0xpolygon/heimdall:1.0.3
622+
service-cpu-limit-2: "2"
623+
service-cpu-request-2: "1"
624+
service-memory-limit-2: 1Gi
625+
service-storage-size-2: 400Gi
626+
service-name-3: indexer
627+
service-image-3: shapeshiftdao/unchained-blockbook:559cfbc
628+
service-cpu-limit-3: "4"
629+
service-cpu-request-3: "2"
630+
service-memory-limit-3: 24Gi
631+
service-storage-size-3: 750Gi
618632

619633
- &polygon-dev
620634
<<: *polygon

‎node/coinstacks/arbitrum-nova/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
8080
const wsServer = new Server({ server })
8181

8282
wsServer.on('connection', (connection) => {
83-
ConnectionHandler.start(connection, registry, prometheus)
83+
ConnectionHandler.start(connection, registry, prometheus, logger)
8484
})
8585

8686
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/arbitrum-nova/api/src/controller.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export const logger = new Logger({
3838
level: process.env.LOG_LEVEL,
3939
})
4040

41-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
41+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
4242
const provider = new ethers.providers.JsonRpcProvider(RPC_URL)
4343
export const gasOracle = new GasOracle({ logger, provider, coinstack: 'arbitrum-nova' })
4444

‎node/coinstacks/arbitrum/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
7878
const wsServer = new Server({ server })
7979

8080
wsServer.on('connection', (connection) => {
81-
ConnectionHandler.start(connection, registry, prometheus)
81+
ConnectionHandler.start(connection, registry, prometheus, logger)
8282
})
8383

8484
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/arbitrum/api/src/controller.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export const logger = new Logger({
3838
level: process.env.LOG_LEVEL,
3939
})
4040

41-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
41+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
4242
const provider = new ethers.providers.JsonRpcProvider(RPC_URL)
4343
export const gasOracle = new GasOracle({ logger, provider, coinstack: 'arbitrum' })
4444

‎node/coinstacks/avalanche/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
7878
const wsServer = new Server({ server })
7979

8080
wsServer.on('connection', (connection) => {
81-
ConnectionHandler.start(connection, registry, prometheus)
81+
ConnectionHandler.start(connection, registry, prometheus, logger)
8282
})
8383

8484
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/avalanche/api/src/controller.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export const logger = new Logger({
3838
level: process.env.LOG_LEVEL,
3939
})
4040

41-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
41+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
4242
const provider = new ethers.providers.JsonRpcProvider(RPC_URL)
4343
export const gasOracle = new GasOracle({ logger, provider, coinstack: 'avalanche' })
4444

‎node/coinstacks/bitcoin/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
7676
const wsServer = new Server({ server })
7777

7878
wsServer.on('connection', (connection) => {
79-
ConnectionHandler.start(connection, registry, prometheus)
79+
ConnectionHandler.start(connection, registry, prometheus, logger)
8080
})
8181

8282
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/bitcoin/api/src/controller.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { bech32 } from 'bech32'
22
import { Blockbook } from '@shapeshiftoss/blockbook'
33
import { Service } from '../../../common/api/src/utxo/service'
44
import { UTXO } from '../../../common/api/src/utxo/controller'
5+
import { Logger } from '@shapeshiftoss/logger'
56

67
const INDEXER_URL = process.env.INDEXER_URL
78
const INDEXER_WS_URL = process.env.INDEXER_WS_URL
@@ -11,7 +12,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set')
1112
if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set')
1213
if (!RPC_URL) throw new Error('RPC_URL env var not set')
1314

14-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
15+
export const logger = new Logger({
16+
namespace: ['unchained', 'coinstacks', 'bitcoin', 'api'],
17+
level: process.env.LOG_LEVEL,
18+
})
19+
20+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
1521

1622
const isXpub = (pubkey: string): boolean => {
1723
return pubkey.startsWith('xpub') || pubkey.startsWith('ypub') || pubkey.startsWith('zpub')

‎node/coinstacks/bitcoincash/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
7878
const wsServer = new Server({ server })
7979

8080
wsServer.on('connection', (connection) => {
81-
ConnectionHandler.start(connection, registry, prometheus)
81+
ConnectionHandler.start(connection, registry, prometheus, logger)
8282
})
8383

8484
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/bitcoincash/api/src/controller.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { bech32 } from 'bech32'
22
import { Blockbook } from '@shapeshiftoss/blockbook'
33
import { Service } from '../../../common/api/src/utxo/service'
44
import { UTXO } from '../../../common/api/src/utxo/controller'
5+
import { Logger } from '@shapeshiftoss/logger'
56

67
const INDEXER_URL = process.env.INDEXER_URL
78
const INDEXER_WS_URL = process.env.INDEXER_WS_URL
@@ -11,7 +12,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set')
1112
if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set')
1213
if (!RPC_URL) throw new Error('RPC_URL env var not set')
1314

14-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
15+
export const logger = new Logger({
16+
namespace: ['unchained', 'coinstacks', 'bitcoincash', 'api'],
17+
level: process.env.LOG_LEVEL,
18+
})
19+
20+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
1521

1622
const isXpub = (pubkey: string): boolean => {
1723
return pubkey.startsWith('xpub') || pubkey.startsWith('ypub') || pubkey.startsWith('zpub')

‎node/coinstacks/bnbsmartchain/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
8080
const wsServer = new Server({ server })
8181

8282
wsServer.on('connection', (connection) => {
83-
ConnectionHandler.start(connection, registry, prometheus)
83+
ConnectionHandler.start(connection, registry, prometheus, logger)
8484
})
8585

8686
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/bnbsmartchain/api/src/controller.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export const logger = new Logger({
3838
level: process.env.LOG_LEVEL,
3939
})
4040

41-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
41+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
4242
const provider = new ethers.providers.JsonRpcProvider(RPC_URL)
4343
export const gasOracle = new GasOracle({ logger, provider, coinstack: 'bnbsmartchain' })
4444

‎node/coinstacks/common/api/src/evm/service.ts

+10-28
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import {
1919
} from './models'
2020
import {
2121
Cursor,
22-
NodeBlock,
2322
DebugCallStack,
2423
ExplorerApiResponse,
2524
TraceCall,
@@ -323,34 +322,17 @@ export class Service implements Omit<BaseAPI, 'getInfo'>, API {
323322
}
324323
}
325324

326-
async handleBlock(hash: string, retryCount = 0): Promise<Array<BlockbookTx>> {
327-
const request: RPCRequest = {
328-
jsonrpc: '2.0',
329-
id: `eth_getBlockByHash-${hash}`,
330-
method: 'eth_getBlockByHash',
331-
params: [hash, false],
332-
}
333-
334-
const { data } = await axios.post<RPCResponse>(this.rpcUrl, request)
335-
336-
if (data.error) throw new Error(`failed to get block: ${hash}: ${data.error.message}`)
337-
338-
// retry if no results are returned, this typically means we queried a node that hasn't indexed the data yet
339-
if (!data.result) {
340-
if (retryCount >= 5) throw new Error(`failed to get block: ${hash}: ${JSON.stringify(data)}`)
341-
retryCount++
342-
await exponentialDelay(retryCount)
343-
return this.handleBlock(hash, retryCount)
325+
async handleBlock(hash: string): Promise<Array<BlockbookTx>> {
326+
try {
327+
const { txs = [], totalPages = 1 } = await this.blockbook.getBlock(hash)
328+
for (let page = 1; page < totalPages; ++page) {
329+
const data = await this.blockbook.getBlock(hash, page)
330+
data.txs && txs.push(...data.txs)
331+
}
332+
return txs
333+
} catch (err) {
334+
throw handleError(err)
344335
}
345-
346-
const block = data.result as NodeBlock
347-
348-
// make best effort to fetch all transactions, but don't fail handling block if a single transaction fails
349-
const txs = await Promise.allSettled(block.transactions.map((hash) => this.blockbook.getTransaction(hash)))
350-
351-
return txs
352-
.filter((tx): tx is PromiseFulfilledResult<BlockbookTx> => tx.status === 'fulfilled')
353-
.map((tx) => tx.value)
354336
}
355337

356338
handleTransaction(tx: BlockbookTx): Tx {

‎node/coinstacks/common/api/src/utxo/service.ts

+10-23
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import axios from 'axios'
22
import axiosRetry from 'axios-retry'
33
import { ApiError as BlockbookApiError, Blockbook, Tx as BlockbookTx } from '@shapeshiftoss/blockbook'
4-
import { AddressFormatter, ApiError, BadRequestError, BaseAPI, Cursor, RPCRequest, RPCResponse, SendTxBody } from '../'
4+
import { AddressFormatter, ApiError, BadRequestError, BaseAPI, Cursor, SendTxBody } from '../'
55
import { Account, Address, API, NetworkFee, NetworkFees, RawTx, Tx, TxHistory, Utxo } from './models'
6-
import { NodeBlock } from './types'
76
import { validatePageSize } from '../utils'
87

98
axiosRetry(axios, { retries: 5, retryDelay: axiosRetry.exponentialDelay })
@@ -31,12 +30,10 @@ export class Service implements Omit<BaseAPI, 'getInfo'>, API {
3130
readonly isXpub: (pubkey: string) => boolean
3231

3332
private readonly blockbook: Blockbook
34-
private readonly rpcUrl: string
3533
private formatAddress: AddressFormatter = (address: string) => address.toLowerCase()
3634

3735
constructor(args: ServiceArgs) {
3836
this.blockbook = args.blockbook
39-
this.rpcUrl = args.rpcUrl
4037
this.isXpub = args.isXpub
4138

4239
if (args.addressFormatter) this.formatAddress = args.addressFormatter
@@ -182,26 +179,16 @@ export class Service implements Omit<BaseAPI, 'getInfo'>, API {
182179
}
183180

184181
async handleBlock(hash: string): Promise<Array<BlockbookTx>> {
185-
const request: RPCRequest = {
186-
jsonrpc: '2.0',
187-
id: `getblock-${hash}`,
188-
method: 'getblock',
189-
params: [hash],
182+
try {
183+
const { txs = [], totalPages = 1 } = await this.blockbook.getBlock(hash)
184+
for (let page = 1; page < totalPages; ++page) {
185+
const data = await this.blockbook.getBlock(hash, page)
186+
data.txs && txs.push(...data.txs)
187+
}
188+
return txs
189+
} catch (err) {
190+
throw handleError(err)
190191
}
191-
192-
const { data } = await axios.post<RPCResponse>(this.rpcUrl, request)
193-
194-
if (data.error) throw new Error(`failed to get block: ${hash}: ${data.error.message}`)
195-
if (!data.result) throw new Error(`failed to get block: ${hash}: ${JSON.stringify(data)}`)
196-
197-
const block = data.result as NodeBlock
198-
199-
// make best effort to fetch all transactions, but don't fail handling block if a single transaction fails
200-
const txs = await Promise.allSettled(block.tx.map((hash) => this.blockbook.getTransaction(hash)))
201-
202-
return txs
203-
.filter((tx): tx is PromiseFulfilledResult<BlockbookTx> => tx.status === 'fulfilled')
204-
.map((tx) => tx.value)
205192
}
206193

207194
handleTransaction(tx: BlockbookTx): Tx {

‎node/coinstacks/common/api/src/websocket.ts

+18-17
Original file line numberDiff line numberDiff line change
@@ -42,49 +42,50 @@ export class ConnectionHandler {
4242
private readonly websocket: WebSocket
4343
private readonly registry: Registry
4444
private readonly prometheus: Prometheus
45+
private readonly logger: Logger
4546
private readonly routes: Record<Topics, Methods>
46-
private readonly pingInterval = 10000
47+
private readonly pingIntervalMs = 10000
4748

4849
private pingTimeout?: NodeJS.Timeout
4950
private subscriptionIds = new Map<string, void>()
5051

51-
private logger = new Logger({ namespace: ['unchained', 'coinstacks', 'common', 'api'], level: process.env.LOG_LEVEL })
52-
53-
private constructor(websocket: WebSocket, registry: Registry, prometheus: Prometheus) {
52+
private constructor(websocket: WebSocket, registry: Registry, prometheus: Prometheus, logger: Logger) {
5453
this.clientId = v4()
5554
this.registry = registry
5655
this.prometheus = prometheus
56+
this.logger = logger.child({ namespace: ['websocket'] })
5757
this.routes = {
5858
txs: {
5959
subscribe: (subscriptionId: string, data?: TxsTopicData) => this.handleSubscribeTxs(subscriptionId, data),
6060
unsubscribe: (subscriptionId: string, data?: TxsTopicData) => this.handleUnsubscribeTxs(subscriptionId, data),
6161
},
6262
}
6363

64-
const interval = setInterval(() => {
65-
this.websocket.ping()
66-
}, this.pingInterval)
64+
this.pingTimeout = undefined
65+
this.prometheus.metrics.websocketCount.inc()
66+
this.websocket = websocket
67+
this.websocket.ping()
6768

68-
this.heartbeat()
69+
const pingInterval = setInterval(() => {
70+
this.websocket.ping()
71+
}, this.pingIntervalMs)
6972

70-
this.websocket = websocket
7173
this.websocket.onerror = (error) => {
7274
this.logger.error({ clientId: this.clientId, error, fn: 'ws.onerror' }, 'websocket error')
73-
this.close(interval)
75+
this.close(pingInterval)
7476
}
7577
this.websocket.onclose = ({ code, reason }) => {
7678
this.prometheus.metrics.websocketCount.dec()
7779
this.logger.debug({ clientId: this.clientId, code, reason, fn: 'ws.close' }, 'websocket closed')
78-
this.close(interval)
80+
this.close(pingInterval)
7981
}
8082
this.websocket.on('pong', () => this.heartbeat())
8183
this.websocket.on('ping', () => this.websocket.pong())
8284
this.websocket.onmessage = (event) => this.onMessage(event)
8385
}
8486

85-
static start(websocket: WebSocket, registry: Registry, prometheus: Prometheus): void {
86-
prometheus.metrics.websocketCount.inc()
87-
new ConnectionHandler(websocket, registry, prometheus)
87+
static start(websocket: WebSocket, registry: Registry, prometheus: Prometheus, logger: Logger): void {
88+
new ConnectionHandler(websocket, registry, prometheus, logger)
8889
}
8990

9091
private heartbeat(): void {
@@ -93,9 +94,9 @@ export class ConnectionHandler {
9394
}
9495

9596
this.pingTimeout = setTimeout(() => {
96-
this.logger.debug({ fn: 'pingTimeout' }, 'heartbeat failed')
97+
this.logger.debug({ clientId: this.clientId, fn: 'pingTimeout' }, 'heartbeat failed')
9798
this.websocket.terminate()
98-
}, this.pingInterval + 1000)
99+
}, this.pingIntervalMs + 1000)
99100
}
100101

101102
private sendError(message: string, subscriptionId: string): void {
@@ -130,7 +131,7 @@ export class ConnectionHandler {
130131
}
131132
}
132133
} catch (err) {
133-
this.logger.error(err, { fn: 'onMessage', event }, 'Error processing message')
134+
this.logger.error(err, { clientId: this.clientId, fn: 'onMessage', event }, 'Error processing message')
134135
}
135136
}
136137

‎node/coinstacks/dogecoin/api/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started'))
7878
const wsServer = new Server({ server })
7979

8080
wsServer.on('connection', (connection) => {
81-
ConnectionHandler.start(connection, registry, prometheus)
81+
ConnectionHandler.start(connection, registry, prometheus, logger)
8282
})
8383

8484
new WebsocketClient(INDEXER_WS_URL, {

‎node/coinstacks/dogecoin/api/src/controller.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Blockbook } from '@shapeshiftoss/blockbook'
22
import { Service } from '../../../common/api/src/utxo/service'
33
import { UTXO } from '../../../common/api/src/utxo/controller'
4+
import { Logger } from '@shapeshiftoss/logger'
45

56
const INDEXER_URL = process.env.INDEXER_URL
67
const INDEXER_WS_URL = process.env.INDEXER_WS_URL
@@ -10,7 +11,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set')
1011
if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set')
1112
if (!RPC_URL) throw new Error('RPC_URL env var not set')
1213

13-
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL })
14+
export const logger = new Logger({
15+
namespace: ['unchained', 'coinstacks', 'dogecoin', 'api'],
16+
level: process.env.LOG_LEVEL,
17+
})
18+
19+
const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger })
1420

1521
const isXpub = (pubkey: string): boolean => {
1622
return pubkey.startsWith('dgub')

0 commit comments

Comments
 (0)