From 2aa8479e00d2a4f349e0ce6a00716b73f2f4211f Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:44:06 -0600 Subject: [PATCH 1/3] feat: thorchain v2 support --- .circleci/config.yml | 16 ++- go/cmd/thorchain/main.go | 27 ++++- go/cmd/thorchain/sample.env | 2 + go/coinstacks/thorchain/api/affiliateFees.go | 121 ++++++++++++------- go/coinstacks/thorchain/daemon/readiness.sh | 4 +- go/coinstacks/thorchain/pulumi/index.ts | 2 +- 6 files changed, 117 insertions(+), 55 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd0404530..6f014480c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -375,7 +375,9 @@ aliases: pulumi-stack: public-us-east-2 pulumi-dir: coinstacks/thorchain/pulumi lcd-url: http://thorchain-svc.unchained.svc.cluster.local:1317 + lcd-v1-url: http://thorchain-v1-svc.unchained.svc.cluster.local:1317 rpc-url: http://thorchain-svc.unchained.svc.cluster.local:27147 + rpc-v1-url: http://thorchain-v1-svc.unchained.svc.cluster.local:27147 ws-url: ws://thorchain-svc.unchained.svc.cluster.local:27147 api-autoscaling: true api-replicas: 2 @@ -387,11 +389,11 @@ aliases: api-memory-request: 1Gi stateful-service-replicas: 2 service-name-1: daemon - service-image-1: registry.gitlab.com/thorchain/thornode:mainnet-1.134.0 + service-image-1: registry.gitlab.com/thorchain/thornode:mainnet-2.135.0 service-cpu-limit-1: "2" service-cpu-request-1: "1" service-memory-limit-1: 16Gi - service-storage-size-1: 3500Gi + service-storage-size-1: 100Gi service-name-2: timescaledb service-image-2: timescale/timescaledb:2.13.0-pg15 service-cpu-limit-2: "2" @@ -410,7 +412,9 @@ aliases: environment: dev pulumi-stack: public-dev-us-east-2 lcd-url: http://thorchain-svc.unchained-dev.svc.cluster.local:1317 + lcd-v1-url: http://thorchain-v1-svc.unchained-dev.svc.cluster.local:1317 rpc-url: http://thorchain-svc.unchained-dev.svc.cluster.local:27147 + rpc-v1-url: http://thorchain-v1-svc.unchained-dev.svc.cluster.local:27147 ws-url: ws://thorchain-svc.unchained-dev.svc.cluster.local:27147 api-replicas: 1 api-max-replicas: 2 @@ -996,11 +1000,17 @@ jobs: <<: *params lcd-url: type: string + lcd-v1-url: + type: string + default: "" grpc-url: type: string default: "" rpc-url: type: string + rpc-v1-url: + type: string + default: "" ws-url: type: string default: "" @@ -1014,8 +1024,10 @@ jobs: name: add custom environment variables command: | echo 'export LCD_URL=<< parameters.lcd-url >>' >> $BASH_ENV + echo 'export LCD_V1_URL=<< parameters.lcd-v1-url >>' >> $BASH_ENV echo 'export GRPC_URL=<< parameters.grpc-url >>' >> $BASH_ENV echo 'export RPC_URL=<< parameters.rpc-url >>' >> $BASH_ENV + echo 'export RPC_V1_URL=<< parameters.rpc-v1-url >>' >> $BASH_ENV echo 'export WS_URL=<< parameters.ws-url >>' >> $BASH_ENV - run: name: pulumi diff --git a/go/cmd/thorchain/main.go b/go/cmd/thorchain/main.go index cc3cfe280..a2eedc904 100644 --- a/go/cmd/thorchain/main.go +++ b/go/cmd/thorchain/main.go @@ -23,9 +23,11 @@ var ( ) type Config struct { - LCDURL string `mapstructure:"LCD_URL"` - RPCURL string `mapstructure:"RPC_URL"` - WSURL string `mapstructure:"WS_URL"` + LCDURL string `mapstructure:"LCD_URL"` + LCDV1URL string `mapstructure:"LCD_V1_URL"` + RPCURL string `mapstructure:"RPC_URL"` + RPCV1URL string `mapstructure:"RPC_V1_URL"` + WSURL string `mapstructure:"WS_URL"` } func main() { @@ -37,7 +39,7 @@ func main() { conf := &Config{} if *envPath == "" { - if err := config.LoadFromEnv(conf, "LCD_URL", "RPC_URL", "WS_URL"); err != nil { + if err := config.LoadFromEnv(conf, "LCD_URL", "LCD_V1_URL", "RPC_URL", "RPC_V1_URL", "WS_URL"); err != nil { logger.Panicf("failed to load config from env: %+v", err) } } else { @@ -59,6 +61,16 @@ func main() { WSURL: conf.WSURL, } + cfgV1 := cosmos.Config{ + Bech32AddrPrefix: "thor", + Bech32PkPrefix: "thorpub", + Bech32ValPrefix: "thorv", + Bech32PkValPrefix: "thorvpub", + Encoding: encoding, + LCDURL: conf.LCDV1URL, + RPCURL: conf.RPCV1URL, + } + prometheus := metrics.NewPrometheus("thorchain") httpClient, err := cosmos.NewHTTPClient(cfg) @@ -66,6 +78,11 @@ func main() { logger.Panicf("failed to create new http client: %+v", err) } + httpClientV1, err := cosmos.NewHTTPClient(cfgV1) + if err != nil { + logger.Panicf("failed to create new http client: %+v", err) + } + blockService, err := cosmos.NewBlockService(httpClient) if err != nil { logger.Panicf("failed to create new block service: %+v", err) @@ -76,7 +93,7 @@ func main() { logger.Panicf("failed to create new websocket client: %+v", err) } - indexer := api.NewAffiliateFeeIndexer(cfg, httpClient) + indexer := api.NewAffiliateFeeIndexer(cfg, []*cosmos.HTTPClient{httpClientV1, httpClient}) if err := indexer.Sync(); err != nil { logger.Panicf("failed to index affiliate fees: %+v", err) } diff --git a/go/cmd/thorchain/sample.env b/go/cmd/thorchain/sample.env index 27b7646ac..4946f6e73 100644 --- a/go/cmd/thorchain/sample.env +++ b/go/cmd/thorchain/sample.env @@ -1,3 +1,5 @@ LCD_URL=https://dev-daemon.thorchain.shapeshift.com/lcd +LCD_V1_URL=https://dev-daemon.thorchain-v1.shapeshift.com/lcd RPC_URL=https://dev-daemon.thorchain.shapeshift.com/rpc +RPC_V1_URL=https://dev-daemon.thorchain-v1.shapeshift.com/rpc WS_URL=wss://dev-daemon.thorchain.shapeshift.com/rpc diff --git a/go/coinstacks/thorchain/api/affiliateFees.go b/go/coinstacks/thorchain/api/affiliateFees.go index 78db3fecf..5df41e876 100644 --- a/go/coinstacks/thorchain/api/affiliateFees.go +++ b/go/coinstacks/thorchain/api/affiliateFees.go @@ -19,6 +19,7 @@ import ( tendermint "github.com/tendermint/tendermint/rpc/jsonrpc/client" "github.com/tendermint/tendermint/types" tmtypes "github.com/tendermint/tendermint/types" + "golang.org/x/sync/errgroup" ) const ( @@ -31,10 +32,10 @@ const ( type AffiliateFeeIndexer struct { AffiliateFees []*AffiliateFee conf cosmos.Config - httpClient *cosmos.HTTPClient + httpClients []*cosmos.HTTPClient mu sync.Mutex - pageCh chan int - resultCh chan *coretypes.ResultBlockSearch + pageChs []chan int + resultChs []chan *coretypes.ResultBlockSearch wg sync.WaitGroup } @@ -48,13 +49,23 @@ type AffiliateFee struct { TxID string } -func NewAffiliateFeeIndexer(conf cosmos.Config, httpClient *cosmos.HTTPClient) *AffiliateFeeIndexer { +func NewAffiliateFeeIndexer(conf cosmos.Config, httpClients []*cosmos.HTTPClient) *AffiliateFeeIndexer { + pageChs := make([]chan int, len(httpClients)) + for i := range pageChs { + pageChs[i] = make(chan int) + } + + resultChs := make([]chan *coretypes.ResultBlockSearch, len(httpClients)) + for i := range resultChs { + resultChs[i] = make(chan *coretypes.ResultBlockSearch) + } + return &AffiliateFeeIndexer{ AffiliateFees: []*AffiliateFee{}, conf: conf, - httpClient: httpClient, - pageCh: make(chan int), - resultCh: make(chan *coretypes.ResultBlockSearch), + httpClients: httpClients, + pageChs: pageChs, + resultChs: resultChs, } } @@ -62,49 +73,69 @@ func (i *AffiliateFeeIndexer) Sync() error { start := time.Now() logger.Info("Started indexing affiliate fees") - defer close(i.resultCh) + defer func() { + for _, resultCh := range i.resultChs { + close(resultCh) + } + }() if err := i.listen(); err != nil { return err } - for w := 0; w < resultWorkers; w++ { - go func() { - for result := range i.resultCh { - for _, b := range result.Blocks { - block := b.Block - i.handleBlock(block) - } + g := new(errgroup.Group) + + for j, httpClient := range i.httpClients { + httpClient := httpClient + resultCh := i.resultChs[j] + pageCh := i.pageChs[j] + + g.Go(func() error { + for w := 0; w < resultWorkers; w++ { + go func() { + for result := range resultCh { + for _, b := range result.Blocks { + block := b.Block + i.handleBlock(httpClient, block) + } + } + }() } - }() - } - - result, err := i.httpClient.BlockSearch(fmt.Sprintf(`"outbound.to='%s'"`, "thor1xmaggkcln5m5fnha2780xrdrulmplvfrz6wj3l"), 1, pageSize) - if err != nil { - return err - } - maxPages := int(math.Ceil(float64(result.TotalCount) / float64(pageSize))) - i.resultCh <- result + result, err := httpClient.BlockSearch(fmt.Sprintf(`"outbound.to='%s'"`, "thor1xmaggkcln5m5fnha2780xrdrulmplvfrz6wj3l"), 1, pageSize) + if err != nil { + return err + } - i.wg.Add(blockWorkers) - for w := 0; w < blockWorkers; w++ { - go i.fetchBlocks() - } + maxPages := int(math.Ceil(float64(result.TotalCount) / float64(pageSize))) + resultCh <- result - go func() { - page := 2 - for { - if page > maxPages { - close(i.pageCh) - break + i.wg.Add(blockWorkers) + for w := 0; w < blockWorkers; w++ { + go i.fetchBlocks(httpClient, pageCh, resultCh) } - i.pageCh <- page - page++ - } - }() - i.wg.Wait() + go func() { + page := 2 + for { + if page > maxPages { + close(pageCh) + break + } + pageCh <- page + page++ + } + }() + + i.wg.Wait() + + return nil + }) + } + + if err := g.Wait(); err != nil { + return err + } logger.Infof("Finished indexing affiliate fees (%s)", time.Since(start)) @@ -180,21 +211,21 @@ func (i *AffiliateFeeIndexer) listen() error { return nil } -func (i *AffiliateFeeIndexer) fetchBlocks() { +func (i *AffiliateFeeIndexer) fetchBlocks(httpClient *cosmos.HTTPClient, pageCh <-chan int, resultCh chan<- *coretypes.ResultBlockSearch) { defer i.wg.Done() - for page := range i.pageCh { - result, err := i.httpClient.BlockSearch(fmt.Sprintf(`"outbound.to='%s'"`, "thor1xmaggkcln5m5fnha2780xrdrulmplvfrz6wj3l"), page, pageSize) + for page := range pageCh { + result, err := httpClient.BlockSearch(fmt.Sprintf(`"outbound.to='%s'"`, "thor1xmaggkcln5m5fnha2780xrdrulmplvfrz6wj3l"), page, pageSize) if err != nil { logger.Panicf("failed to fetch blocks for page: %d: %+v", page, err) } - i.resultCh <- result + resultCh <- result } } -func (i *AffiliateFeeIndexer) handleBlock(block *tmtypes.Block) { - blockResult, err := i.httpClient.BlockResults(int(block.Height)) +func (i *AffiliateFeeIndexer) handleBlock(httpClient *cosmos.HTTPClient, block *tmtypes.Block) { + blockResult, err := httpClient.BlockResults(int(block.Height)) if err != nil { logger.Panicf("failed to handle block: %d: %+v", block.Height, err) } diff --git a/go/coinstacks/thorchain/daemon/readiness.sh b/go/coinstacks/thorchain/daemon/readiness.sh index 9abbd4328..9c2b2a41d 100644 --- a/go/coinstacks/thorchain/daemon/readiness.sh +++ b/go/coinstacks/thorchain/daemon/readiness.sh @@ -20,9 +20,9 @@ CATCHING_UP=$(echo $STATUS | jq -r '.result.sync_info.catching_up') NUM_PEERS=$(echo $NET_INFO | jq -r '.result.n_peers') status_curls=( - "curl -sf -m 3 https://rpc.ninerealms.com/status" + "curl -sf -m 3 https://rpc-v2.ninerealms.com/status" # referer header now required to avoid being blocked - "curl -sf -m 3 -H \"Referer: https://app.thorswap.finance\" https://rpc.thorswap.net/status" + "curl -sf -m 3 -H \"Referer: https://app.thorswap.finance\" https://rpc-v2.thorswap.net/status" ) if [[ $IS_SYNCING == false && $CATCHING_UP == false ]]; then diff --git a/go/coinstacks/thorchain/pulumi/index.ts b/go/coinstacks/thorchain/pulumi/index.ts index 0f39087f8..bbcad50eb 100644 --- a/go/coinstacks/thorchain/pulumi/index.ts +++ b/go/coinstacks/thorchain/pulumi/index.ts @@ -16,7 +16,7 @@ export = async (): Promise => { ...service, dataDir: '/root', env: { - CHAIN_ID: 'thorchain-mainnet-v1', + CHAIN_ID: 'thorchain-mainnet-v2', NET: config.network, }, ports: { From b84e566773225a15db90b5401bf9e668cfed550d Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:18:14 -0600 Subject: [PATCH 2/3] update chain id --- go/coinstacks/thorchain/pulumi/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/coinstacks/thorchain/pulumi/index.ts b/go/coinstacks/thorchain/pulumi/index.ts index bbcad50eb..4dd646731 100644 --- a/go/coinstacks/thorchain/pulumi/index.ts +++ b/go/coinstacks/thorchain/pulumi/index.ts @@ -16,7 +16,7 @@ export = async (): Promise => { ...service, dataDir: '/root', env: { - CHAIN_ID: 'thorchain-mainnet-v2', + CHAIN_ID: 'thorchain-1', NET: config.network, }, ports: { From 6168ba108c3680cc25d6dd1d8cd842664db47f47 Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:18:36 -0600 Subject: [PATCH 3/3] comment out thorswap endpoint, will update after fork --- go/coinstacks/thorchain/daemon/readiness.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/coinstacks/thorchain/daemon/readiness.sh b/go/coinstacks/thorchain/daemon/readiness.sh index 9c2b2a41d..de6799947 100644 --- a/go/coinstacks/thorchain/daemon/readiness.sh +++ b/go/coinstacks/thorchain/daemon/readiness.sh @@ -22,7 +22,7 @@ NUM_PEERS=$(echo $NET_INFO | jq -r '.result.n_peers') status_curls=( "curl -sf -m 3 https://rpc-v2.ninerealms.com/status" # referer header now required to avoid being blocked - "curl -sf -m 3 -H \"Referer: https://app.thorswap.finance\" https://rpc-v2.thorswap.net/status" + #"curl -sf -m 3 -H \"Referer: https://app.thorswap.finance\" https://rpc-v2.thorswap.net/status" ) if [[ $IS_SYNCING == false && $CATCHING_UP == false ]]; then