Performance improvements in uni-info-watcher event processing (#348)

* Avoid loading relations when fetching entity in mapping code * Load relations for Position entity separately in nfpm mapping code * Remove join table for token whitelistPools and save as array of ids * Add indexing of token properties in erc20-watcher and fix smoke-tests (#4) * Add indexing of token properties in erc20-watcher * Fix uni-info-watcher smoke-test * Fix for saving token whitelistPools as array of ids * Replace eth calls in uni-info-watcher with storage calls (#5) * Remove entity relation foreign keys and only save entity id * Use only typeorm delete for removeEntities * Implement CLI for checking config endpoints Co-authored-by: prathamesh0 <[email protected]>
vulcanize · Jun 7, 2022 · 8410da4 · 8410da4
1 parent 170e8ef
commit 8410da4
Show file tree

Hide file tree

Showing 37 changed files with 691 additions and 601 deletions.
diff --git a/README.md b/README.md
@@ -8,16 +8,35 @@ Install packages (Node.JS v16.13.1):
 
 ```bash
 yarn
+
+yarn build
 ```
 
 ### Services
 
 The default config files used by the watchers assume the following services are setup and running on localhost:
 
 * `vulcanize/go-ethereum` on port 8545
-* `vulcanize/ipld-eth-server` with native GQL API enabled, on port 8082
+* `vulcanize/ipld-eth-server` with native GQL API enabled on port 8082 and RPC API on port 8081
 * `postgraphile` on the `vulcanize/ipld-eth-server` database, on port 5000
 
+To check whether the endpoints in watcher config are working, run:
+
+```bash
+cd packages/util
+
+yarn check-config --config-file ../erc20-watcher/environments/local.toml
+
+# Check config file in other watcher.
+yarn check-config --config-file ../uni-watcher/environments/local.toml
+# vulcanize:check-config Checking ipld-eth-server GQL endpoint http://127.0.0.1:8082/graphql +0ms
+# vulcanize:check-config ipld-eth-server GQL endpoint working +33ms
+# vulcanize:check-config Checking postgraphile GQL endpoint http://127.0.0.1:5000/graphql +1ms
+# vulcanize:check-config postgraphile GQL endpoint working +12ms
+# vulcanize:check-config Checking RPC endpoint http://127.0.0.1:8081 +1ms
+# vulcanize:check-config RPC endpoint working +25ms
+```
+
 #### Note
 
 * In `vulcanize/ipld-eth-server`, add the following statement to `[ethereum]` section in `environments/config.toml`:

diff --git a/packages/erc20-watcher/src/database.ts b/packages/erc20-watcher/src/database.ts
@@ -10,6 +10,9 @@ import { Database as BaseDatabase, QueryOptions, Where } from '@vulcanize/util';
 
 import { Allowance } from './entity/Allowance';
 import { Balance } from './entity/Balance';
+import { Name } from './entity/Name';
+import { Symbol } from './entity/Symbol';
+import { Decimals } from './entity/Decimals';
 import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';
 import { SyncStatus } from './entity/SyncStatus';
@@ -62,6 +65,31 @@ export class Database {
       .getOne();
   }
 
+  async getName ({ blockHash, token }: { blockHash: string, token: string }): Promise<Name | undefined> {
+    return this._conn.getRepository(Name)
+      .findOne({
+        blockHash,
+        token
+      });
+  }
+
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  async getSymbol ({ blockHash, token }: { blockHash: string, token: string }): Promise<Symbol | undefined> {
+    return this._conn.getRepository(Symbol)
+      .findOne({
+        blockHash,
+        token
+      });
+  }
+
+  async getDecimals ({ blockHash, token }: { blockHash: string, token: string }): Promise<Decimals | undefined> {
+    return this._conn.getRepository(Decimals)
+      .findOne({
+        blockHash,
+        token
+      });
+  }
+
   async saveBalance ({ blockHash, blockNumber, token, owner, value, proof }: DeepPartial<Balance>): Promise<Balance> {
     const repo = this._conn.getRepository(Balance);
     const entity = repo.create({ blockHash, blockNumber, token, owner, value, proof });
@@ -74,6 +102,25 @@ export class Database {
     return repo.save(entity);
   }
 
+  async saveName ({ blockHash, blockNumber, token, value, proof }: DeepPartial<Name>): Promise<Name> {
+    const repo = this._conn.getRepository(Name);
+    const entity = repo.create({ blockHash, blockNumber, token, value, proof });
+    return repo.save(entity);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  async saveSymbol ({ blockHash, blockNumber, token, value, proof }: DeepPartial<Symbol>): Promise<Symbol> {
+    const repo = this._conn.getRepository(Symbol);
+    const entity = repo.create({ blockHash, blockNumber, token, value, proof });
+    return repo.save(entity);
+  }
+
+  async saveDecimals ({ blockHash, blockNumber, token, value, proof }: DeepPartial<Decimals>): Promise<Decimals> {
+    const repo = this._conn.getRepository(Decimals);
+    const entity = repo.create({ blockHash, blockNumber, token, value, proof });
+    return repo.save(entity);
+  }
+
   async getContracts (): Promise<Contract[]> {
     const repo = this._conn.getRepository(Contract);
 

diff --git a/packages/erc20-watcher/src/entity/Allowance.ts b/packages/erc20-watcher/src/entity/Allowance.ts
@@ -6,7 +6,7 @@ import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
-@Index(['blockHash', 'blockNumber', 'token', 'owner', 'spender'], { unique: true })
+@Index(['blockHash', 'token', 'owner', 'spender'], { unique: true })
 export class Allowance {
   @PrimaryGeneratedColumn()
   id!: number;

diff --git a/packages/erc20-watcher/src/entity/Balance.ts b/packages/erc20-watcher/src/entity/Balance.ts
@@ -6,7 +6,7 @@ import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
-@Index(['blockHash', 'blockNumber', 'token', 'owner'], { unique: true })
+@Index(['blockHash', 'token', 'owner'], { unique: true })
 export class Balance {
   @PrimaryGeneratedColumn()
   id!: number;

diff --git a/packages/erc20-watcher/src/entity/Decimals.ts b/packages/erc20-watcher/src/entity/Decimals.ts
@@ -0,0 +1,27 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+
+@Entity()
+@Index(['blockHash', 'token'], { unique: true })
+export class Decimals {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @Column('varchar', { length: 66 })
+  blockHash!: string;
+
+  @Column('integer')
+  blockNumber!: number;
+
+  @Column('varchar', { length: 42 })
+  token!: string;
+
+  @Column('integer')
+  value!: number;
+
+  @Column('text', { nullable: true })
+  proof!: string;
+}
diff --git a/packages/erc20-watcher/src/entity/Name.ts b/packages/erc20-watcher/src/entity/Name.ts
@@ -0,0 +1,27 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+
+@Entity()
+@Index(['blockHash', 'token'], { unique: true })
+export class Name {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @Column('varchar', { length: 66 })
+  blockHash!: string;
+
+  @Column('integer')
+  blockNumber!: number;
+
+  @Column('varchar', { length: 42 })
+  token!: string;
+
+  @Column('varchar')
+  value!: string;
+
+  @Column('text', { nullable: true })
+  proof!: string;
+}
diff --git a/packages/erc20-watcher/src/entity/Symbol.ts b/packages/erc20-watcher/src/entity/Symbol.ts
@@ -0,0 +1,27 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+
+@Entity()
+@Index(['blockHash', 'token'], { unique: true })
+export class Symbol {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @Column('varchar', { length: 66 })
+  blockHash!: string;
+
+  @Column('integer')
+  blockNumber!: number;
+
+  @Column('varchar', { length: 42 })
+  token!: string;
+
+  @Column('varchar')
+  value!: string;
+
+  @Column('text', { nullable: true })
+  proof!: string;
+}
diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts
@@ -180,8 +180,21 @@ export class Indexer {
   }
 
   async name (blockHash: string, token: string): Promise<ValueResult> {
+    const entity = await this._db.getName({ blockHash, token });
+    if (entity) {
+      log('name: db hit.');
+
+      return {
+        value: entity.value,
+        proof: JSON.parse(entity.proof)
+      };
+    }
+
+    log('name: db miss, fetching from upstream server');
     let result: ValueResult;
 
+    const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash);
+
     if (this._serverMode === ETH_CALL_MODE) {
       const value = await fetchTokenName(this._ethProvider, blockHash, token);
 
@@ -190,14 +203,27 @@ export class Indexer {
       result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_name');
     }
 
-    // log(JSONbig.stringify(result, null, 2));
+    await this._db.saveName({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) });
 
     return result;
   }
 
   async symbol (blockHash: string, token: string): Promise<ValueResult> {
+    const entity = await this._db.getSymbol({ blockHash, token });
+    if (entity) {
+      log('symbol: db hit.');
+
+      return {
+        value: entity.value,
+        proof: JSON.parse(entity.proof)
+      };
+    }
+
+    log('symbol: db miss, fetching from upstream server');
     let result: ValueResult;
 
+    const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash);
+
     if (this._serverMode === ETH_CALL_MODE) {
       const value = await fetchTokenSymbol(this._ethProvider, blockHash, token);
 
@@ -206,14 +232,27 @@ export class Indexer {
       result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_symbol');
     }
 
-    // log(JSONbig.stringify(result, null, 2));
+    await this._db.saveSymbol({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) });
 
     return result;
   }
 
   async decimals (blockHash: string, token: string): Promise<ValueResult> {
+    const entity = await this._db.getDecimals({ blockHash, token });
+    if (entity) {
+      log('decimals: db hit.');
+
+      return {
+        value: entity.value,
+        proof: JSON.parse(entity.proof)
+      };
+    }
+
+    log('decimals: db miss, fetching from upstream server');
     let result: ValueResult;
 
+    const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash);
+
     if (this._serverMode === ETH_CALL_MODE) {
       const value = await fetchTokenDecimals(this._ethProvider, blockHash, token);
 
@@ -224,6 +263,8 @@ export class Indexer {
       throw new Error('Not implemented.');
     }
 
+    await this._db.saveDecimals({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) });
+
     return result;
   }
 

diff --git a/packages/uni-info-watcher/package.json b/packages/uni-info-watcher/package.json
@@ -28,6 +28,7 @@
     "test": "mocha src/**/*.test.ts",
     "test:gpev": "mocha src/get-prev-entity.test.ts",
     "test:server": "mocha src/server.test.ts",
+    "test:init": "ts-node test/init.ts",
     "build": "tsc",
     "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
     "server:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/server.js",
@@ -36,7 +37,7 @@
     "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
     "job-runner:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/job-runner.js",
     "job-runner:dev": "DEBUG=vulcanize:* nodemon --watch src src/job-runner.ts",
-    "smoke-test": "mocha src/smoke.test.ts",
+    "smoke-test": "yarn test:init && mocha src/smoke.test.ts",
     "fill": "DEBUG=vulcanize:* node --enable-source-maps dist/fill.js",
     "fill:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/fill.js",
     "fill:dev": "DEBUG=vulcanize:* ts-node src/fill.ts",