Merge pull request #123 from Synthetixio/feat/optimize-calls

Optimize extractors service

extractors/configs/arbitrum_mainnet.yaml

@@ -4,6 +4,7 @@ blocks:
   min_block: "232500000"
   requests_per_second: 25
   block_increment: 4000
+  chunk_size: 80
 
 eth_calls:
   - contract_name: "CoreProxy"
@@ -19,6 +20,7 @@ eth_calls:
     min_block: "218M"
     requests_per_second: 25
     block_increment: 4000
+    chunk_size: 80
 
   - contract_name: "CoreProxy"
     package_name: "system"
@@ -33,3 +35,4 @@ eth_calls:
     min_block: "218M"
     requests_per_second: 25
     block_increment: 4000
+    chunk_size: 80

extractors/configs/arbitrum_sepolia.yaml

@@ -4,6 +4,8 @@ blocks:
   min_block: "41M"
   requests_per_second: 25
   block_increment: 4000
+  chunk_size: 80
+
 
 eth_calls:
   - contract_name: "CoreProxy"
@@ -19,6 +21,7 @@ eth_calls:
     min_block: "41M"
     requests_per_second: 25
     block_increment: 4000
+    chunk_size: 80
 
   - contract_name: "CoreProxy"
     package_name: "system"
@@ -33,3 +36,4 @@ eth_calls:
     min_block: "41M"
     requests_per_second: 25
     block_increment: 4000
+    chunk_size: 80

extractors/configs/base_mainnet.yaml

@@ -3,6 +3,8 @@ network_id: 8453
 blocks:
   min_block: "7.5M"
   requests_per_second: 25
+  block_increment: 500
+  chunk_size: 80
 
 eth_calls:
   - contract_name: "CoreProxy"
@@ -13,6 +15,8 @@ eth_calls:
       - [1, "0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2"]
     min_block: "7.5M"
     requests_per_second: 25
+    block_increment: 500
+    chunk_size: 80
 
   - contract_name: "CoreProxy"
     package_name: "system"
@@ -21,4 +25,6 @@ eth_calls:
       - [1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"]
       - [1, "0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2"]
     min_block: "7.5M"
-    requests_per_second: 25
+    requests_per_second: 25
+    block_increment: 500
+    chunk_size: 80

extractors/configs/base_sepolia.yaml

@@ -3,6 +3,8 @@ network_id: 84532
 blocks:
   min_block: "8M"
   requests_per_second: 25
+  block_increment: 500
+  chunk_size: 80
 
 eth_calls:
   - contract_name: "CoreProxy"
@@ -12,11 +14,15 @@ eth_calls:
       - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"]
     min_block: "8M"
     requests_per_second: 25
+    block_increment: 500
+    chunk_size: 80
 
   - contract_name: "CoreProxy"
     package_name: "system"
     function_name: "getVaultDebt"
     inputs:
       - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"]
     min_block: "8M"
-    requests_per_second: 25
+    requests_per_second: 25
+    block_increment: 500
+    chunk_size: 80

extractors/configs/eth_mainnet.yaml

@@ -4,6 +4,7 @@ blocks:
   min_block: "20000000"
   requests_per_second: 25
   block_increment: 150
+  chunk_size: 50
 
 eth_calls:
   - contract_name: "CoreProxy"
@@ -14,6 +15,7 @@ eth_calls:
     min_block: "20000000"
     requests_per_second: 25
     block_increment: 150
+    chunk_size: 50
 
   - contract_name: "CoreProxy"
     package_name: "system"
@@ -22,4 +24,5 @@ eth_calls:
       - [1, "0xC011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F"]
     min_block: "20000000"
     requests_per_second: 25
-    block_increment: 150
+    block_increment: 150
+    chunk_size: 50

extractors/main.py

@@ -37,14 +37,21 @@
             print(f"No configuration found with name {args.name}")
 else:
     # run everything
+    exceptions = []
     try:
         extract_blocks(network_id=network_id, **block_config)
     except Exception as e:
+        exceptions.append(e)
         print(f"Error extracting blocks: {e}")
 
     for eth_call_config in eth_call_configs:
         try:
             extract_data(network_id=network_id, **eth_call_config)
         except Exception as e:
+            exceptions.append(e)
             print(f"Error extracting eth_call {eth_call_config.get('name')}: {e}")
             continue
+
+    # if there are any exceptions, raise the first one
+    if len(exceptions) > 0:
+        raise Exception(exceptions[0])

extractors/requirements.txt

@@ -5,5 +5,5 @@ duckdb==0.10.2
 polars-lts-cpu==1.1.0
 pandas
 numpy
-synthetix==0.1.13
+synthetix==0.1.19
 web3==6.20.2

extractors/src/extract.py

@@ -29,6 +29,7 @@ def extract_data(
     min_block=0,
     requests_per_second=25,
     block_increment=500,
+    chunk_size=1000,
 ):
     if network_id not in CHAIN_CONFIGS:
         raise ValueError(f"Network id {network_id} not supported")
@@ -52,6 +53,7 @@ def extract_data(
         blocks=[f"{min_block}:latest:{block_increment}"],
         rpc=snx.provider_rpc,
         requests_per_second=requests_per_second,
+        chunk_size=chunk_size,
         output_dir=output_dir,
         hex=True,
         exclude_failed=True,
@@ -67,6 +69,7 @@ def extract_blocks(
     min_block=0,
     requests_per_second=25,
     block_increment=500,
+    chunk_size=1000,
 ):
     if network_id not in CHAIN_CONFIGS:
         raise ValueError(f"Network id {network_id} not supported")
@@ -83,6 +86,7 @@ def extract_blocks(
         blocks=[f"{min_block}:latest:{block_increment}"],
         rpc=snx.provider_rpc,
         requests_per_second=requests_per_second,
+        chunk_size=chunk_size,
         output_dir=output_dir,
         hex=True,
         exclude_failed=True,