diff --git a/extractors/configs/arbitrum_mainnet.yaml b/extractors/configs/arbitrum_mainnet.yaml index 9154f18d..c412f1f7 100644 --- a/extractors/configs/arbitrum_mainnet.yaml +++ b/extractors/configs/arbitrum_mainnet.yaml @@ -4,6 +4,7 @@ blocks: min_block: "232500000" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 eth_calls: - contract_name: "CoreProxy" @@ -19,6 +20,7 @@ eth_calls: min_block: "218M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -33,3 +35,4 @@ eth_calls: min_block: "218M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 diff --git a/extractors/configs/arbitrum_sepolia.yaml b/extractors/configs/arbitrum_sepolia.yaml index 263442fb..aea491e0 100644 --- a/extractors/configs/arbitrum_sepolia.yaml +++ b/extractors/configs/arbitrum_sepolia.yaml @@ -4,6 +4,8 @@ blocks: min_block: "41M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 + eth_calls: - contract_name: "CoreProxy" @@ -19,6 +21,7 @@ eth_calls: min_block: "41M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -33,3 +36,4 @@ eth_calls: min_block: "41M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 diff --git a/extractors/configs/base_mainnet.yaml b/extractors/configs/base_mainnet.yaml index 49c4e46d..5b5f8c22 100644 --- a/extractors/configs/base_mainnet.yaml +++ b/extractors/configs/base_mainnet.yaml @@ -3,6 +3,8 @@ network_id: 8453 blocks: min_block: "7.5M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 eth_calls: - contract_name: "CoreProxy" @@ -13,6 +15,8 @@ eth_calls: - [1, "0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2"] min_block: "7.5M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -21,4 +25,6 @@ eth_calls: - [1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"] - [1, "0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2"] min_block: "7.5M" - requests_per_second: 25 \ No newline at end of file + requests_per_second: 25 + block_increment: 500 + chunk_size: 80 diff --git a/extractors/configs/base_sepolia.yaml b/extractors/configs/base_sepolia.yaml index f8987b36..174e222a 100644 --- a/extractors/configs/base_sepolia.yaml +++ b/extractors/configs/base_sepolia.yaml @@ -3,6 +3,8 @@ network_id: 84532 blocks: min_block: "8M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 eth_calls: - contract_name: "CoreProxy" @@ -12,6 +14,8 @@ eth_calls: - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"] min_block: "8M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -19,4 +23,6 @@ eth_calls: inputs: - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"] min_block: "8M" - requests_per_second: 25 \ No newline at end of file + requests_per_second: 25 + block_increment: 500 + chunk_size: 80 diff --git a/extractors/configs/eth_mainnet.yaml b/extractors/configs/eth_mainnet.yaml index 0043c03b..7684d391 100644 --- a/extractors/configs/eth_mainnet.yaml +++ b/extractors/configs/eth_mainnet.yaml @@ -4,6 +4,7 @@ blocks: min_block: "20000000" requests_per_second: 25 block_increment: 150 + chunk_size: 50 eth_calls: - contract_name: "CoreProxy" @@ -14,6 +15,7 @@ eth_calls: min_block: "20000000" requests_per_second: 25 block_increment: 150 + chunk_size: 50 - contract_name: "CoreProxy" package_name: "system" @@ -22,4 +24,5 @@ eth_calls: - [1, "0xC011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F"] min_block: "20000000" requests_per_second: 25 - block_increment: 150 \ No newline at end of file + block_increment: 150 + chunk_size: 50 diff --git a/extractors/main.py b/extractors/main.py index 579d1213..5e40236b 100644 --- a/extractors/main.py +++ b/extractors/main.py @@ -37,14 +37,21 @@ print(f"No configuration found with name {args.name}") else: # run everything + exceptions = [] try: extract_blocks(network_id=network_id, **block_config) except Exception as e: + exceptions.append(e) print(f"Error extracting blocks: {e}") for eth_call_config in eth_call_configs: try: extract_data(network_id=network_id, **eth_call_config) except Exception as e: + exceptions.append(e) print(f"Error extracting eth_call {eth_call_config.get('name')}: {e}") continue + + # if there are any exceptions, raise the first one + if len(exceptions) > 0: + raise Exception(exceptions[0]) diff --git a/extractors/requirements.txt b/extractors/requirements.txt index 6d8b7f2e..344a1aec 100644 --- a/extractors/requirements.txt +++ b/extractors/requirements.txt @@ -5,5 +5,5 @@ duckdb==0.10.2 polars-lts-cpu==1.1.0 pandas numpy -synthetix==0.1.13 +synthetix==0.1.19 web3==6.20.2 \ No newline at end of file diff --git a/extractors/src/extract.py b/extractors/src/extract.py index c0af3ca3..d3b613e4 100644 --- a/extractors/src/extract.py +++ b/extractors/src/extract.py @@ -29,6 +29,7 @@ def extract_data( min_block=0, requests_per_second=25, block_increment=500, + chunk_size=1000, ): if network_id not in CHAIN_CONFIGS: raise ValueError(f"Network id {network_id} not supported") @@ -52,6 +53,7 @@ def extract_data( blocks=[f"{min_block}:latest:{block_increment}"], rpc=snx.provider_rpc, requests_per_second=requests_per_second, + chunk_size=chunk_size, output_dir=output_dir, hex=True, exclude_failed=True, @@ -67,6 +69,7 @@ def extract_blocks( min_block=0, requests_per_second=25, block_increment=500, + chunk_size=1000, ): if network_id not in CHAIN_CONFIGS: raise ValueError(f"Network id {network_id} not supported") @@ -83,6 +86,7 @@ def extract_blocks( blocks=[f"{min_block}:latest:{block_increment}"], rpc=snx.provider_rpc, requests_per_second=requests_per_second, + chunk_size=chunk_size, output_dir=output_dir, hex=True, exclude_failed=True,