From e4a9963aa9f35457fdc6972e82a62928a565effe Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 19 Sep 2024 16:39:28 -0600 Subject: [PATCH 1/4] adjust scripts --- extractors/main.py | 7 +++++++ extractors/src/extract.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/extractors/main.py b/extractors/main.py index 579d1213..5e40236b 100644 --- a/extractors/main.py +++ b/extractors/main.py @@ -37,14 +37,21 @@ print(f"No configuration found with name {args.name}") else: # run everything + exceptions = [] try: extract_blocks(network_id=network_id, **block_config) except Exception as e: + exceptions.append(e) print(f"Error extracting blocks: {e}") for eth_call_config in eth_call_configs: try: extract_data(network_id=network_id, **eth_call_config) except Exception as e: + exceptions.append(e) print(f"Error extracting eth_call {eth_call_config.get('name')}: {e}") continue + + # if there are any exceptions, raise the first one + if len(exceptions) > 0: + raise Exception(exceptions[0]) diff --git a/extractors/src/extract.py b/extractors/src/extract.py index c0af3ca3..d3b613e4 100644 --- a/extractors/src/extract.py +++ b/extractors/src/extract.py @@ -29,6 +29,7 @@ def extract_data( min_block=0, requests_per_second=25, block_increment=500, + chunk_size=1000, ): if network_id not in CHAIN_CONFIGS: raise ValueError(f"Network id {network_id} not supported") @@ -52,6 +53,7 @@ def extract_data( blocks=[f"{min_block}:latest:{block_increment}"], rpc=snx.provider_rpc, requests_per_second=requests_per_second, + chunk_size=chunk_size, output_dir=output_dir, hex=True, exclude_failed=True, @@ -67,6 +69,7 @@ def extract_blocks( min_block=0, requests_per_second=25, block_increment=500, + chunk_size=1000, ): if network_id not in CHAIN_CONFIGS: raise ValueError(f"Network id {network_id} not supported") @@ -83,6 +86,7 @@ def extract_blocks( blocks=[f"{min_block}:latest:{block_increment}"], rpc=snx.provider_rpc, requests_per_second=requests_per_second, + chunk_size=chunk_size, output_dir=output_dir, hex=True, exclude_failed=True, From 8fa209c8eb9cebef74a7180f67093bf9c451ad1f Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 19 Sep 2024 16:39:36 -0600 Subject: [PATCH 2/4] adjust configs --- extractors/configs/arbitrum_mainnet.yaml | 3 +++ extractors/configs/arbitrum_sepolia.yaml | 4 ++++ extractors/configs/base_mainnet.yaml | 8 +++++++- extractors/configs/base_sepolia.yaml | 8 +++++++- extractors/configs/eth_mainnet.yaml | 5 ++++- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/extractors/configs/arbitrum_mainnet.yaml b/extractors/configs/arbitrum_mainnet.yaml index 9154f18d..c412f1f7 100644 --- a/extractors/configs/arbitrum_mainnet.yaml +++ b/extractors/configs/arbitrum_mainnet.yaml @@ -4,6 +4,7 @@ blocks: min_block: "232500000" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 eth_calls: - contract_name: "CoreProxy" @@ -19,6 +20,7 @@ eth_calls: min_block: "218M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -33,3 +35,4 @@ eth_calls: min_block: "218M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 diff --git a/extractors/configs/arbitrum_sepolia.yaml b/extractors/configs/arbitrum_sepolia.yaml index 263442fb..aea491e0 100644 --- a/extractors/configs/arbitrum_sepolia.yaml +++ b/extractors/configs/arbitrum_sepolia.yaml @@ -4,6 +4,8 @@ blocks: min_block: "41M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 + eth_calls: - contract_name: "CoreProxy" @@ -19,6 +21,7 @@ eth_calls: min_block: "41M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -33,3 +36,4 @@ eth_calls: min_block: "41M" requests_per_second: 25 block_increment: 4000 + chunk_size: 80 diff --git a/extractors/configs/base_mainnet.yaml b/extractors/configs/base_mainnet.yaml index 49c4e46d..5b5f8c22 100644 --- a/extractors/configs/base_mainnet.yaml +++ b/extractors/configs/base_mainnet.yaml @@ -3,6 +3,8 @@ network_id: 8453 blocks: min_block: "7.5M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 eth_calls: - contract_name: "CoreProxy" @@ -13,6 +15,8 @@ eth_calls: - [1, "0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2"] min_block: "7.5M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -21,4 +25,6 @@ eth_calls: - [1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"] - [1, "0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2"] min_block: "7.5M" - requests_per_second: 25 \ No newline at end of file + requests_per_second: 25 + block_increment: 500 + chunk_size: 80 diff --git a/extractors/configs/base_sepolia.yaml b/extractors/configs/base_sepolia.yaml index f8987b36..174e222a 100644 --- a/extractors/configs/base_sepolia.yaml +++ b/extractors/configs/base_sepolia.yaml @@ -3,6 +3,8 @@ network_id: 84532 blocks: min_block: "8M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 eth_calls: - contract_name: "CoreProxy" @@ -12,6 +14,8 @@ eth_calls: - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"] min_block: "8M" requests_per_second: 25 + block_increment: 500 + chunk_size: 80 - contract_name: "CoreProxy" package_name: "system" @@ -19,4 +23,6 @@ eth_calls: inputs: - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"] min_block: "8M" - requests_per_second: 25 \ No newline at end of file + requests_per_second: 25 + block_increment: 500 + chunk_size: 80 diff --git a/extractors/configs/eth_mainnet.yaml b/extractors/configs/eth_mainnet.yaml index 0043c03b..7684d391 100644 --- a/extractors/configs/eth_mainnet.yaml +++ b/extractors/configs/eth_mainnet.yaml @@ -4,6 +4,7 @@ blocks: min_block: "20000000" requests_per_second: 25 block_increment: 150 + chunk_size: 50 eth_calls: - contract_name: "CoreProxy" @@ -14,6 +15,7 @@ eth_calls: min_block: "20000000" requests_per_second: 25 block_increment: 150 + chunk_size: 50 - contract_name: "CoreProxy" package_name: "system" @@ -22,4 +24,5 @@ eth_calls: - [1, "0xC011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F"] min_block: "20000000" requests_per_second: 25 - block_increment: 150 \ No newline at end of file + block_increment: 150 + chunk_size: 50 From 14b1f203544bc639507ac4de7854497423da2535 Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 19 Sep 2024 16:39:42 -0600 Subject: [PATCH 3/4] update environment --- extractors/cryo | 1 + extractors/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 160000 extractors/cryo diff --git a/extractors/cryo b/extractors/cryo new file mode 160000 index 00000000..c3a2f1df --- /dev/null +++ b/extractors/cryo @@ -0,0 +1 @@ +Subproject commit c3a2f1df688f85b91448c8d89f2b4e640ce7651c diff --git a/extractors/requirements.txt b/extractors/requirements.txt index 6d8b7f2e..344a1aec 100644 --- a/extractors/requirements.txt +++ b/extractors/requirements.txt @@ -5,5 +5,5 @@ duckdb==0.10.2 polars-lts-cpu==1.1.0 pandas numpy -synthetix==0.1.13 +synthetix==0.1.19 web3==6.20.2 \ No newline at end of file From 0d69067a1e71c6a47133cbd37bb69dea60e29a17 Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 19 Sep 2024 16:45:08 -0600 Subject: [PATCH 4/4] delete --- extractors/cryo | 1 - 1 file changed, 1 deletion(-) delete mode 160000 extractors/cryo diff --git a/extractors/cryo b/extractors/cryo deleted file mode 160000 index c3a2f1df..00000000 --- a/extractors/cryo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3a2f1df688f85b91448c8d89f2b4e640ce7651c