Don't log single oracle downtime (#271)

* Don't log single oracle downtime Signed-off-by: cyc60 <[email protected]> * Process RetryError in logging Signed-off-by: cyc60 <[email protected]> --------- Signed-off-by: cyc60 <[email protected]>
stakewise · Jan 18, 2024 · 1fb31b0 · 1fb31b0
1 parent 69a7a47
commit 1fb31b0
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/src/exits/tasks.py b/src/exits/tasks.py
@@ -3,7 +3,9 @@
 import time
 from random import shuffle
 
+from aiohttp import ClientError
 from eth_typing import BlockNumber, BLSPubkey
+from tenacity import RetryError
 from web3 import Web3
 from web3.types import HexStr
 
@@ -13,7 +15,7 @@
 from src.common.metrics import metrics
 from src.common.tasks import BaseTask
 from src.common.typings import Oracles
-from src.common.utils import get_current_timestamp, is_block_finalized
+from src.common.utils import get_current_timestamp, is_block_finalized, warning_verbose
 from src.config.settings import settings
 from src.exits.consensus import get_validator_public_keys
 from src.exits.execution import submit_exit_signatures
@@ -108,12 +110,16 @@ async def _fetch_outdated_indexes(oracles: Oracles, update_block: BlockNumber |
     shuffle(endpoints)
 
     for oracle_endpoint in endpoints:
-        response = await get_oracle_outdated_signatures_response(oracle_endpoint)
+        try:
+            response = await get_oracle_outdated_signatures_response(oracle_endpoint)
+        except (ClientError, RetryError) as e:
+            warning_verbose(str(e))
+            continue
         if not update_block or response['exit_signature_block_number'] >= update_block:
             outdated_indexes = [val['index'] for val in response['validators']]
             metrics.outdated_signatures.set(len(outdated_indexes))
             return outdated_indexes
-    raise RuntimeError('Oracles have not synced exit signatures yet')
+    raise RuntimeError('Oracles are down or have not synced exit signatures yet')
 
 
 async def _update_exit_signatures(