diff --git a/gaia/tasks/defined_tasks/weather/processing/weather_logic.py b/gaia/tasks/defined_tasks/weather/processing/weather_logic.py index 14b7f7ed..f957f1cb 100644 --- a/gaia/tasks/defined_tasks/weather/processing/weather_logic.py +++ b/gaia/tasks/defined_tasks/weather/processing/weather_logic.py @@ -1624,9 +1624,19 @@ async def calculate_era5_miner_score( return False stored_manifest_hash = stored_response_data["verification_hash_claimed"] - token_data_tuple = await _request_fresh_token( - task_instance, miner_hotkey, job_id - ) + + max_attempts = task_instance.config.get("era5_token_attempts", 3) + for attempt in range(1, max_attempts + 1): + token_data_tuple = await _request_fresh_token( + task_instance, miner_hotkey, job_id + ) + if token_data_tuple is not None: + break + + delay = attempt * 10 + logger.info(f"[FinalScore] Miner {miner_hotkey}: Attempt {attempt}/{max_attempts + 1} failed to get token_data_tuple, waiting for {delay}s.") + await asyncio.sleep(delay) + if token_data_tuple is None: # Check if miner is still registered before treating this as a critical error is_registered = await _is_miner_registered(task_instance, miner_hotkey) diff --git a/gaia/tasks/defined_tasks/weather/weather_task.py b/gaia/tasks/defined_tasks/weather/weather_task.py index 7593745d..e9cf0ec5 100644 --- a/gaia/tasks/defined_tasks/weather/weather_task.py +++ b/gaia/tasks/defined_tasks/weather/weather_task.py @@ -197,6 +197,9 @@ def parse_int_list(env_var, default_list): config["verification_timeout_seconds"] = int( os.getenv("WEATHER_VERIFICATION_TIMEOUT_SECONDS", "3600") ) + config["era5_token_attempts"] = int( + os.getenv("ERA5_TOKEN_ATTEMPTS", "3") + ) # More frequent checking for progressive scoring (check every 30 minutes instead of 1 hour) config["final_scoring_check_interval_seconds"] = int( os.getenv("WEATHER_FINAL_SCORING_INTERVAL_S", "1800")