diff --git a/gittensor/utils/github_api_tools.py b/gittensor/utils/github_api_tools.py index 75a0e3d7e..50cc4b86c 100644 --- a/gittensor/utils/github_api_tools.py +++ b/gittensor/utils/github_api_tools.py @@ -9,7 +9,7 @@ from math import ceil from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional -from gittensor.utils.utils import parse_repo_name +from gittensor.utils.utils import backoff_seconds, parse_repo_name if TYPE_CHECKING: from gittensor.classes import FileChange as FileChangeType @@ -299,7 +299,7 @@ def get_merge_base_sha(repository: str, base_sha: str, head_sha: str, token: str return None if attempt < max_attempts - 1: - backoff_delay = min(5 * (2 ** (attempt)), 30) + backoff_delay = backoff_seconds(attempt) bt.logging.warning( f'Compare API for {repository} failed with status {response.status_code} ' f'(attempt {attempt + 1}/{max_attempts}), retrying in {backoff_delay}s...' @@ -308,7 +308,7 @@ def get_merge_base_sha(repository: str, base_sha: str, head_sha: str, token: str except requests.exceptions.RequestException as e: if attempt < max_attempts - 1: - backoff_delay = min(5 * (2 ** (attempt)), 30) + backoff_delay = backoff_seconds(attempt) bt.logging.warning( f'Compare API error for {repository} (attempt {attempt + 1}/{max_attempts}): {e}, ' f'retrying in {backoff_delay}s...' @@ -377,7 +377,7 @@ def get_pull_request_file_changes(repository: str, pr_number: int, token: str) - attempt += 1 if attempt < max_attempts: - backoff_delay = min(5 * (2 ** (attempt - 1)), 30) + backoff_delay = backoff_seconds(attempt - 1) bt.logging.warning( f'File changes request for PR #{pr_number} in {repository} failed with {last_error} ' f'(attempt {attempt}/{max_attempts}), per_page={per_page}, retrying in {backoff_delay}s...' @@ -391,7 +391,7 @@ def get_pull_request_file_changes(repository: str, pr_number: int, token: str) - attempt += 1 if attempt < max_attempts: - backoff_delay = min(5 * (2 ** (attempt - 1)), 30) + backoff_delay = backoff_seconds(attempt - 1) bt.logging.warning( f'File changes request error for PR #{pr_number} in {repository} ' f'(attempt {attempt}/{max_attempts}): {e}, retrying in {backoff_delay}s...' @@ -575,7 +575,7 @@ def execute_graphql_query( # Retry on failure if attempt < (max_attempts - 1): - backoff_delay = min(5 * (2**attempt), 30) # max of 30 second wait between retries + backoff_delay = backoff_seconds(attempt) bt.logging.warning( f'GraphQL request failed with status {response.status_code} ' f'(attempt {attempt + 1}/{max_attempts}), retrying in {backoff_delay}s...' @@ -589,7 +589,7 @@ def execute_graphql_query( except requests.exceptions.RequestException as e: if attempt < (max_attempts - 1): - backoff_delay = min(5 * (2**attempt), 30) + backoff_delay = backoff_seconds(attempt) bt.logging.warning( f'GraphQL request exception (attempt {attempt + 1}/{max_attempts}), ' f'retrying in {backoff_delay}s: {e}' @@ -666,7 +666,7 @@ def get_github_graphql_query( if attempt < (max_attempts - 1): old_limit = limit limit = max(limit // 2, 10) - backoff_delay = min(2 * (2**attempt), 15) + backoff_delay = backoff_seconds(attempt, base=2, cap=15) bt.logging.warning( f'GraphQL RESOURCE_LIMITS_EXCEEDED (attempt {attempt + 1}/{max_attempts}), ' f'page size {old_limit} -> {limit}, retrying in {backoff_delay}s...' @@ -683,7 +683,7 @@ def get_github_graphql_query( # HTTP error - log and retry if attempt < (max_attempts - 1): - backoff_delay = min(5 * (2**attempt), 30) + backoff_delay = backoff_seconds(attempt) if response.status_code in (502, 503, 504): limit = max(limit // 2, 10) bt.logging.warning( @@ -703,7 +703,7 @@ def get_github_graphql_query( except requests.exceptions.RequestException as e: if attempt < (max_attempts - 1): - backoff_delay = min(5 * (2**attempt), 30) + backoff_delay = backoff_seconds(attempt) bt.logging.warning( f'GraphQL request connection error (attempt {attempt + 1}/{max_attempts}): {e}, retrying in {backoff_delay}s...' ) diff --git a/gittensor/utils/mirror/client.py b/gittensor/utils/mirror/client.py index c5b8f21bb..b5925326a 100644 --- a/gittensor/utils/mirror/client.py +++ b/gittensor/utils/mirror/client.py @@ -22,6 +22,7 @@ MirrorPullRequestFilesResponse, MirrorPullRequestsResponse, ) +from gittensor.utils.utils import backoff_seconds class MirrorRequestError(RuntimeError): @@ -103,7 +104,7 @@ def _get(self, path: str, params: Optional[dict] = None) -> dict: except requests.RequestException as e: last_error = f'request exception: {e}' if attempt < self.max_attempts - 1: - backoff = min(5 * (2**attempt), 30) + backoff = backoff_seconds(attempt) bt.logging.warning( f'Mirror GET {path} raised {e} ' f'(attempt {attempt + 1}/{self.max_attempts}), retrying in {backoff}s...' @@ -120,7 +121,7 @@ def _get(self, path: str, params: Optional[dict] = None) -> dict: last_error = f'status {response.status_code}: {response.text[:200]}' if attempt < self.max_attempts - 1: - backoff = min(5 * (2**attempt), 30) + backoff = backoff_seconds(attempt) bt.logging.warning( f'Mirror GET {path} failed ({last_error}) ' f'(attempt {attempt + 1}/{self.max_attempts}), retrying in {backoff}s...' diff --git a/gittensor/utils/utils.py b/gittensor/utils/utils.py index eeafc16b2..4e1b3276f 100644 --- a/gittensor/utils/utils.py +++ b/gittensor/utils/utils.py @@ -6,6 +6,10 @@ from typing import Dict +def backoff_seconds(attempt: int, base: int = 5, cap: int = 30) -> int: + return min(base * (2**attempt), cap) + + def parse_repo_name(repo_data: Dict): """Normalizes and converts repository name from dict""" return f'{repo_data["owner"]["login"]}/{repo_data["name"]}'.lower()