Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion gittensor/validator/issue_discovery/mirror_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,10 @@ async def run_mirror_issue_discovery(
try:
response = client.get_miner_issues(evaluation.github_id, since=lookback_date)
except MirrorRequestError as e:
bt.logging.warning(f'├─ UID {uid}: mirror issue fetch failed ({e}) — skipped this miner')
bt.logging.error(
f'├─ UID {uid}: mirror issue fetch failed ({e}). '
f'Issue discovery score will be 0 for this miner this round.'
)
fetch_errors += 1
continue

Expand Down Expand Up @@ -190,6 +193,15 @@ async def run_mirror_issue_discovery(
f'{cache_stats.fetch_failures} fetch failures)'
)

if fetch_errors > 0:
total_eligible = processed + fetch_errors + no_issues
if total_eligible >= 5 and fetch_errors / total_eligible >= 0.5:
bt.logging.error(
f'High mirror issue-fetch failure rate: {fetch_errors}/{total_eligible} eligible miners '
f'failed ({fetch_errors / total_eligible:.0%}). Mirror service may be unavailable — '
f'issue discovery scores are 0 for all affected miners this round.'
)


def _build_solving_pr_cache(
miner_evaluations: Dict[int, MinerEvaluation],
Expand Down
6 changes: 5 additions & 1 deletion gittensor/validator/oss_contributions/mirror/load.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,11 @@ def load_mirror_miner_prs(
try:
response = client.get_miner_pulls(mirror_eval.github_id, since=lookback_date)
except MirrorRequestError as e:
bt.logging.error(f'Mirror fetch failed for UID {mirror_eval.uid}: {e}')
bt.logging.error(
f'Mirror PR fetch failed for UID {mirror_eval.uid} '
f'(hotkey {mirror_eval.hotkey[:8]}): {e}. '
f'Mirror-enabled repo scores will be 0 this round for this miner.'
)
mirror_eval.fetch_failed = True
return

Expand Down
63 changes: 63 additions & 0 deletions tests/validator/issue_discovery/test_mirror_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,3 +682,66 @@ def test_all_mirror_miner_below_threshold_passes_spam(self):
# Below threshold → spam_mult=1.0 → discovery score is non-zero
assert eval_.issue_discovery_score > 0
assert eval_.total_open_issues == 2


# ============================================================================
# Aggregate outage detection
# ============================================================================


class TestAggregateOutageDetection:
"""Verify that a high fetch-failure rate triggers an error-level outage alert."""

def _run_with_patched_bt(self, evals, side_effect):
from unittest.mock import patch

client = Mock()
client.get_miner_issues.side_effect = side_effect
with patch('gittensor.validator.issue_discovery.mirror_scan.bt') as mock_bt:
_run(
run_mirror_issue_discovery(
evals,
_mirror_repos('entrius/gittensor-ui'),
_EMPTY_LANGS,
_EMPTY_TOKEN_CONFIG,
client=client,
)
)
return mock_bt.logging.error.call_args_list

def _outage_logged(self, calls):
return any('Mirror service may be unavailable' in c.args[0] for c in calls)

def test_total_failure_across_ten_miners_logs_outage(self):
evals = {i: _eval(uid=i, github_id=f'g{i}') for i in range(10)}
calls = self._run_with_patched_bt(evals, MirrorRequestError('service unavailable'))
assert self._outage_logged(calls)

def test_majority_failure_at_boundary_logs_outage(self):
# 5 fail, 5 succeed (exactly 50%) — threshold is >= 0.5 so this fires
def _side(github_id, since=None):
return MirrorRequestError('boom') if github_id < 'g5' else _response([])

evals = {i: _eval(uid=i, github_id=f'g{i}') for i in range(10)}
calls = self._run_with_patched_bt(
evals,
lambda gid, since=None: (_ for _ in ()).throw(MirrorRequestError('boom')) if gid < 'g5' else _response([]),
)
assert self._outage_logged(calls)

def test_low_failure_rate_does_not_log_outage(self):
# 2 out of 10 fail = 20% — below threshold
def _side(github_id, since=None):
if github_id in ('g0', 'g1'):
raise MirrorRequestError('boom')
return _response([])

evals = {i: _eval(uid=i, github_id=f'g{i}') for i in range(10)}
calls = self._run_with_patched_bt(evals, _side)
assert not self._outage_logged(calls)

def test_small_population_guard_suppresses_alert(self):
# Only 3 miners — total_eligible < 5 guard prevents false positive
evals = {i: _eval(uid=i, github_id=f'g{i}') for i in range(3)}
calls = self._run_with_patched_bt(evals, MirrorRequestError('boom'))
assert not self._outage_logged(calls)
Loading