Release/v0.0.38 (#72)

MitchellLeeZA · mitchell · web-flow · commit 80103e8a7f27 · 2025-11-26T10:32:15.000+02:00
* Reduced refresh interval to 10 minutes instead of 20 minutes. * Reduced immunity of new miners - since requests is lower and we didn't take into account multiple validators. * Updated snippet fetcher to not be seen as a bot (#71) Co-authored-by: mitchell <mitchell@policydock.com> * Miner Weight Update (#70) * Refactored code. * Updated .gitignore to exclude output and results. * Added burn_weights_by_ranking. * Added unit tests. * Updated .gitignore. * Removed unwanted files. * Fixed as per PR comments. --------- Co-authored-by: mitchell <mitchell@policydock.com> * Updated domain_is_recently_registered to fix error validating domain: can't subtract offset-naive and offset-aware datetimes. --------- Co-authored-by: mitchell <mitchell@policydock.com>
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,8 @@ __pycache__/
 env-veridex
 env/
 miner/openai/
+miner/best_miner/
+output/
+results/
+.idea/
+.env
diff --git a/shared/environment_variables.py b/shared/environment_variables.py
@@ -17,5 +17,5 @@
 INITIAL_WEIGHT = 0.7
 
 NEUTRAL_SCORE=10
-IMMUNITY_PERIOD = 350 # Ensures new miners have a full day to prove themselves, even if other miners have been idle.
+IMMUNITY_PERIOD = 100 # Ensures new miners have a full day to prove themselves, even if other miners have been idle.
 IMMUNITY_WEIGHT = 0.5
diff --git a/tests/manual/test_snippet_fetcher.py b/tests/manual/test_snippet_fetcher.py
@@ -10,6 +10,13 @@
 from shared.veridex_protocol import SourceEvidence
 from validator.snippet_fetcher import SnippetFetcher
 
+# Array of hard-coded test URLs for snippet fetching
+TEST_URLS = [
+    "https://nmaahc.si.edu/explore/stories/unforgettable-nat-king-cole-flip-wilson-american-television",
+    "https://www.pbs.org/wnet/nature/blog/killer-whale-fact-sheet/",
+    "https://hms.harvard.edu/news/screen-time-brain"
+]
+
 def get_config():
     parser = argparse.ArgumentParser()
     parser.add_argument("--custom", default="my_custom_value", help="Custom value")
@@ -52,10 +59,61 @@ async def main(url):
         duration = time.perf_counter() - start
         print(f"Time taken for {url}:  {duration:.4f} seconds")
 
+async def main_all_urls():
+    """Fetch all URLs in the TEST_URLS array"""
+    miner_uid = 1
+    snippet_fetcher = SnippetFetcher()
+
+    try:
+        for i, url in enumerate(TEST_URLS):
+            print(f"\n{'='*80}")
+            print(f"Fetching URL {i+1}/{len(TEST_URLS)}: {url}")
+            print(f"{'='*80}\n")
+
+            start = time.perf_counter()
+            try:
+                page_text = await snippet_fetcher.fetch_entire_page(f"test-{i}", miner_uid, url)
+                duration = time.perf_counter() - start
+
+                print(f"\n{'='*80}")
+                print(f"Result for {url}:")
+                print(f"Length: {len(page_text)} characters")
+                print(f"Time taken: {duration:.4f} seconds")
+                print(f"{'='*80}\n")
+                print(page_text[:500] + "..." if len(page_text) > 500 else page_text)
+                print("\n")
+            except Exception as e:
+                duration = time.perf_counter() - start
+                print(f"Error fetching {url}: {e} (took {duration:.4f} seconds)")
+    finally:
+        await snippet_fetcher.client.aclose()
+
 # Entry point
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Run Snippet Fetcher for url.")
-    parser.add_argument("--url", type=str, default="https://studyfinds.org/content-overload-streaming", help="Page to fetch")
+    parser.add_argument(
+        "--url",
+        type=str,
+        default=TEST_URLS[0],
+        help="Page to fetch (default: first URL in TEST_URLS array)"
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Fetch all URLs in TEST_URLS array"
+    )
+    parser.add_argument(
+        "--list",
+        action="store_true",
+        help="List all URLs in TEST_URLS array"
+    )
     args = parser.parse_args()
 
-    asyncio.run(main(args.url))
+    if args.list:
+        print("Available test URLs:")
+        for i, url in enumerate(TEST_URLS, 1):
+            print(f"  {i}. {url}")
+    elif args.all:
+        asyncio.run(main_all_urls())
+    else:
+        asyncio.run(main(args.url))
diff --git a/tests/unit_tests/__init__.py b/tests/unit_tests/__init__.py
@@ -0,0 +1 @@
+
diff --git a/tests/unit_tests/test_distribute_weights_by_ranking.py b/tests/unit_tests/test_distribute_weights_by_ranking.py
@@ -0,0 +1,258 @@
+import unittest
+import sys
+import os
+
+# Add the parent directory to the path to import the validator module
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+
+# Create a comprehensive mock bittensor module before importing validator_daemon
+class MockBt:
+    class logging:
+        @staticmethod
+        def set_trace():
+            pass  # Mock for debugging
+        @staticmethod
+        def warning(msg):
+            print(f"WARNING: {msg}")
+
+        @staticmethod
+        def info(msg):
+            print(f"INFO: {msg}")
+
+        @staticmethod
+        def error(msg):
+            print(f"ERROR: {msg}")
+
+        @staticmethod
+        def config(*args, **kwargs):
+            pass
+
+        @staticmethod
+        def add_args(*args, **kwargs):
+            pass
+
+    class subtensor:
+        @staticmethod
+        def add_args(*args, **kwargs):
+            pass
+
+    class wallet:
+        @staticmethod
+        def add_args(*args, **kwargs):
+            pass
+
+# Patch the bittensor module before importing
+sys.modules['bittensor'] = MockBt()
+
+# Import the actual function and constants from validator_daemon
+from validator.validator_daemon import distribute_weights_by_ranking, DEFAULT_TOTAL_WEIGHT, RANKING_EMISSION_TOP_PERC
+
+
+class TestDistributeWeightsByRanking(unittest.TestCase):
+    """Test suite for distribute_weights_by_ranking function"""
+
+    def test_empty_scores(self):
+        """Test with empty scores array"""
+        moving_scores = []
+        result = distribute_weights_by_ranking(moving_scores)
+        self.assertEqual(result, [])
+
+    def test_single_score(self):
+        """Test with single score"""
+        moving_scores = [10.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0], int(DEFAULT_TOTAL_WEIGHT))
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+
+    def test_two_scores_descending(self):
+        """Test with two scores in descending order"""
+        moving_scores = [10.0, 5.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 2)
+        # Top should get 50% of total_weight
+        expected_top = int(DEFAULT_TOTAL_WEIGHT * RANKING_EMISSION_TOP_PERC)
+        # Second should get 25% of total_weight
+        expected_second = int(DEFAULT_TOTAL_WEIGHT * 0.25)
+        # Remainder goes to top
+        remainder = int(DEFAULT_TOTAL_WEIGHT) - expected_top - expected_second
+        expected_top += remainder
+
+        self.assertEqual(result[0], expected_top)  # Highest score gets top allocation
+        self.assertEqual(result[1], expected_second)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+
+    def test_two_scores_ascending(self):
+        """Test with two scores in ascending order - should still rank correctly"""
+        moving_scores = [5.0, 10.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 2)
+        # Index 1 has higher score, so should get top allocation
+        self.assertGreater(result[1], result[0])
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+
+    def test_geometric_progression(self):
+        """Test that weights follow geometric progression (50%, 25%, 12.5%, etc.)"""
+        moving_scores = [10.0, 8.0, 6.0, 4.0, 2.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 5)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+
+        # Verify ranking order (higher scores get more weight)
+        self.assertGreater(result[0], result[1])  # Top > second
+        self.assertGreater(result[1], result[2])  # Second > third
+        self.assertGreater(result[2], result[3])  # Third > fourth
+        self.assertGreater(result[3], result[4])  # Fourth > fifth
+
+        # Verify approximate percentages (allowing for rounding)
+        total = float(sum(result))
+        self.assertAlmostEqual(result[0] / total, RANKING_EMISSION_TOP_PERC, delta=0.01)
+        self.assertAlmostEqual(result[1] / total, 0.25, delta=0.01)
+        self.assertAlmostEqual(result[2] / total, 0.125, delta=0.01)
+
+    def test_equal_scores(self):
+        """Test with equal scores - first one should get top allocation"""
+        moving_scores = [5.0, 5.0, 5.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+        # First index gets top allocation (stable sort)
+        self.assertGreaterEqual(result[0], result[1])
+        self.assertGreaterEqual(result[1], result[2])
+
+    def test_zero_scores(self):
+        """Test with all zero scores"""
+        moving_scores = [0.0, 0.0, 0.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+        # All should get equal allocation when scores are equal
+        self.assertGreaterEqual(min(result), 0)
+
+    def test_mixed_positive_negative(self):
+        """Test with mixed positive and negative scores"""
+        moving_scores = [10.0, -5.0, 3.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+        # Highest score (10.0) should get most weight
+        self.assertGreater(result[0], result[2])  # 10.0 > 3.0
+        self.assertGreater(result[2], result[1])  # 3.0 > -5.0
+
+    def test_custom_total_weight(self):
+        """Test with custom total_weight"""
+        moving_scores = [10.0, 5.0]
+        custom_total = 1000.0
+        result = distribute_weights_by_ranking(moving_scores, total_weight=custom_total)
+
+        self.assertEqual(len(result), 2)
+        self.assertEqual(sum(result), int(custom_total))
+        # Top should get 50% of custom_total
+        expected_top = int(custom_total * RANKING_EMISSION_TOP_PERC)
+        remainder = int(custom_total) - expected_top - int(custom_total * 0.25)
+        expected_top += remainder
+        self.assertEqual(result[0], expected_top)
+
+    def test_custom_top_percentage(self):
+        """Test with custom top_percentage"""
+        moving_scores = [10.0, 5.0, 3.0]
+        custom_percentage = 0.6  # 60% for top
+        result = distribute_weights_by_ranking(moving_scores, top_percentage=custom_percentage)
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+
+        # Verify top gets approximately 60%
+        total = float(sum(result))
+        self.assertAlmostEqual(result[0] / total, custom_percentage, delta=0.01)
+        # Second should get 30% (60% * 0.5)
+        self.assertAlmostEqual(result[1] / total, 0.30, delta=0.01)
+
+    def test_large_number_of_scores(self):
+        """Test with many scores"""
+        moving_scores = [float(i) for i in range(20, 0, -1)]  # 20 down to 1
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 20)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+
+        # Verify descending order
+        for i in range(len(result) - 1):
+            self.assertGreaterEqual(result[i], result[i + 1])
+
+    def test_very_small_scores(self):
+        """Test with very small score values"""
+        moving_scores = [0.001, 0.002, 0.003]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+        # Highest score should get most weight
+        self.assertGreater(result[2], result[1])
+        self.assertGreater(result[1], result[0])
+
+    def test_very_large_scores(self):
+        """Test with very large score values"""
+        moving_scores = [1e10, 2e10, 3e10]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT))
+        # Highest score should get most weight
+        self.assertGreater(result[2], result[1])
+        self.assertGreater(result[1], result[0])
+
+    def test_sum_equals_total_weight(self):
+        """Test that sum always equals total_weight exactly"""
+        test_cases = [
+            [1.0],
+            [1.0, 2.0],
+            [1.0, 2.0, 3.0, 4.0, 5.0],
+            [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+        ]
+
+        for moving_scores in test_cases:
+            with self.subTest(moving_scores=moving_scores):
+                result = distribute_weights_by_ranking(moving_scores)
+                self.assertEqual(sum(result), int(DEFAULT_TOTAL_WEIGHT),
+                               f"Failed for scores: {moving_scores}")
+
+    def test_ranking_preserved(self):
+        """Test that ranking is preserved (higher scores get more weight)"""
+        moving_scores = [1.0, 5.0, 3.0, 7.0, 2.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        # Expected ranking: index 3 (7.0) > index 1 (5.0) > index 2 (3.0) > index 4 (2.0) > index 0 (1.0)
+        self.assertGreater(result[3], result[1])  # 7.0 > 5.0
+        self.assertGreater(result[1], result[2])  # 5.0 > 3.0
+        self.assertGreater(result[2], result[4])  # 3.0 > 2.0
+        self.assertGreater(result[4], result[0])  # 2.0 > 1.0
+
+    def test_all_weights_non_negative(self):
+        """Test that all weights are non-negative"""
+        moving_scores = [-10.0, -5.0, 0.0, 5.0, 10.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        for weight in result:
+            self.assertGreaterEqual(weight, 0, "All weights should be non-negative")
+
+    def test_returns_integers(self):
+        """Test that function returns list of integers"""
+        moving_scores = [1.0, 2.0, 3.0]
+        result = distribute_weights_by_ranking(moving_scores)
+
+        self.assertIsInstance(result, list)
+        for weight in result:
+            self.assertIsInstance(weight, int, "All weights should be integers")
+
+
+if __name__ == '__main__':
+    unittest.main()
+
diff --git a/validator/api_server.py b/validator/api_server.py
@@ -44,7 +44,7 @@
 
 load_dotenv()
 
-REFRESH_INTERVAL_SECONDS =  60 * 20
+REFRESH_INTERVAL_SECONDS =  60 * 10
 NUMBER_OF_MINERS = 3
 
 semaphore = asyncio.Semaphore(5)  # Limit to 10 threads at a time
diff --git a/validator/domain_validator.py b/validator/domain_validator.py
@@ -2,7 +2,7 @@
 import bittensor as bt
 import sys
 import asyncio
-from datetime import datetime
+from datetime import datetime, timezone
 
 domain_limiter = asyncio.Semaphore(5) # Max 5 concurrent threads
 
@@ -26,7 +26,13 @@ async def domain_is_recently_registered(domain) -> bool:
                 # Give benefit of doubt if error happened: return False
                 return False
 
-            return (datetime.now() - creation_date).days <= 30  # Adjust the days threshold
+            # Ensure both datetimes are timezone-aware for comparison
+            now = datetime.now(timezone.utc)
+            if creation_date.tzinfo is None:
+                # If creation_date is naive, assume it's UTC
+                creation_date = creation_date.replace(tzinfo=timezone.utc)
+            
+            return (now - creation_date).days <= 30  # Adjust the days threshold
         except Exception as e:
             # Give benefit of doubt if error happened: return False
             bt.logging.error(f"Error validating domain: {e}")
diff --git a/validator/snippet_fetcher.py b/validator/snippet_fetcher.py
diff --git a/validator/validator_daemon.py b/validator/validator_daemon.py
