prio: order tied URLs per network and day, not per request

A tie-break that differs on every request spreads the probes on a
network over different URLs, so fewer of them measure the same URL and
can confirm each other's results. Last week, in most of the 30 most
measured countries, 35-81% of the URLs a network measured on a day were
measured by two or more runs that day.

Seed the shuffle with the probe's country, network and UTC day instead:
probes on the same network get tied URLs in the same order during a
day, and in a new order the next day, so over several days the network
still works through more of its list, and other networks in the country
start elsewhere in it. TIE_BREAK_SLOT_SECONDS sets the slot; a shorter
one spreads measurements over the list sooner, which matters for
noticing new blocking quickly, with fewer probes per URL in each slot.

The entries come from a query without ORDER BY, so they are sorted by
URL before the seeded shuffle, or the same seed could give different
orders. compute_priorities without a seed keeps the entries' order, for
show_countries_prioritization.

Author: aagbsn

ooniapi/common/src/common/prio.py

@@ -23,7 +23,8 @@
 
 from functools import lru_cache
 import random
-from typing import Annotated, Dict, List, Tuple
+import time
+from typing import Annotated, Dict, List, Optional, Tuple
 import logging
 
 import sqlalchemy as sa
@@ -53,8 +54,24 @@ def match_prio_rule(cz, pr: dict) -> bool:
     return True
 
 
-def compute_priorities(entries: tuple, prio_rules: tuple) -> list:
-    # Order based on (msmt_cnt / priority) to provide balancing
+# Probes on the same network get URLs that tie on weight in the same order
+# within a slot, so that they measure the same URLs and confirm each other's
+# results, and in a new order in the next slot, so that over several slots
+# they work through more of the list. A shorter slot spreads measurements
+# over the list sooner, with fewer probes per URL in each slot.
+TIE_BREAK_SLOT_SECONDS = 86400
+
+
+def tie_break_seed(probe_cc: str, probe_asn: int, now: Optional[float] = None) -> str:
+    """Seed for ordering a network's tied URLs during the current slot"""
+    slot = int((time.time() if now is None else now) // TIE_BREAK_SLOT_SECONDS)
+    return f"{probe_cc.upper()}:{probe_asn}:{slot}"
+
+
+def compute_priorities(entries: tuple, prio_rules: tuple, seed: Optional[str] = None) -> list:
+    """Order based on (msmt_cnt / priority) to provide balancing.
+    With seed, URLs with the same weight come out in an order picked by the
+    seed (see tie_break_seed), otherwise in the order of entries."""
     test_list = []
     for e in entries:
         # Calculate priority for an URL
@@ -68,11 +85,13 @@ def compute_priorities(entries: tuple, prio_rules: tuple) -> list:
         o["weight"] = priority / max(e["msmt_cnt"], 0.1)
         test_list.append(o)
 
-    # URLs with the same weight, common when measurement counts are low, come
-    # out in a random order: probes that test only the top of the list then
-    # don't all test the same URLs. sorted() is stable, so shuffling first
-    # breaks the ties without changing the ranking.
-    random.shuffle(test_list)
+    if seed is not None:
+        # entries come from a query without ORDER BY: sort them so that the
+        # same seed gives the same order. sorted() is stable, so shuffling
+        # first breaks ties between equal weights without changing the
+        # ranking.
+        test_list.sort(key=lambda k: (k["url"], k["cc"]))
+        random.Random(seed).shuffle(test_list)
     return sorted(test_list, key=lambda k: k["weight"], reverse=True)
 
 
@@ -140,7 +159,7 @@ def generate_test_list(
     log.info("fetched %d url entries", len(entries))
     prio_rules = fetch_prioritization_rules(clickhouse, country_code)
     log.info("fetched %d priority rules", len(prio_rules))
-    li = compute_priorities(entries, prio_rules)
+    li = compute_priorities(entries, prio_rules, tie_break_seed(country_code, probe_asn))
     # Filter unwanted category codes, replace ZZ, trim priority <= 0
     out = []
     for entry in li:

ooniapi/services/ooniprobe/tests/test_prio.py

@@ -1,3 +1,5 @@
+import random
+
 import pytest
 
 from ooniprobe.common import prio
@@ -187,7 +189,7 @@ async def test_debug_prioritization(client):
     assert resp.headers["content-type"] == "application/json"
 
 
-def test_compute_priorities_random_tie_break():
+def tied_entries():
     # 20 URLs with the same weight and one with a higher weight
     entries = [
         {"category_code": "NEWS", "domain": f"d{i}.org", "url": f"https://d{i}.org/", "cc": "ZZ", "msmt_cnt": 0}
@@ -198,13 +200,42 @@ def test_compute_priorities_random_tie_break():
         {"category_code": "NEWS", "cc": "*", "domain": "*", "priority": 100, "url": "*"},
         {"category_code": "*", "cc": "*", "domain": "top.org", "priority": 100, "url": "*"},
     ]
-    orders = set()
-    for _ in range(20):
-        out = prio.compute_priorities(entries, prio_rules)
-        # the ranking holds: highest weight first, weights never increase
-        assert out[0]["url"] == "https://top.org/"
-        assert [o["weight"] for o in out] == sorted((o["weight"] for o in out), reverse=True)
-        orders.add(tuple(o["url"] for o in out[1:]))
-    # the 20 tied URLs come out in different orders; all 20 runs giving the
-    # same order has probability 20!^-19
-    assert len(orders) > 1
+    return entries, prio_rules
+
+
+def urls(out):
+    return [o["url"] for o in out]
+
+
+def test_compute_priorities_seeded_tie_break():
+    entries, prio_rules = tied_entries()
+    out = prio.compute_priorities(entries, prio_rules, seed="IT:30722:20368")
+    # the ranking holds: highest weight first, weights never increase
+    assert out[0]["url"] == "https://top.org/"
+    assert [o["weight"] for o in out] == sorted((o["weight"] for o in out), reverse=True)
+
+    # the same seed gives the same order, whatever order the entries come in
+    # (the query that fetches them has no ORDER BY)
+    shuffled = list(entries)
+    random.Random(1).shuffle(shuffled)
+    assert urls(prio.compute_priorities(shuffled, prio_rules, seed="IT:30722:20368")) == urls(out)
+
+    # other networks and other slots get the tied URLs in other orders; any
+    # two of these being equal has probability 1/20!
+    seeds = ["IT:30722:20368", "IT:30722:20369", "IT:3269:20368", "DE:30722:20368"]
+    assert len({tuple(urls(prio.compute_priorities(entries, prio_rules, seed=s))) for s in seeds}) == len(seeds)
+
+
+def test_compute_priorities_without_seed_keeps_entry_order():
+    entries, prio_rules = tied_entries()
+    out = prio.compute_priorities(entries, prio_rules)
+    assert urls(out) == ["https://top.org/"] + [f"https://d{i}.org/" for i in range(20)]
+
+
+def test_tie_break_seed_changes_per_network_and_day():
+    day = 86400
+    t = 20368 * day + 3600
+    assert prio.tie_break_seed("it", 30722, t) == prio.tie_break_seed("IT", 30722, t + 3600)
+    assert prio.tie_break_seed("IT", 30722, t) != prio.tie_break_seed("IT", 30722, t + day)
+    assert prio.tie_break_seed("IT", 30722, t) != prio.tie_break_seed("IT", 3269, t)
+    assert prio.tie_break_seed("IT", 30722, t) != prio.tie_break_seed("DE", 30722, t)