Duplicate from different engines

compass/web/search.py

@@ -215,10 +215,16 @@ def _apply_blacklist_filters(results, url_blacklist, url_whitelist):
 
 
 def _apply_duplicate_filters(results):
-    """Mark duplicate rows per search engine and URL"""
+    """Mark duplicate rows by URL, across all search engines
+
+    Winner selection follows the priority encoded in
+    :func:`_link_sort_key`, so the engine listed first in the user
+    config only wins among entries that are otherwise tied (same
+    ``query_rank`` and ``query_index``).
+    """
     winners = {}
     for entry in _active_results_sorted(results):
-        key = (entry["search_engine"], entry["url"])
+        key = entry["url"]
         winner = winners.get(key)
         if winner is None:
             winners[key] = entry
@@ -267,7 +273,7 @@ def _link_sort_key(entry):
         entry["query_rank"],
         -duplicate_count,
         entry["query_index"],
-        entry["search_engine"],
+        entry["se_order"],
         entry["_order"],
     )
 

tests/python/unit/web/test_web_search.py

@@ -78,6 +78,56 @@ def test_apply_duplicate_filters_keeps_best_and_tracks_duplicates():
     assert loser["filtered_reason"] == "duplicate"
 
 
+def test_apply_duplicate_filters_collapses_across_search_engines():
+    """Same URL from different engines should be marked duplicate
+
+    When ``query_rank`` and ``query_index`` tie, the entry from the
+    search engine listed first in the config (lower ``se_order``)
+    wins.
+    """
+    results = [
+        {
+            "url": "https://example.com/a.pdf",
+            "query": "q1",
+            "query_index": 0,
+            "se_order": 1,
+            "search_engine": "TestSearch",
+            "query_rank": 1,
+            "overall_rank": None,
+            "filtered_reason": None,
+            "_order": 0,
+        },
+        {
+            "url": "https://example.com/a.pdf",
+            "query": "q1",
+            "query_index": 0,
+            "se_order": 0,
+            "search_engine": "SerpAPIGoogleSearch",
+            "query_rank": 1,
+            "overall_rank": None,
+            "filtered_reason": None,
+            "_order": 1,
+        },
+    ]
+
+    search_module._apply_duplicate_filters(results)
+
+    winner = results[1]
+    loser = results[0]
+
+    assert winner["filtered_reason"] is None
+    assert winner["search_engine"] == "SerpAPIGoogleSearch"
+    assert winner["duplicates"] == [
+        {
+            "url": "https://example.com/a.pdf",
+            "query": "q1",
+            "search_engine": "TestSearch",
+            "query_rank": 1,
+        }
+    ]
+    assert loser["filtered_reason"] == "duplicate"
+
+
 def test_apply_top_n_filters_assigns_overall_rank_and_beyond_top_n():
     """Assign overall rank and mark entries beyond requested top-N"""
     results = [