diff --git a/compass/web/search.py b/compass/web/search.py index deb60ffbc..eed4af356 100644 --- a/compass/web/search.py +++ b/compass/web/search.py @@ -215,10 +215,16 @@ def _apply_blacklist_filters(results, url_blacklist, url_whitelist): def _apply_duplicate_filters(results): - """Mark duplicate rows per search engine and URL""" + """Mark duplicate rows by URL, across all search engines + + Winner selection follows the priority encoded in + :func:`_link_sort_key`, so the engine listed first in the user + config only wins among entries that are otherwise tied (same + ``query_rank`` and ``query_index``). + """ winners = {} for entry in _active_results_sorted(results): - key = (entry["search_engine"], entry["url"]) + key = entry["url"] winner = winners.get(key) if winner is None: winners[key] = entry @@ -267,7 +273,7 @@ def _link_sort_key(entry): entry["query_rank"], -duplicate_count, entry["query_index"], - entry["search_engine"], + entry["se_order"], entry["_order"], ) diff --git a/tests/python/unit/web/test_web_search.py b/tests/python/unit/web/test_web_search.py index 192f8323f..b7299ec50 100644 --- a/tests/python/unit/web/test_web_search.py +++ b/tests/python/unit/web/test_web_search.py @@ -78,6 +78,56 @@ def test_apply_duplicate_filters_keeps_best_and_tracks_duplicates(): assert loser["filtered_reason"] == "duplicate" +def test_apply_duplicate_filters_collapses_across_search_engines(): + """Same URL from different engines should be marked duplicate + + When ``query_rank`` and ``query_index`` tie, the entry from the + search engine listed first in the config (lower ``se_order``) + wins. + """ + results = [ + { + "url": "https://example.com/a.pdf", + "query": "q1", + "query_index": 0, + "se_order": 1, + "search_engine": "TestSearch", + "query_rank": 1, + "overall_rank": None, + "filtered_reason": None, + "_order": 0, + }, + { + "url": "https://example.com/a.pdf", + "query": "q1", + "query_index": 0, + "se_order": 0, + "search_engine": "SerpAPIGoogleSearch", + "query_rank": 1, + "overall_rank": None, + "filtered_reason": None, + "_order": 1, + }, + ] + + search_module._apply_duplicate_filters(results) + + winner = results[1] + loser = results[0] + + assert winner["filtered_reason"] is None + assert winner["search_engine"] == "SerpAPIGoogleSearch" + assert winner["duplicates"] == [ + { + "url": "https://example.com/a.pdf", + "query": "q1", + "search_engine": "TestSearch", + "query_rank": 1, + } + ] + assert loser["filtered_reason"] == "duplicate" + + def test_apply_top_n_filters_assigns_overall_rank_and_beyond_top_n(): """Assign overall rank and mark entries beyond requested top-N""" results = [