document generic search query flow

Author: mikkeldenker

crates/core/src/generic_query/get_homepage.rs

@@ -72,7 +72,7 @@ impl GenericQuery for GetHomepageQuery {
         FirstDocCollector::with_shard_id(ctx.shard_id)
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         FirstDocCollector::without_shard_id()
     }
 

crates/core/src/generic_query/get_site_urls.rs

@@ -65,7 +65,7 @@ impl GenericQuery for GetSiteUrlsQuery {
             .disable_offset()
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         Self::Collector::new()
             .with_limit(self.limit as usize)
             .with_offset(self.offset.unwrap_or(0) as usize)

crates/core/src/generic_query/get_webpage.rs

@@ -68,7 +68,7 @@ impl GenericQuery for GetWebpageQuery {
         FirstDocCollector::with_shard_id(ctx.shard_id)
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         FirstDocCollector::without_shard_id()
     }
 

crates/core/src/generic_query/mod.rs

@@ -14,6 +14,26 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+//! # Main flow
+//! ```md
+//! `coordinator`  <------>  `searcher`
+//! -----------------------------------
+//! send query to searcher
+//!                        search index
+//!                        collect fruits
+//!                        send fruits to coordinator
+//! merge fruits
+//! filter fruits
+//!     for each shard
+//! send fruits to searchers
+//!                        construct intermediate output
+//!                           from fruits
+//!                        send intermediate output to coordinator
+//! merge intermediate outputs
+//! return final output
+//! ---------------------------------------------------
+//! ```
+
 use crate::{inverted_index::ShardId, search_ctx, Result};
 
 pub mod top_key_phrases;
@@ -34,6 +54,8 @@ pub use get_site_urls::GetSiteUrlsQuery;
 pub mod collector;
 pub use collector::Collector;
 
+/// A generic query that can be executed on a searcher
+/// against an index.
 pub trait GenericQuery: Send + Sync + bincode::Encode + bincode::Decode + Clone {
     type Collector: Collector;
     type TantivyQuery: tantivy::query::Query;
@@ -42,7 +64,7 @@ pub trait GenericQuery: Send + Sync + bincode::Encode + bincode::Decode + Clone
 
     fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery;
     fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector;
-    fn remote_collector(&self) -> Self::Collector;
+    fn coordinator_collector(&self) -> Self::Collector;
 
     fn filter_fruit_shards(
         &self,

crates/core/src/generic_query/size.rs

@@ -70,7 +70,7 @@ impl GenericQuery for SizeQuery {
         SizeCollector::new().with_shard_id(ctx.shard_id)
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         SizeCollector::new()
     }
 

crates/core/src/generic_query/top_key_phrases.rs

@@ -47,7 +47,7 @@ impl GenericQuery for TopKeyPhrasesQuery {
         TopKeyPhrasesCollector::new(self.top_n).with_shard_id(ctx.shard_id)
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopKeyPhrasesCollector::new(self.top_n)
     }
 

crates/core/src/inverted_index/search.rs

@@ -341,7 +341,9 @@ impl InvertedIndex {
             From<<Q::Collector as generic_query::Collector>::Fruit>,
     {
         let fruit = self.search_initial_generic(query)?;
-        let mut fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;
+        let mut fruit = query
+            .coordinator_collector()
+            .merge_fruits(vec![fruit.into()])?;
 
         if let Some(shard_id) = self.shard_id {
             fruit = query.filter_fruit_shards(shard_id, fruit);

crates/core/src/searcher/distributed.rs

@@ -414,7 +414,7 @@ impl SearchClient for DistributedSearcher {
         >: From<<Q as sonic::service::Message<SearchService>>::Response>,
         <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
             From<<Q::Collector as generic_query::Collector>::Fruit> {
-        let collector = query.remote_collector();
+        let collector = query.coordinator_collector();
 
         let res = self.conn().await
             .send(query, &AllShardsSelector, &RandomReplicaSelector)
@@ -518,7 +518,7 @@ impl SearchClient for DistributedSearcher {
             queries
                 .iter()
                 .zip_eq(fruits.into_iter())
-                .map(|(query, shard_fruits)| query.remote_collector().merge_fruits(shard_fruits))
+                .map(|(query, shard_fruits)| query.coordinator_collector().merge_fruits(shard_fruits))
                 .collect::<Result<Vec<_>, _>>()
     }
 

crates/core/src/webgraph/query/backlink.rs

@@ -182,7 +182,7 @@ impl Query for BacklinksQuery {
         collector
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopDocsCollector::from(self.limit).enable_offset()
     }
 
@@ -314,7 +314,7 @@ impl Query for HostBacklinksQuery {
         collector
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopDocsCollector::from(self.limit)
             .enable_offset()
             .with_deduplicator(HostDeduplicator)
@@ -467,7 +467,7 @@ impl Query for FullBacklinksQuery {
         collector
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopDocsCollector::from(self.limit).enable_offset()
     }
 
@@ -617,7 +617,7 @@ impl Query for FullHostBacklinksQuery {
         collector
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopDocsCollector::from(self.limit)
             .enable_offset()
             .with_deduplicator(HostDeduplicator)
@@ -742,7 +742,7 @@ impl Query for BacklinksWithLabelsQuery {
         collector
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopDocsCollector::from(self.limit).enable_offset()
     }
 

crates/core/src/webgraph/query/between.rs

@@ -113,7 +113,7 @@ impl Query for FullLinksBetweenQuery {
         collector
     }
 
-    fn remote_collector(&self) -> Self::Collector {
+    fn coordinator_collector(&self) -> Self::Collector {
         TopDocsCollector::from(self.limit).enable_offset()
     }