feat: support to search across pages (Myriad-Dreamin#99)

* dev: create folder

* dev: support it

* build: update cargo.lock

Cargo.toml

@@ -41,6 +41,7 @@ tokio = { version = "1.42", features = ["full"] }
 serde = { version = "1" }
 serde_json = "1"
 toml = "0.8"
+regex = "1.8.1"
 
 # web
 warp = { version = "0.3", features = ["compression"] }
@@ -54,6 +55,9 @@ clap_complete_fig = "4.5"
 env_logger = "0.11"
 log = "0.4.25"
 
+# search
+elasticlunr-rs = "3.0.2"
+
 # misc
 vergen = { version = "9.0.4", features = ["build", "cargo", "rustc"] }
 vergen-gitcl = { version = "1.0.1" }

cli/Cargo.toml

@@ -27,7 +27,6 @@ clap_complete.workspace = true
 clap_complete_fig.workspace = true
 
 comemo.workspace = true
-# chrono.workspace = true
 tokio.workspace = true
 indexmap = "2"
 url = "2"
@@ -40,15 +39,12 @@ toml.workspace = true
 env_logger.workspace = true
 log.workspace = true
 
-# flate2.workspace = true
-
-# codespan-reporting.workspace = true
-# human-panic.workspace = true
-
 reflexo-typst.workspace = true
 reflexo-vec2svg.workspace = true
 handlebars.workspace = true
 pathdiff.workspace = true
+elasticlunr-rs.workspace = true
+regex.workspace = true
 
 [build-dependencies]
 anyhow.workspace = true

cli/src/meta.rs

@@ -60,3 +60,56 @@ pub struct BuildMeta {
     #[serde(rename = "dest-dir")]
     pub dest_dir: String,
 }
+
+/// Configuration of the search functionality of the HTML renderer.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(default, rename_all = "kebab-case")]
+pub struct Search {
+    /// Enable the search feature. Default: `true`.
+    pub enable: bool,
+    /// Maximum number of visible results. Default: `30`.
+    pub limit_results: u32,
+    /// The number of words used for a search result teaser. Default: `30`.
+    pub teaser_word_count: u32,
+    /// Define the logical link between multiple search words.
+    /// If true, all search words must appear in each result. Default: `false`.
+    pub use_boolean_and: bool,
+    /// Boost factor for the search result score if a search word appears in the
+    /// header. Default: `2`.
+    pub boost_title: u8,
+    /// Boost factor for the search result score if a search word appears in the
+    /// hierarchy. The hierarchy contains all titles of the parent documents
+    /// and all parent headings. Default: `1`.
+    pub boost_hierarchy: u8,
+    /// Boost factor for the search result score if a search word appears in the
+    /// text. Default: `1`.
+    pub boost_paragraph: u8,
+    /// True if the searchword `micro` should match `microwave`. Default:
+    /// `true`.
+    pub expand: bool,
+    /// Documents are split into smaller parts, separated by headings. This
+    /// defines, until which level of heading documents should be split.
+    /// Default: `3`. (`### This is a level 3 heading`)
+    pub heading_split_level: u8,
+    /// Copy JavaScript files for the search functionality to the output
+    /// directory? Default: `true`.
+    pub copy_js: bool,
+}
+
+impl Default for Search {
+    fn default() -> Search {
+        // Please update the documentation of `Search` when changing values!
+        Search {
+            enable: true,
+            limit_results: 30,
+            teaser_word_count: 30,
+            use_boolean_and: false,
+            boost_title: 2,
+            boost_hierarchy: 1,
+            boost_paragraph: 1,
+            expand: true,
+            heading_split_level: 3,
+            copy_js: true,
+        }
+    }
+}

cli/src/project.rs

@@ -10,7 +10,7 @@ use serde_json::json;
 use crate::{
     error::prelude::*,
     meta::{BookMeta, BookMetaContent, BookMetaElem, BuildMeta},
-    render::{DataDict, HtmlRenderer, TypstRenderer},
+    render::{DataDict, HtmlRenderer, SearchRenderer, TypstRenderer},
     theme::Theme,
     utils::{create_dirs, make_absolute, release_packages, write_file, UnwrapOrExit},
     CompileArgs, MetaSource,
@@ -48,6 +48,7 @@ pub struct Project {
     pub theme: Theme,
     pub tr: TypstRenderer,
     pub hr: HtmlRenderer,
+    pub sr: SearchRenderer,
 
     pub book_meta: Option<BookMeta>,
     pub build_meta: Option<BuildMeta>,
@@ -100,13 +101,15 @@ impl Project {
 
         let tr = TypstRenderer::new(args);
         let hr = HtmlRenderer::new(&theme);
+        let sr = SearchRenderer::default();
 
         let mut proj = Self {
             dest_dir: tr.dest_dir.clone(),
 
             theme,
             tr,
             hr,
+            sr,
 
             book_meta: None,
             build_meta: None,
@@ -298,6 +301,21 @@ impl Project {
             include_bytes!("../../assets/artifacts/book.mjs"),
         )?;
 
+        if self.sr.config.copy_js {
+            write_file(
+                self.dest_dir.join("internal/searcher.js"),
+                include_bytes!("../../assets/artifacts/searcher.js"),
+            )?;
+            write_file(
+                self.dest_dir.join("internal/mark.min.js"),
+                include_bytes!("../../assets/artifacts/mark.min.js"),
+            )?;
+            write_file(
+                self.dest_dir.join("internal/elasticlunr.min.js"),
+                include_bytes!("../../assets/artifacts/elasticlunr.min.js"),
+            )?;
+        }
+
         self.prepare_chapters();
         for ch in self.chapters.clone() {
             if let Some(path) = ch.get("path") {
@@ -322,6 +340,10 @@ impl Project {
             }
         }
 
+        if self.sr.config.copy_js {
+            self.sr.render_search_index(&self.dest_dir)?;
+        }
+
         Ok(())
     }
 
@@ -404,23 +426,29 @@ impl Project {
         chapters
     }
 
-    pub fn compile_chapter(&mut self, _ch: DataDict, path: &str) -> ZResult<ChapterArtifact> {
-        let rel_data_path = std::path::Path::new(&self.path_to_root)
+    pub fn compile_chapter(&mut self, path: &str) -> ZResult<ChapterArtifact> {
+        let file_name = std::path::Path::new(&self.path_to_root)
             .join(path)
-            .with_extension("")
-            .to_str()
-            .ok_or_else(|| error_once!("path_to_root is not a valid utf-8 string"))?
-            // windows
-            .replace('\\', "/");
+            .with_extension("");
 
         // todo: description for single document
-        let mut description = "".to_owned();
+        let mut full_digest = "".to_owned();
         if self.need_compile() {
             let doc = self.tr.compile_page(Path::new(path))?;
-            description = self.tr.generate_desc(&doc)?;
+            full_digest = self.tr.generate_desc(&doc)?;
         }
+        let description = match full_digest.char_indices().nth(512) {
+            Some((idx, _)) => full_digest[..idx].to_owned(),
+            None => full_digest,
+        };
 
-        let dynamic_load_trampoline = self
+        let rel_data_path = file_name
+            .to_str()
+            .ok_or_else(|| error_once!("path_to_root is not a valid utf-8 string"))?
+            // windows
+            .replace('\\', "/");
+
+        let content = self
             .hr
             .handlebars
             .render(
@@ -434,45 +462,57 @@ impl Project {
             ))?;
 
         Ok(ChapterArtifact {
-            content: dynamic_load_trampoline.to_owned(),
-            description: escape_str::<AttributeEscapes>(
-                &description.chars().take(512).collect::<String>(),
-            )
-            .into_owned(),
+            content,
+            description,
         })
     }
 
     pub fn render_chapter(&mut self, chapter_data: DataDict, path: &str) -> ZResult<String> {
         let instant = std::time::Instant::now();
-        log::info!("rendering chapter {}", path);
-        // println!("RC = {:?}", rc);
+
+        let file_path = std::path::Path::new(&self.path_to_root)
+            .join(path)
+            .with_extension("");
+
+        log::info!("rendering chapter {path}");
+
+        // Compiles the chapter
+        let art = self.compile_chapter(path)?;
+
+        let title = chapter_data
+            .get("name")
+            .and_then(|t| t.as_str())
+            .ok_or_else(|| error_once!("no name in chapter data"))?;
+        self.index_search(&file_path.with_extension("html"), title, &art.description);
+
+        // Creates the data to inject in the template
         let data = serde_json::to_value(self.book_meta.clone())
             .map_err(map_string_err("render_chapter,convert_to<BookMeta>"))?;
         let mut data: DataDict = serde_json::from_value(data)
             .map_err(map_string_err("render_chapter,convert_to<BookMeta>"))?;
 
-        // inject chapters
-        data.insert("chapters".to_owned(), json!(self.chapters));
+        // Injects search configuration
+        let search = &self.sr.config;
+        data.insert("search_enabled".to_owned(), json!(search.enable));
+        data.insert(
+            "search_js".to_owned(),
+            json!(search.enable && search.copy_js),
+        );
 
+        // Injects module path
         let renderer_module = format!("{}internal/typst_ts_renderer_bg.wasm", self.path_to_root);
         data.insert("renderer_module".to_owned(), json!(renderer_module));
 
-        // inject content
+        // Injects description
+        let desc = escape_str::<AttributeEscapes>(&art.description).into_owned();
+        data.insert("description".to_owned(), serde_json::Value::String(desc));
 
-        let art = self.compile_chapter(chapter_data, path)?;
-
-        // inject content
-        data.insert(
-            "description".to_owned(),
-            serde_json::Value::String(art.description),
-        );
+        data.insert("chapters".to_owned(), json!(self.chapters));
         data.insert("content".to_owned(), serde_json::Value::String(art.content));
-
-        // inject path_to_root
         data.insert("path_to_root".to_owned(), json!(self.path_to_root));
 
         let index_html = self.hr.render_index(data, path);
-        log::info!("rendering chapter {} in {:?}", path, instant.elapsed());
+        log::info!("rendering chapter {path} in {:?}", instant.elapsed());
         Ok(index_html)
     }
 

cli/src/render/mod.rs

@@ -1,9 +1,9 @@
 pub mod html;
-use std::collections::BTreeMap;
-
 pub use self::html::*;
-
 pub mod typst;
 pub use self::typst::*;
+pub mod search;
+pub use self::search::*;
 
+use std::collections::BTreeMap;
 pub type DataDict = BTreeMap<String, serde_json::Value>;