Merge pull request #779 from Martin005/sourcepos_chars

feat: Add parse option for char-based columns in Sourcepos

Author: kivikakk

fuzz/fuzz_targets/all_options.rs

@@ -134,6 +134,7 @@ struct FuzzParseOptions {
     default_info_string: bool,
     broken_link_callback: bool,
     escaped_char_spans: bool,
+    sourcepos_chars: bool,
 }
 
 impl FuzzParseOptions {
@@ -159,6 +160,7 @@ impl FuzzParseOptions {
                 None
             },
             escaped_char_spans: self.escaped_char_spans,
+            sourcepos_chars: self.sourcepos_chars,
         }
     }
 }

src/main.rs

@@ -167,6 +167,11 @@ struct Cli {
     #[arg(long)]
     ignore_empty_links: bool,
 
+    /// Report column positions in sourcepos as a Unicode character count
+    /// rather than UTF-8 byte offsets
+    #[arg(long)]
+    sourcepos_chars: bool,
+
     /// Minimise escapes in CommonMark output using a trial-and-error algorithm
     #[arg(long)]
     experimental_minimize_commonmark: bool,
@@ -323,6 +328,7 @@ fn main() -> Result<(), Box<dyn Error>> {
         .relaxed_tasklist_matching(cli.relaxed_tasklist_character)
         .relaxed_autolinks(cli.relaxed_autolinks)
         .ignore_setext(cli.ignore_setext)
+        .sourcepos_chars(cli.sourcepos_chars)
         .build();
 
     let render = options::Render::builder()

src/nodes.rs

@@ -806,16 +806,23 @@ impl From<(usize, usize, usize, usize)> for Sourcepos {
 
 /// Represents the 1-based line and column positions of a given character.
 ///
-/// The `column` value is measured in UTF-8 byte offsets (1-based),
+/// By default, the `column` value is measured in UTF-8 byte offsets (1-based),
 /// matching cmark behavior. This means multi-byte UTF-8 characters
 /// (for example, `ö` or `好`) increase the column count by their byte
 /// length rather than by Rust's `char` count.
+///
+/// Enable [`parse.sourcepos_chars`][crate::options::Parse#structfield.sourcepos_chars] to have
+/// column values reported as a Unicode character count instead.
 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct LineColumn {
     /// The 1-based line number of the character.
     pub line: usize,
-    /// The 1-based column number of the character, counted as UTF-8 bytes.
-    /// For example, a 3-byte UTF-8 character increments the column by 3.
+    /// The 1-based column number of the character.
+    ///
+    /// By default this is counted in UTF-8 bytes (so a 3-byte character
+    /// increments the column by 3).  Enable
+    /// [`parse.sourcepos_chars`][crate::options::Parse#structfield.sourcepos_chars] to have
+    /// it reported as a Unicode character count instead.
     pub column: usize,
 }
 

src/parser/mod.rs

@@ -53,7 +53,49 @@ pub fn parse_document<'a>(arena: &'a Arena<'a>, md: &str, options: &Options) ->
         }
         .into(),
     );
-    Parser::new(arena, root, options).parse(md)
+    let document = Parser::new(arena, root, options).parse(md);
+    if options.parse.sourcepos_chars {
+        convert_sourcepos_columns_to_chars(document, md);
+    }
+    document
+}
+
+/// Convert all byte-based column values in the AST's sourcepos to char-based.
+fn convert_sourcepos_columns_to_chars(document: Node<'_>, md: &str) {
+    let lines: Vec<&str> = md.lines().collect();
+
+    let convert = |lc: &mut nodes::LineColumn| {
+        if lc.column == 0 {
+            return;
+        }
+        if let Some(line) = lines.get(lc.line.wrapping_sub(1)) {
+            lc.column = byte_col_to_char_col(line, lc.column);
+        }
+    };
+
+    for node in document.descendants() {
+        let mut ast = node.data_mut();
+        convert(&mut ast.sourcepos.start);
+        convert(&mut ast.sourcepos.end);
+    }
+}
+
+/// Convert a 1-based byte column index to a 1-based char column index for the given line.
+fn byte_col_to_char_col(line: &str, byte_col: usize) -> usize {
+    // If the byte column points past the end of the line (e.g. position
+    // after the final byte, such as softbreak/newline positions),
+    // map it to the position after the last Unicode character.
+    if byte_col > line.len() {
+        return line.chars().count() + 1;
+    }
+
+    // Count all chars whose start byte offset is <= byte_idx.
+    // Since `char_indices()` yields (byte_offset_of_char_start, char),
+    // `take_while` includes exactly the chars that start at or before `byte_idx`.
+    let byte_idx = byte_col - 1;
+    line.char_indices()
+        .take_while(|&(i, _)| i <= byte_idx)
+        .count()
 }
 
 /// Return whether the byte at the given offset passes the callback.

src/parser/options.rs

@@ -861,6 +861,37 @@ pub struct Parse<'c> {
     /// this option to be enabled.
     #[cfg_attr(feature = "bon", builder(default))]
     pub escaped_char_spans: bool,
+
+    /// When enabled, the [`column`][crate::nodes::LineColumn::column] values in
+    /// [`Sourcepos`][crate::nodes::Sourcepos] are counted as Unicode characters
+    /// (i.e. `char`s) rather than as UTF-8 bytes.
+    ///
+    /// By default, column values follow cmark behaviour: each byte of a
+    /// multi-byte UTF-8 character counts as a separate column. Enabling this
+    /// option converts those byte-based columns to character-based columns after
+    /// parsing, so that a 3-byte character such as `好` occupies only one
+    /// column position instead of three.
+    ///
+    /// ```rust
+    /// # use comrak::{Arena, parse_document, Options};
+    /// let arena = Arena::new();
+    /// let mut options = Options::default();
+    ///
+    /// // Default (byte-based): "好" spans columns 1-3
+    /// let root = parse_document(&arena, "好", &options);
+    /// let sp = root.first_child().unwrap().data().sourcepos;
+    /// assert_eq!(sp.start.column, 1);
+    /// assert_eq!(sp.end.column, 3);
+    ///
+    /// // Char-based: "好" occupies only column 1
+    /// options.parse.sourcepos_chars = true;
+    /// let root = parse_document(&arena, "好", &options);
+    /// let sp = root.first_child().unwrap().data().sourcepos;
+    /// assert_eq!(sp.start.column, 1);
+    /// assert_eq!(sp.end.column, 1);
+    /// ```
+    #[cfg_attr(feature = "bon", builder(default))]
+    pub sourcepos_chars: bool,
 }
 
 /// The type of the callback used when a reference link is encountered with no

src/tests.rs

@@ -40,6 +40,7 @@ mod rewriter;
 mod shortcodes;
 #[path = "tests/sourcepos.rs"]
 mod sourcepos_;
+mod sourcepos_chars;
 mod spoiler;
 mod strikethrough;
 mod subscript;
@@ -339,6 +340,17 @@ where
     }
 }
 
+macro_rules! assert_ast_match_set_opt_single {
+    ($opts:ident; $optclass:ident.$optname:ident = $val:expr_2021) => {
+        $opts.$optclass.$optname = $val;
+    };
+    ($opts:ident; $optclass:ident.$optname:ident) => {
+        $opts.$optclass.$optname = true;
+    };
+}
+
+pub(crate) use assert_ast_match_set_opt_single;
+
 macro_rules! assert_ast_match {
     ([ $( $optclass:ident.$optname:ident ),* ], $( $md:literal )+, $amt:tt,) => {
         assert_ast_match!(
@@ -354,6 +366,13 @@ macro_rules! assert_ast_match {
             |#[allow(unused_variables)] opts| {$(opts.$optclass.$optname = $val;)*},
         );
     };
+    ([ $( $optclass:ident.$optname:ident $(= $val:expr_2021)? ),* ], $( $md:literal )+, $amt:tt) => {
+        crate::tests::assert_ast_match_i(
+            concat!( $( $md ),+ ),
+            ast!($amt),
+            |#[allow(unused_variables)] opts| { $( assert_ast_match_set_opt_single!(opts; $optclass.$optname $(= $val)? ); )* },
+        );
+    };
     ([ $( $optclass:ident.$optname:ident ),* ], $( $md:literal )+, $amt:tt) => {
         assert_ast_match!(
             [ $( $optclass.$optname  = true),* ],
@@ -397,6 +416,15 @@ impl AstMatchTree {
                         assert_eq!(text, &ncb.literal, "CodeBlock literal should match");
                         asserted_text = true;
                     }
+                    NodeValue::Code(ref nc) => {
+                        assert_eq!(text, &nc.literal, "Code literal should match");
+                        asserted_text = true;
+                    }
+                    #[cfg(feature = "shortcodes")]
+                    NodeValue::ShortCode(ref nsc) => {
+                        assert_eq!(text, &nsc.code, "Shortcode code should match");
+                        asserted_text = true;
+                    }
                     NodeValue::HtmlBlock(ref nhb) => {
                         assert_eq!(text, &nhb.literal, "HtmlBlock literal should match");
                         asserted_text = true;