Vendor cocoindex_ops_text

Author: Ubuntu

Cargo.lock

@@ -7,8 +7,8 @@ edition = "2021"
 name = "coco_rs"
 
 [dependencies]
-# 核心引擎依赖 (本地路径)
-cocoindex_ops_text = { path = "../cocoindex/rust/ops_text" }
+# Vendored syntax-aware text chunking crate
+cocoindex_ops_text = { path = "vendor/cocoindex_ops_text" }
 
 # 异步与 IO
 tokio = { version = "1.48", features = ["full"] }

Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "cocoindex_ops_text"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+
+[dependencies]
+anyhow = { version = "1.0", features = ["std"] }
+globset = "0.4"
+regex = "1.12.2"
+unicase = "2.8.1"
+
+tree-sitter = "0.25.10"
+tree-sitter-language = "0.1.7"
+# Per language tree-sitter parsers
+tree-sitter-c = "0.24.1"
+tree-sitter-cpp = "0.23.4"
+tree-sitter-c-sharp = "0.23.1"
+tree-sitter-css = "0.23.2"
+tree-sitter-fortran = "0.5.1"
+tree-sitter-go = "0.23.4"
+tree-sitter-html = "0.23.2"
+tree-sitter-java = "0.23.5"
+tree-sitter-javascript = "0.23.1"
+tree-sitter-json = "0.24.8"
+# The other more popular crate tree-sitter-kotlin requires tree-sitter < 0.23 for now
+tree-sitter-kotlin-ng = "1.1.0"
+tree-sitter-md = "0.5.3"
+tree-sitter-pascal = "0.10.2"
+tree-sitter-php = "0.23.11"
+tree-sitter-python = "0.23.6"
+tree-sitter-r = "1.2.0"
+tree-sitter-ruby = "0.23.1"
+tree-sitter-rust = "0.24.0"
+tree-sitter-scala = "0.24.0"
+tree-sitter-sequel = "0.3.11"
+tree-sitter-swift = "0.7.1"
+tree-sitter-toml-ng = "0.7.0"
+tree-sitter-typescript = "0.23.2"
+tree-sitter-xml = "0.7.0"
+tree-sitter-yaml = "0.7.2"
+tree-sitter-solidity = "1.2.13"
+
+[dev-dependencies]

vendor/cocoindex_ops_text/Cargo.toml

@@ -0,0 +1,11 @@
+//! Extra text processing utilities for CocoIndex.
+//!
+//! This crate provides text processing functionality including:
+//! - Programming language detection and tree-sitter support
+//! - Text splitting by separators
+//! - Recursive text chunking with syntax awareness
+
+pub(crate) mod output_positions;
+pub mod pattern_matcher;
+pub mod prog_langs;
+pub mod split;

vendor/cocoindex_ops_text/src/lib.rs

@@ -0,0 +1,322 @@
+//! Internal module for computing output positions from byte offsets.
+
+/// A text range specified by byte offsets.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TextRange {
+    /// Start byte offset (inclusive).
+    pub start: usize,
+    /// End byte offset (exclusive).
+    pub end: usize,
+}
+
+impl TextRange {
+    /// Create a new text range.
+    pub fn new(start: usize, end: usize) -> Self {
+        Self { start, end }
+    }
+
+    /// Get the length of the range in bytes.
+    pub fn len(&self) -> usize {
+        self.end - self.start
+    }
+
+    /// Check if the range is empty.
+    pub fn is_empty(&self) -> bool {
+        self.start >= self.end
+    }
+}
+
+/// Output position information with character offset and line/column.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct OutputPosition {
+    /// Character (not byte) offset from the start of the text.
+    pub char_offset: usize,
+    /// 1-based line number.
+    pub line: u32,
+    /// 1-based column number.
+    pub column: u32,
+}
+
+/// Position tracking helper that converts byte offsets to character positions.
+pub(crate) struct Position {
+    /// The byte offset in the text.
+    pub byte_offset: usize,
+    /// Computed output position (populated by `set_output_positions`).
+    pub output: Option<OutputPosition>,
+}
+
+impl Position {
+    /// Create a new position with the given byte offset.
+    pub fn new(byte_offset: usize) -> Self {
+        Self {
+            byte_offset,
+            output: None,
+        }
+    }
+}
+
+/// Fill OutputPosition for the requested byte offsets.
+///
+/// This function efficiently computes character offsets, line numbers, and column
+/// numbers for a set of byte positions in a single pass through the text.
+pub(crate) fn set_output_positions<'a>(
+    text: &str,
+    positions: impl Iterator<Item = &'a mut Position>,
+) {
+    let mut positions = positions.collect::<Vec<_>>();
+    positions.sort_by_key(|o| o.byte_offset);
+
+    let mut positions_iter = positions.iter_mut();
+    let Some(mut next_position) = positions_iter.next() else {
+        return;
+    };
+
+    let mut char_offset = 0;
+    let mut line = 1;
+    let mut column = 1;
+    for (byte_offset, ch) in text.char_indices() {
+        while next_position.byte_offset == byte_offset {
+            next_position.output = Some(OutputPosition {
+                char_offset,
+                line,
+                column,
+            });
+            if let Some(p) = positions_iter.next() {
+                next_position = p
+            } else {
+                return;
+            }
+        }
+        char_offset += 1;
+        if ch == '\n' {
+            line += 1;
+            column = 1;
+        } else {
+            column += 1;
+        }
+    }
+
+    loop {
+        next_position.output = Some(OutputPosition {
+            char_offset,
+            line,
+            column,
+        });
+        if let Some(p) = positions_iter.next() {
+            next_position = p
+        } else {
+            return;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_set_output_positions_simple() {
+        let text = "abc";
+        let mut start = Position::new(0);
+        let mut end = Position::new(3);
+
+        set_output_positions(text, vec![&mut start, &mut end].into_iter());
+
+        assert_eq!(
+            start.output,
+            Some(OutputPosition {
+                char_offset: 0,
+                line: 1,
+                column: 1,
+            })
+        );
+        assert_eq!(
+            end.output,
+            Some(OutputPosition {
+                char_offset: 3,
+                line: 1,
+                column: 4,
+            })
+        );
+    }
+
+    #[test]
+    fn test_set_output_positions_with_newlines() {
+        let text = "ab\ncd\nef";
+        let mut pos1 = Position::new(0);
+        let mut pos2 = Position::new(3); // 'c'
+        let mut pos3 = Position::new(6); // 'e'
+        let mut pos4 = Position::new(8); // end
+
+        set_output_positions(
+            text,
+            vec![&mut pos1, &mut pos2, &mut pos3, &mut pos4].into_iter(),
+        );
+
+        assert_eq!(
+            pos1.output,
+            Some(OutputPosition {
+                char_offset: 0,
+                line: 1,
+                column: 1,
+            })
+        );
+        assert_eq!(
+            pos2.output,
+            Some(OutputPosition {
+                char_offset: 3,
+                line: 2,
+                column: 1,
+            })
+        );
+        assert_eq!(
+            pos3.output,
+            Some(OutputPosition {
+                char_offset: 6,
+                line: 3,
+                column: 1,
+            })
+        );
+        assert_eq!(
+            pos4.output,
+            Some(OutputPosition {
+                char_offset: 8,
+                line: 3,
+                column: 3,
+            })
+        );
+    }
+
+    #[test]
+    fn test_set_output_positions_multibyte() {
+        // Test with emoji (4-byte UTF-8 character)
+        let text = "abc\u{1F604}def"; // abc + emoji (4 bytes) + def
+        let mut start = Position::new(0);
+        let mut before_emoji = Position::new(3);
+        let mut after_emoji = Position::new(7); // byte position after emoji
+        let mut end = Position::new(10);
+
+        set_output_positions(
+            text,
+            vec![&mut start, &mut before_emoji, &mut after_emoji, &mut end].into_iter(),
+        );
+
+        assert_eq!(
+            start.output,
+            Some(OutputPosition {
+                char_offset: 0,
+                line: 1,
+                column: 1,
+            })
+        );
+        assert_eq!(
+            before_emoji.output,
+            Some(OutputPosition {
+                char_offset: 3,
+                line: 1,
+                column: 4,
+            })
+        );
+        assert_eq!(
+            after_emoji.output,
+            Some(OutputPosition {
+                char_offset: 4, // 3 chars + 1 emoji
+                line: 1,
+                column: 5,
+            })
+        );
+        assert_eq!(
+            end.output,
+            Some(OutputPosition {
+                char_offset: 7, // 3 + 1 + 3
+                line: 1,
+                column: 8,
+            })
+        );
+    }
+
+    #[test]
+    fn test_translate_bytes_to_chars_detailed() {
+        // Comprehensive test moved from cocoindex
+        let text = "abc\u{1F604}def";
+        let mut start1 = Position::new(0);
+        let mut end1 = Position::new(3);
+        let mut start2 = Position::new(3);
+        let mut end2 = Position::new(7);
+        let mut start3 = Position::new(7);
+        let mut end3 = Position::new(10);
+        let mut end_full = Position::new(text.len());
+
+        let offsets = vec![
+            &mut start1,
+            &mut end1,
+            &mut start2,
+            &mut end2,
+            &mut start3,
+            &mut end3,
+            &mut end_full,
+        ];
+
+        set_output_positions(text, offsets.into_iter());
+
+        assert_eq!(
+            start1.output,
+            Some(OutputPosition {
+                char_offset: 0,
+                line: 1,
+                column: 1,
+            })
+        );
+        assert_eq!(
+            end1.output,
+            Some(OutputPosition {
+                char_offset: 3,
+                line: 1,
+                column: 4,
+            })
+        );
+        assert_eq!(
+            start2.output,
+            Some(OutputPosition {
+                char_offset: 3,
+                line: 1,
+                column: 4,
+            })
+        );
+        assert_eq!(
+            end2.output,
+            Some(OutputPosition {
+                char_offset: 4,
+                line: 1,
+                column: 5,
+            })
+        );
+        assert_eq!(
+            end3.output,
+            Some(OutputPosition {
+                char_offset: 7,
+                line: 1,
+                column: 8,
+            })
+        );
+        assert_eq!(
+            end_full.output,
+            Some(OutputPosition {
+                char_offset: 7,
+                line: 1,
+                column: 8,
+            })
+        );
+    }
+
+    #[test]
+    fn test_text_range() {
+        let range = TextRange::new(0, 10);
+        assert_eq!(range.len(), 10);
+        assert!(!range.is_empty());
+
+        let empty = TextRange::new(5, 5);
+        assert_eq!(empty.len(), 0);
+        assert!(empty.is_empty());
+    }
+}