fix(semantic): split sentences that exceed chunk_size

src/chonkie/chunker/semantic.py

@@ -395,6 +395,42 @@ def _group_sentences(
 
         return groups
 
+    def _split_oversized_sentence(self, sentence: Sentence) -> list[Sentence]:
+        """Split a single sentence that exceeds chunk_size into token-sized pieces.
+
+        A sentence longer than chunk_size can never fit into any group, so grouping
+        alone cannot enforce the limit. We split its text at the token level, the same
+        way TokenChunker does, so no resulting piece exceeds chunk_size.
+
+        Args:
+            sentence: A sentence whose token_count exceeds chunk_size
+
+        Returns:
+            List of sentences, each with token_count <= chunk_size
+
+        """
+        tokens = self.tokenizer.encode(sentence.text)
+        token_groups = [
+            tokens[i : i + self.chunk_size] for i in range(0, len(tokens), self.chunk_size)
+        ]
+
+        pieces = []
+        current_start = sentence.start_index
+        for group in token_groups:
+            # Decode one group at a time: decode() is part of the tokenizer
+            # protocol, decode_batch() is not guaranteed for custom tokenizers.
+            text = self.tokenizer.decode(group)
+            pieces.append(
+                Sentence(
+                    text=text,
+                    start_index=current_start,
+                    end_index=current_start + len(text),
+                    token_count=len(group),
+                )
+            )
+            current_start += len(text)
+        return pieces
+
     def _split_groups(self, groups: list[list[Sentence]]) -> list[list[Sentence]]:
         """Split groups that exceed chunk_size into smaller groups.
 
@@ -418,14 +454,23 @@ def _split_groups(self, groups: list[list[Sentence]]) -> list[list[Sentence]]:
                 current_token_count = 0
 
                 for sentence in group:
-                    if current_token_count + sentence.token_count <= self.chunk_size:
-                        current_group.append(sentence)
-                        current_token_count += sentence.token_count
-                    else:
-                        if current_group:
-                            final_groups.append(current_group)
-                        current_group = [sentence]
-                        current_token_count = sentence.token_count
+                    # A single sentence larger than chunk_size can't fit in any
+                    # group, so split it into token-sized pieces first.
+                    pieces = (
+                        self._split_oversized_sentence(sentence)
+                        if sentence.token_count > self.chunk_size
+                        else [sentence]
+                    )
+
+                    for piece in pieces:
+                        if current_token_count + piece.token_count <= self.chunk_size:
+                            current_group.append(piece)
+                            current_token_count += piece.token_count
+                        else:
+                            if current_group:
+                                final_groups.append(current_group)
+                            current_group = [piece]
+                            current_token_count = piece.token_count
 
                 if current_group:
                     final_groups.append(current_group)
@@ -465,18 +510,10 @@ def chunk(self, text: str) -> list[Chunk]:
 
         # Handle edge cases - too few sentences
         if len(sentences) <= self.similarity_window:
-            # If we have any sentences, return them as a single chunk
+            # If we have any sentences, group them together, still enforcing
+            # chunk_size so a few large sentences don't produce an oversized chunk.
             if sentences:
-                text = "".join([s.text for s in sentences])
-                token_count = sum([s.token_count for s in sentences])
-                return [
-                    Chunk(
-                        text=text,
-                        start_index=0,
-                        end_index=len(text),
-                        token_count=token_count,
-                    ),
-                ]
+                return self._create_chunks(self._split_groups([sentences]))
             else:
                 return []
 

tests/chunkers/test_semantic_chunker.py

@@ -75,6 +75,63 @@ def test_semantic_chunker_chunking(embedding_model: BaseEmbeddings, sample_text:
     assert all([chunk.end_index is not None for chunk in chunks])
 
 
+def test_semantic_chunker_oversized_single_sentence(embedding_model: BaseEmbeddings) -> None:
+    """A single sentence longer than chunk_size must still be split to respect the limit.
+
+    Regression test for #222: a sentence with no delimiters that exceeds chunk_size
+    was emitted as one oversized chunk because grouping only splits between sentences.
+    """
+    chunk_size = 100
+    chunker = SemanticChunker(
+        embedding_model=embedding_model,
+        chunk_size=chunk_size,
+        threshold=0.5,
+        min_sentences_per_chunk=1,
+    )
+
+    # One long run of words with no sentence delimiters => a single "sentence"
+    # far larger than chunk_size, surrounded by normal sentences.
+    normal = "This is a normal sentence. Here is another one. And a third here. A fourth now. "
+    oversized = "word " * 800
+    text = normal + oversized + ". " + normal
+
+    chunks = chunker.chunk(text)
+
+    assert len(chunks) > 0
+    assert all([chunk.token_count <= chunk_size for chunk in chunks]), (
+        "Every chunk must respect chunk_size even when a single sentence is oversized"
+    )
+    # Guard against token_count being underreported: re-tokenize the actual text.
+    assert all([chunker.tokenizer.count_tokens(chunk.text) <= chunk_size for chunk in chunks]), (
+        "The tokenized chunk text must also stay within chunk_size"
+    )
+
+
+def test_semantic_chunker_oversized_sentence_below_similarity_window(
+    embedding_model: BaseEmbeddings,
+) -> None:
+    """The few-sentences early-return path must also enforce chunk_size (#222)."""
+    chunk_size = 100
+    chunker = SemanticChunker(
+        embedding_model=embedding_model,
+        chunk_size=chunk_size,
+        threshold=0.5,
+        similarity_window=3,
+    )
+
+    # Fewer sentences than similarity_window, but one is far larger than chunk_size.
+    text = "Short intro. " + ("word " * 600)
+    chunks = chunker.chunk(text)
+
+    assert len(chunks) > 1
+    assert all([chunk.token_count <= chunk_size for chunk in chunks]), (
+        "The few-sentences path must also split oversized sentences"
+    )
+    assert all([chunker.tokenizer.count_tokens(chunk.text) <= chunk_size for chunk in chunks]), (
+        "The tokenized chunk text must also stay within chunk_size"
+    )
+
+
 def test_semantic_chunker_empty_text(embedding_model: BaseEmbeddings) -> None:
     """Test that the SemanticChunker can handle empty text input."""
     chunker = SemanticChunker(