From 365341788bee74c0792a40f2c1fee96c044d2a02 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:11:59 -0700 Subject: [PATCH 1/2] Validate float chunk_overlap in TokenChunker, not just int chunk_overlap accepts an int or a float (a fraction of chunk_size). Only the int case was checked against chunk_size, so a float that resolves to int(chunk_overlap * chunk_size) >= chunk_size slipped through. That made the stride chunk_size - chunk_overlap zero or negative: chunk_overlap=1.0 crashed chunk() with 'range() arg 3 must not be zero', and chunk_overlap=1.5 silently returned zero chunks, dropping the entire input. Validate the resolved chunk_overlap after the int/float conversion so both paths are held to the same 'less than chunk_size' invariant, and reject a negative overlap as well. --- src/chonkie/chunker/token.py | 10 +++++-- tests/chunkers/test_token_chunker.py | 39 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/chonkie/chunker/token.py b/src/chonkie/chunker/token.py index 095bcca43..6cbdc4d75 100644 --- a/src/chonkie/chunker/token.py +++ b/src/chonkie/chunker/token.py @@ -48,14 +48,20 @@ def __init__( super().__init__(tokenizer) if chunk_size <= 0: raise ValueError("chunk_size must be positive") - if isinstance(chunk_overlap, int) and chunk_overlap >= chunk_size: - raise ValueError("chunk_overlap must be less than chunk_size") # Assign the values if they make sense self.chunk_size = chunk_size self.chunk_overlap = ( chunk_overlap if isinstance(chunk_overlap, int) else int(chunk_overlap * chunk_size) ) + # Validate the resolved overlap so a float chunk_overlap is held to the + # same invariant as an int one. A float that resolves to >= chunk_size + # would make the chunk stride (chunk_size - chunk_overlap) zero or + # negative, which crashes with range(..., 0) or silently drops all text. + if self.chunk_overlap < 0: + raise ValueError("chunk_overlap must be non-negative") + if self.chunk_overlap >= chunk_size: + raise ValueError("chunk_overlap must be less than chunk_size") self._use_multiprocessing = False diff --git a/tests/chunkers/test_token_chunker.py b/tests/chunkers/test_token_chunker.py index 37e04ba36..8b89d5959 100644 --- a/tests/chunkers/test_token_chunker.py +++ b/tests/chunkers/test_token_chunker.py @@ -309,3 +309,42 @@ def test_token_chunker_return_type(tiktokenizer: Encoding, sample_text: str) -> chunks = chunker.chunk(sample_text) assert all([type(chunk) is Chunk for chunk in chunks]) assert all([len(tiktokenizer.encode(chunk.text)) <= 512 for chunk in chunks]) + + +def test_token_chunker_rejects_int_overlap_equal_to_chunk_size() -> None: + """An int chunk_overlap equal to chunk_size is invalid (existing behavior).""" + with pytest.raises(ValueError, match="chunk_overlap must be less than chunk_size"): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=10) + + +def test_token_chunker_rejects_float_overlap_resolving_to_chunk_size() -> None: + """A float chunk_overlap of 1.0 resolves to chunk_size and must be rejected. + + Previously only the int path was validated, so this constructed a chunker with + a zero stride and chunk() raised 'range() arg 3 must not be zero'. + """ + with pytest.raises(ValueError, match="chunk_overlap must be less than chunk_size"): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=1.0) + + +def test_token_chunker_rejects_float_overlap_exceeding_chunk_size() -> None: + """A float chunk_overlap > 1.0 resolves above chunk_size and must be rejected. + + Previously this produced a negative stride and chunk() silently returned zero + chunks, dropping the entire input text. + """ + with pytest.raises(ValueError, match="chunk_overlap must be less than chunk_size"): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=1.5) + + +def test_token_chunker_accepts_valid_fractional_overlap_and_chunks() -> None: + """A valid fractional float overlap resolves correctly and chunks non-empty text.""" + chunker = TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=0.5) + assert chunker.chunk_overlap == 5 + text = "x" * 40 + chunks = chunker.chunk(text) + assert len(chunks) > 1 + assert "".join(dict.fromkeys([c.text[0] for c in chunks])) == "x" + # Every original character is covered by the first chunk's start through the last chunk's end. + assert chunks[0].start_index == 0 + assert chunks[-1].end_index == len(text) From c5e05db7f70c30547f1abcb9107cafc3445acae3 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Sat, 18 Jul 2026 16:37:14 -0700 Subject: [PATCH 2/2] Reject negative TokenChunker chunk_overlap before float resolution. Small negative floats truncated to 0 via int() and bypassed the post-resolution check. --- src/chonkie/chunker/token.py | 10 ++++------ tests/chunkers/test_token_chunker.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/chonkie/chunker/token.py b/src/chonkie/chunker/token.py index 6cbdc4d75..0dffc6a22 100644 --- a/src/chonkie/chunker/token.py +++ b/src/chonkie/chunker/token.py @@ -48,18 +48,16 @@ def __init__( super().__init__(tokenizer) if chunk_size <= 0: raise ValueError("chunk_size must be positive") + # Validate raw overlap before int() resolution; a small negative float + # like -0.05 truncates to 0 and would otherwise bypass the check below. + if chunk_overlap < 0: + raise ValueError("chunk_overlap must be non-negative") # Assign the values if they make sense self.chunk_size = chunk_size self.chunk_overlap = ( chunk_overlap if isinstance(chunk_overlap, int) else int(chunk_overlap * chunk_size) ) - # Validate the resolved overlap so a float chunk_overlap is held to the - # same invariant as an int one. A float that resolves to >= chunk_size - # would make the chunk stride (chunk_size - chunk_overlap) zero or - # negative, which crashes with range(..., 0) or silently drops all text. - if self.chunk_overlap < 0: - raise ValueError("chunk_overlap must be non-negative") if self.chunk_overlap >= chunk_size: raise ValueError("chunk_overlap must be less than chunk_size") diff --git a/tests/chunkers/test_token_chunker.py b/tests/chunkers/test_token_chunker.py index 8b89d5959..f4bca1766 100644 --- a/tests/chunkers/test_token_chunker.py +++ b/tests/chunkers/test_token_chunker.py @@ -348,3 +348,14 @@ def test_token_chunker_accepts_valid_fractional_overlap_and_chunks() -> None: # Every original character is covered by the first chunk's start through the last chunk's end. assert chunks[0].start_index == 0 assert chunks[-1].end_index == len(text) + + +def test_token_chunker_rejects_negative_overlap() -> None: + """Negative chunk_overlap values must be rejected before resolution.""" + with pytest.raises(ValueError, match="chunk_overlap must be non-negative"): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=-1) + with pytest.raises(ValueError, match="chunk_overlap must be non-negative"): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=-0.1) + # Small negative floats truncate to 0 via int(); must still be rejected. + with pytest.raises(ValueError, match="chunk_overlap must be non-negative"): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=-0.05)