From a46395b9b7c942c2927dcfba61e9b797472115a7 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Tue, 4 Aug 2026 19:44:40 +0530 Subject: [PATCH 01/14] =?UTF-8?q?feat:=20add=20moss-chunking=20=E2=80=94?= =?UTF-8?q?=20pluggable=20chunking=20strategies=20over=20a=20shared=20chun?= =?UTF-8?q?k=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunking isn't in any Moss SDK, so every app rolls its own. The two Python ones in this repo both already wrap chunks in DocumentInfo and still can't be treated uniformly: moss-pikachu emits {path}#chunk-0001 with path/filename/chunk/ extension/modified_at, moss-llamaindex emits {filename}-p{page}-c{idx} with source/page. Zero metadata keys in common. Splitting strategy is contested and content-dependent, so it stays pluggable via a ChunkingStrategy Protocol. The output isn't contested, so this pins it: stable zero-padded IDs, and a declared locator unit (char/line/page) rather than a fixed position field list, since offsets are meaningless for a PDF and pages are meaningless for a source file. Ships CharSplitter (pikachu's 1800/300), SentenceSplitter (llamaindex's 400 words / 2 sentences, regex-based so there's no nltk corpus to provision), ParagraphSplitter and RecursiveSplitter — the sentence/paragraph/recursive set named in ROADMAP.md:112, with semantic deferred since embeddings are a different shape of dependency. Layout mirrors moss-data-connector; ingest drops the template's auto_id, since random UUIDs would defeat the stable IDs. Python only for now — vscode and moss-md-indexer are TS. Depends only on moss. 48 tests. --- .github/workflows/publish-moss-chunking.yml | 134 ++++++++ packages/moss-chunking/.gitignore | 10 + packages/moss-chunking/README.md | 134 ++++++++ packages/moss-chunking/pyproject.toml | 55 +++ packages/moss-chunking/src/__init__.py | 41 +++ packages/moss-chunking/src/chunk.py | 101 ++++++ packages/moss-chunking/src/enrich.py | 41 +++ packages/moss-chunking/src/ingest.py | 27 ++ packages/moss-chunking/src/strategies.py | 291 ++++++++++++++++ packages/moss-chunking/tests/test_chunk.py | 65 ++++ .../moss-chunking/tests/test_strategies.py | 203 +++++++++++ packages/moss-chunking/uv.lock | 318 ++++++++++++++++++ 12 files changed, 1420 insertions(+) create mode 100644 .github/workflows/publish-moss-chunking.yml create mode 100644 packages/moss-chunking/.gitignore create mode 100644 packages/moss-chunking/README.md create mode 100644 packages/moss-chunking/pyproject.toml create mode 100644 packages/moss-chunking/src/__init__.py create mode 100644 packages/moss-chunking/src/chunk.py create mode 100644 packages/moss-chunking/src/enrich.py create mode 100644 packages/moss-chunking/src/ingest.py create mode 100644 packages/moss-chunking/src/strategies.py create mode 100644 packages/moss-chunking/tests/test_chunk.py create mode 100644 packages/moss-chunking/tests/test_strategies.py create mode 100644 packages/moss-chunking/uv.lock diff --git a/.github/workflows/publish-moss-chunking.yml b/.github/workflows/publish-moss-chunking.yml new file mode 100644 index 00000000..13b5f4ed --- /dev/null +++ b/.github/workflows/publish-moss-chunking.yml @@ -0,0 +1,134 @@ +name: Publish moss-chunking + +permissions: + contents: read + +on: + workflow_dispatch: + +concurrency: + group: moss-chunking-release + cancel-in-progress: false + +jobs: + determine-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.compute.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Read version from pyproject + id: compute + shell: python + run: | + import os, pathlib, re, sys + + text = pathlib.Path("packages/moss-chunking/pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'(?m)^version\s*=\s*"([^"]+)"', text) + if not match: + print("Could not find version in pyproject", file=sys.stderr) + sys.exit(1) + version = match.group(1) + + out = pathlib.Path(os.environ["GITHUB_OUTPUT"]) + with out.open("a", encoding="utf-8") as fh: + fh.write(f"version={version}\n") + + print(f"Publishing version: {version}") + + # Build + smoke-test on every supported Python version. Publishing waits for + # the whole matrix (see the publish job's `needs`), so a failure on any + # version blocks the upload/tag instead of racing it. + build-test: + needs: determine-version + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build distributions + working-directory: packages/moss-chunking + run: | + rm -rf dist + python -m build + + - name: Smoke test import + shell: python + run: | + import glob, subprocess, sys + + wheels = glob.glob("packages/moss-chunking/dist/*.whl") + if not wheels: + raise SystemExit("No wheel found in dist/") + + wheel = wheels[0] + subprocess.check_call([sys.executable, "-m", "pip", "install", "--force-reinstall", wheel]) + + import importlib + module = importlib.import_module("moss_chunking") + print("import ok; sample attrs:", dir(module)[:5]) + + # Only runs once every build-test matrix leg has passed. + publish: + needs: [determine-version, build-test] + runs-on: ubuntu-latest + permissions: + contents: write + env: + VERSION: ${{ needs.determine-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Fail if this version was already released + run: | + git fetch --tags --force + if git rev-parse "moss-chunking-v${VERSION}" >/dev/null 2>&1; then + echo "::error::moss-chunking v${VERSION} is already tagged/released; bump the version in pyproject.toml before releasing." + exit 1 + fi + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build distributions + working-directory: packages/moss-chunking + run: | + rm -rf dist + python -m build + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + twine upload packages/moss-chunking/dist/* + + - name: Tag release + run: | + git config user.name "github-actions" + git config user.email "github-actions@users.noreply.github.com" + git tag "moss-chunking-v${VERSION}" + git push origin "moss-chunking-v${VERSION}" diff --git a/packages/moss-chunking/.gitignore b/packages/moss-chunking/.gitignore new file mode 100644 index 00000000..b86b0644 --- /dev/null +++ b/packages/moss-chunking/.gitignore @@ -0,0 +1,10 @@ +build/ +dist/ +*.egg-info/ +__pycache__/ +*.py[cod] +.venv/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.env diff --git a/packages/moss-chunking/README.md b/packages/moss-chunking/README.md new file mode 100644 index 00000000..0a211bee --- /dev/null +++ b/packages/moss-chunking/README.md @@ -0,0 +1,134 @@ +# moss-chunking + +Pluggable text chunking for Moss. Splitters are swappable; the shape of what they +emit is not. + +## Why this exists + +Chunking isn't in any Moss SDK, so every app rolls its own. Two of them, in this +repo, both already wrap chunks in the SDK's `DocumentInfo` — and still can't be +treated uniformly, because they agree on nothing inside it: + +| | `examples/moss-pikachu` | `apps/moss-llamaindex` | +| --- | --- | --- | +| id | `{path}#chunk-0001` | `{filename}-p{page}-c{idx}` | +| metadata | `path`, `filename`, `chunk`, `extension`, `modified_at` | `source`, `page` | +| split | 1800 chars / 300 overlap | 400 words / 2-sentence overlap | + +Zero metadata keys in common. The envelope is shared; the contract is missing. + +Splitting strategy is genuinely contested and content-dependent — code, markdown +and transcripts all want different cuts — which is why it stays pluggable, and +why this is a standalone package rather than something frozen into five language +runtimes. But the *output* isn't contested. Nobody wants a bespoke ID scheme; +they wrote one because none was written down. + +So this package pins the output and leaves the cutting open. + +## Install + +```bash +uv pip install -e ".[dev]" # from packages/moss-chunking +``` + +Depends only on `moss`. Sentence detection is regex-based rather than nltk-backed +so there is no model download or corpus to provision. + +## Use + +```python +from moss_chunking import SentenceSplitter, chunk_document, ingest + +docs = chunk_document(text, source="notes.md", strategy=SentenceSplitter()) +await ingest(docs, project_id, project_key, "my-index") +``` + +`docs` are ordinary `DocumentInfo`s — they go anywhere the SDK takes documents; +`ingest` is just the connector template's one-call shortcut into a fresh index. + +## The contract + +Every chunk, from every strategy, carries: + +| key | meaning | +| --- | --- | +| `source` | what was chunked — path, URL, document name | +| `chunk_index` | position in the sequence, from `0` | +| `locator_type` | `char`, `line` or `page` | +| `locator_start` / `locator_end` | position, in that unit | + +IDs are `{source}#chunk-{index:04d}` — zero-padded so they sort in cut order, and +stable across runs so re-chunking an unchanged document replaces its chunks +rather than duplicating them. Values are all strings, because Moss types metadata +as `Dict[str, str]`. + +That stability is only worth something if you keep it on the way in, which is why +`ingest` drops the one option the connector template it mirrors does offer: +`auto_id`. Random UUIDs defeat the contract — re-indexing an unchanged document +appends a second copy of every chunk instead of replacing what is already there. + +**Position is not a fixed field list.** Plain text is located by character offset, +PDFs by page, code by line; character offsets are meaningless for a PDF and page +numbers are meaningless for a source file. So a chunk declares its unit instead of +assuming one. That's the one real design call in the package. + +Pass source-level facts a splitter can't know via `extra`: + +```python +chunk_document(text, "notes.md", CharSplitter(), extra={"extension": "md"}) +``` + +`extra` cannot shadow the reserved keys above — that's an error, not a silent +overwrite. + +## Strategies + +| | cuts on | ceiling | notes | +| --- | --- | --- | --- | +| `CharSplitter` | fixed character windows | hard | pikachu's 1800/300 are the defaults | +| `SentenceSplitter` | sentence boundaries | soft | llamaindex's 400 words / 2 sentences are the defaults | +| `ParagraphSplitter` | blank lines | soft | never breaks a paragraph, even an oversized one | +| `RecursiveSplitter` | paragraphs → lines → sentences → words | hard | falls back to a hard cut if nothing fits | + +Soft ceiling means a single unit larger than the budget is emitted whole rather +than cut. `RecursiveSplitter` is the one to reach for when the ceiling must hold. + +Write your own by implementing `split(text) -> Iterable[Chunk]`: + +```python +class MyStrategy: + def split(self, text: str) -> Iterator[Chunk]: + yield Chunk(text=..., index=..., locator_type="line", locator_start=..., locator_end=...) +``` + +A strategy never builds a `DocumentInfo` — that's the contract's job, and handing +it to callers is exactly how pikachu and llamaindex drifted apart. + +The invariant the tests enforce for every strategy: +`text[chunk.locator_start:chunk.locator_end] == chunk.text`. Offsets point into +the original string, never a normalized copy. Break it and position metadata +becomes decorative — you can address a chunk but not find it again. + +Semantic chunking is not here yet: it needs embeddings, which makes it a +different shape of dependency. It's the obvious next strategy. + +## Enrichment + +Moss scores BM25 over chunk text, so facts that live only in metadata — the +filename, the folder — are invisible to the keyword half of a hybrid query. +Restating them in the text makes them matchable. Pikachu already does this by +hand; here it's a composable post-step that works with any strategy: + +```python +from moss_chunking import prepend_source_context + +doc = prepend_source_context(doc, filename="notes.md", path="/docs/notes.md") +``` + +ID and metadata are untouched, so an enriched chunk stays addressable. + +## Tests + +```bash +.venv/bin/python -m pytest -q +``` diff --git a/packages/moss-chunking/pyproject.toml b/packages/moss-chunking/pyproject.toml new file mode 100644 index 00000000..6481df38 --- /dev/null +++ b/packages/moss-chunking/pyproject.toml @@ -0,0 +1,55 @@ +[project] +name = "moss-chunking" +version = "0.0.1" +description = "Pluggable text chunking strategies that emit Moss DocumentInfo under a shared ID and metadata contract." +readme = "README.md" +requires-python = ">=3.10,<3.15" +license = { text = "BSD-2-Clause" } +authors = [{ name = "InferEdge Inc.", email = "contact@moss.dev" }] +keywords = ["moss", "chunking", "splitter", "retrieval", "rag"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Text Processing :: Linguistic", +] +dependencies = [ + "moss>=1.1.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "python-dotenv>=1.0.0", + "ruff>=0.5.0", +] + +[project.urls] +Homepage = "https://github.com/usemoss/moss" +Repository = "https://github.com/usemoss/moss" +Source = "https://github.com/usemoss/moss/tree/main/packages/moss-chunking" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +# Flat layout: src/ itself IS the package. +[tool.setuptools] +packages = ["moss_chunking"] +package-dir = { "moss_chunking" = "src" } + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "B", "UP"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/packages/moss-chunking/src/__init__.py b/packages/moss-chunking/src/__init__.py new file mode 100644 index 00000000..c225dfff --- /dev/null +++ b/packages/moss-chunking/src/__init__.py @@ -0,0 +1,41 @@ +"""Pluggable text chunking for Moss. + +Splitters are swappable; the shape of what they emit is not. Every strategy in +here yields chunks that carry a stable ID, declared position metadata, and the +SDK's own `DocumentInfo` — see `chunk.py` for why that contract is the point of +the package. + + from moss_chunking import SentenceSplitter, chunk_document, ingest + + docs = chunk_document(text, source="notes.md", strategy=SentenceSplitter()) + await ingest(docs, project_id, project_key, "my-index") +""" + +from .chunk import LOCATOR_TYPES, RESERVED_KEYS, Chunk, LocatorType, chunk_id +from .enrich import prepend_context, prepend_source_context +from .ingest import ingest +from .strategies import ( + CharSplitter, + ChunkingStrategy, + ParagraphSplitter, + RecursiveSplitter, + SentenceSplitter, + chunk_document, +) + +__all__ = [ + "LOCATOR_TYPES", + "RESERVED_KEYS", + "CharSplitter", + "Chunk", + "ChunkingStrategy", + "LocatorType", + "ParagraphSplitter", + "RecursiveSplitter", + "SentenceSplitter", + "chunk_document", + "chunk_id", + "ingest", + "prepend_context", + "prepend_source_context", +] diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py new file mode 100644 index 00000000..49e0dcf2 --- /dev/null +++ b/packages/moss-chunking/src/chunk.py @@ -0,0 +1,101 @@ +"""The chunk contract: stable IDs, position metadata, and Moss `DocumentInfo`. + +Splitters disagree about *how* to cut text, and that is fine — they are meant to +be swappable. What they must not disagree about is what a chunk looks like once +it comes out. Today they do: `moss-pikachu` emits `{path}#chunk-0001` with +path/filename/chunk/extension/modified_at, while `moss-llamaindex` emits +`{filename}-p{page}-c{idx}` with source/page. Both already wrap chunks in +`DocumentInfo`, so the envelope is shared — but they have no metadata key in +common, so nothing downstream can treat their chunks uniformly. + +This module pins down the inside of that envelope. + +Position is deliberately *not* a fixed field list. Plain text is located by +character offset, paginated documents by page, code by line; character offsets +are meaningless for a PDF and page numbers are meaningless for a source file. So +every chunk carries a locator whose unit is declared rather than assumed. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal, get_args + +from moss import DocumentInfo + +#: Unit a chunk's position is measured in. +LocatorType = Literal["char", "line", "page"] + +LOCATOR_TYPES: tuple[str, ...] = get_args(LocatorType) + +#: Metadata keys the contract owns. `Chunk.extra` may not shadow them. +RESERVED_KEYS = frozenset({"source", "chunk_index", "locator_type", "locator_start", "locator_end"}) + + +def chunk_id(source: str, index: int) -> str: + """Build a chunk's stable ID. + + Zero-padded so IDs sort lexicographically in the order they were cut, which + is what pikachu arrived at independently. `source` is whatever identifies the + original — a file path, a URL, a document name — and must be stable across + runs: re-chunking an unchanged document has to reproduce the same IDs, or + every chunk gets re-added instead of replaced. + """ + if not source: + raise ValueError("source must be a non-empty string") + if index < 0: + raise ValueError(f"index must be >= 0, got {index}") + return f"{source}#chunk-{index:04d}" + + +@dataclass(frozen=True) +class Chunk: + """One cut of a document, before it becomes a `DocumentInfo`. + + Splitters yield these. Positions stay real integers here; the conversion to + Moss's string-only metadata happens once, in `to_document`. + """ + + text: str + index: int + locator_type: LocatorType + locator_start: int + locator_end: int + extra: dict[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.index < 0: + raise ValueError(f"index must be >= 0, got {self.index}") + if self.locator_type not in LOCATOR_TYPES: + raise ValueError( + f"locator_type must be one of {LOCATOR_TYPES}, got {self.locator_type!r}" + ) + if self.locator_start < 0: + raise ValueError(f"locator_start must be >= 0, got {self.locator_start}") + if self.locator_end < self.locator_start: + raise ValueError( + f"locator_end ({self.locator_end}) must be >= locator_start ({self.locator_start})" + ) + clashes = RESERVED_KEYS & self.extra.keys() + if clashes: + raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") + + def to_document(self, source: str) -> DocumentInfo: + """Render this chunk as a Moss `DocumentInfo`. + + Every metadata value is stringified because Moss types metadata as + `Dict[str, str]`. An int left in there would fail at the SDK boundary, + which is a worse place to discover it than here. + """ + return DocumentInfo( + id=chunk_id(source, self.index), + text=self.text, + metadata={ + "source": source, + "chunk_index": str(self.index), + "locator_type": self.locator_type, + "locator_start": str(self.locator_start), + "locator_end": str(self.locator_end), + **self.extra, + }, + ) diff --git a/packages/moss-chunking/src/enrich.py b/packages/moss-chunking/src/enrich.py new file mode 100644 index 00000000..4266b3eb --- /dev/null +++ b/packages/moss-chunking/src/enrich.py @@ -0,0 +1,41 @@ +"""Optional post-steps that rewrite a chunk's text after it is cut. + +These are retrieval-quality tricks, not splitting tricks, so they live outside +`ChunkingStrategy` and compose with any of them. Keeping them separate is also +what makes them adoptable: pikachu already does the trick below by hand, and it +would keep its own copy if the only way to get it were to switch splitters. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +from moss import DocumentInfo + + +def prepend_context(doc: DocumentInfo, fields: Mapping[str, str]) -> DocumentInfo: + """Return `doc` with `fields` written into the top of its text. + + Moss's hybrid search scores BM25 over chunk text, so facts that live only in + metadata — the filename, the folder it sits in — are invisible to the keyword + half of the query. Restating them in the text makes them matchable. This is + pikachu's `enrich_chunk_body`, generalised: it prepends `Filename:` and + `Path:` so a search for a file by name finds it. + + The ID and metadata are untouched, so an enriched chunk is still addressable + exactly as the contract says it is. + """ + if not fields: + return doc + header = "\n".join(f"{key}: {value}" for key, value in fields.items()) + return DocumentInfo( + id=doc.id, + text=f"{header}\n\n{doc.text}", + metadata=doc.metadata, + embedding=getattr(doc, "embedding", None), + ) + + +def prepend_source_context(doc: DocumentInfo, filename: str, path: str) -> DocumentInfo: + """`prepend_context` with pikachu's exact field set, for file-backed chunks.""" + return prepend_context(doc, {"Filename": filename, "Path": path}) diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py new file mode 100644 index 00000000..da91cfe2 --- /dev/null +++ b/packages/moss-chunking/src/ingest.py @@ -0,0 +1,27 @@ +"""Copy chunks into a Moss index.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from moss import DocumentInfo, MossClient, MutationResult + + +async def ingest( + documents: Iterable[DocumentInfo], + project_id: str, + project_key: str, + index_name: str, + model_id: str | None = None, +) -> MutationResult | None: + """Copy every chunk into a fresh Moss index. + + Deliberately without the connector template's `auto_id` option: random UUIDs + would defeat the contract's stable IDs, and re-indexing an unchanged document + would append duplicates instead of replacing what is already there. + """ + docs = list(documents) + if not docs: + return None + client = MossClient(project_id, project_key) + return await client.create_index(index_name, docs, model_id=model_id) diff --git a/packages/moss-chunking/src/strategies.py b/packages/moss-chunking/src/strategies.py new file mode 100644 index 00000000..e8d3d118 --- /dev/null +++ b/packages/moss-chunking/src/strategies.py @@ -0,0 +1,291 @@ +"""Built-in splitters. One class per strategy, all yielding `Chunk`. + +Every splitter here reports `char` locators, because they all cut plain text and +character offsets are what plain text has. A splitter over paginated or line- +oriented content reports `page` or `line` instead — the contract does not care, +so long as the unit is declared. + +The shared invariant, which the tests enforce: for a chunk produced from `text`, +`text[chunk.locator_start:chunk.locator_end] == chunk.text`. Offsets point into +the original string, never into a normalized or stripped copy of it. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import replace +from typing import Protocol + +from moss import DocumentInfo + +from .chunk import Chunk + +# A sentence ends at .!? — optionally followed by a closing quote or bracket — +# and is separated from the next by whitespace. Each lookbehind branch is a fixed +# width, which is what Python's `re` requires. +_SENTENCE_BOUNDARY = re.compile(r"(?:(?<=[.!?][\"'”’)\]])|(?<=[.!?]))\s+") + +# A blank line, plus any whitespace padding around it. +_PARAGRAPH_BOUNDARY = re.compile(r"\n[ \t]*\n\s*") + +Span = tuple[int, int] + + +class ChunkingStrategy(Protocol): + """Cut `text` into ordered `Chunk`s. + + Implementations own only the cutting. IDs and metadata belong to the + contract (see `chunk.py`), so a strategy never builds a `DocumentInfo` + itself — that is precisely the freedom that let pikachu and llamaindex drift + apart. + """ + + def split(self, text: str) -> Iterable[Chunk]: ... + + +def _trim(text: str, spans: Iterable[Span]) -> list[Span]: + """Shrink each span past its surrounding whitespace, dropping blank ones.""" + trimmed: list[Span] = [] + for start, end in spans: + raw = text[start:end] + stripped = raw.strip() + if not stripped: + continue + lead = len(raw) - len(raw.lstrip()) + trimmed.append((start + lead, start + lead + len(stripped))) + return trimmed + + +def _split_spans(text: str, boundary: re.Pattern[str]) -> list[Span]: + """Spans of `text` between matches of `boundary`, whitespace trimmed.""" + spans: list[Span] = [] + pos = 0 + for match in boundary.finditer(text): + spans.append((pos, match.start())) + pos = match.end() + spans.append((pos, len(text))) + return _trim(text, spans) + + +def _hard_spans(text: str, max_chars: int) -> list[Span]: + """Last resort: slice at `max_chars` when no separator gets a piece small.""" + windows = [ + (offset, min(offset + max_chars, len(text))) for offset in range(0, len(text), max_chars) + ] + return _trim(text, windows) + + +class CharSplitter: + """Fixed-width character windows with a trailing overlap. + + The pikachu strategy: its 1800/300 settings are the defaults here. + """ + + def __init__(self, chunk_chars: int = 1800, overlap: int = 300) -> None: + if chunk_chars < 1: + raise ValueError(f"chunk_chars must be >= 1, got {chunk_chars}") + if overlap < 0: + raise ValueError(f"overlap must be >= 0, got {overlap}") + if overlap >= chunk_chars: + raise ValueError( + f"overlap ({overlap}) must be < chunk_chars ({chunk_chars}), " + "otherwise chunking never advances" + ) + self.chunk_chars = chunk_chars + self.overlap = overlap + + def split(self, text: str) -> Iterator[Chunk]: + length = len(text) + start = 0 + index = 0 + while start < length: + end = min(start + self.chunk_chars, length) + for piece_start, piece_end in _trim(text, [(start, end)]): + yield Chunk( + text=text[piece_start:piece_end], + index=index, + locator_type="char", + locator_start=piece_start, + locator_end=piece_end, + ) + index += 1 + if end >= length: + break + start = max(end - self.overlap, start + 1) + + +class SentenceSplitter: + """Whole sentences accumulated up to a word budget, overlapping by sentence. + + The llamaindex strategy: its 400-word / 2-sentence settings are the defaults. + Sentence detection is regex-based rather than nltk-backed, so the package + stays dependency-free apart from `moss` itself. + """ + + def __init__(self, max_words: int = 400, overlap_sentences: int = 2) -> None: + if max_words < 1: + raise ValueError(f"max_words must be >= 1, got {max_words}") + if overlap_sentences < 0: + raise ValueError(f"overlap_sentences must be >= 0, got {overlap_sentences}") + self.max_words = max_words + # Clamp, as llamaindex does, so a large overlap cannot stall progress. + self.overlap_sentences = min(overlap_sentences, max(1, max_words // 100)) + + def split(self, text: str) -> Iterator[Chunk]: + sentences = _split_spans(text, _SENTENCE_BOUNDARY) + if not sentences: + return + + cursor = 0 + index = 0 + while cursor < len(sentences): + group_start = cursor + words = 0 + while cursor < len(sentences): + start, end = sentences[cursor] + sentence_words = len(text[start:end].split()) + # The first sentence always goes in, even if it blows the budget + # on its own — otherwise nothing would ever be emitted for it. + if cursor > group_start and words + sentence_words > self.max_words: + break + words += sentence_words + cursor += 1 + + chunk_start = sentences[group_start][0] + chunk_end = sentences[cursor - 1][1] + yield Chunk( + text=text[chunk_start:chunk_end], + index=index, + locator_type="char", + locator_start=chunk_start, + locator_end=chunk_end, + ) + index += 1 + + if cursor < len(sentences): + cursor = max(cursor - self.overlap_sentences, group_start + 1) + + +class ParagraphSplitter: + """Whole paragraphs packed up to a character budget. + + Paragraphs are never broken apart: one longer than `max_chars` is emitted on + its own rather than cut, on the grounds that an author's paragraph break is + better evidence of a topic boundary than an arbitrary offset. Use + `RecursiveSplitter` when a hard ceiling matters more than that. + """ + + def __init__(self, max_chars: int = 1800) -> None: + if max_chars < 1: + raise ValueError(f"max_chars must be >= 1, got {max_chars}") + self.max_chars = max_chars + + def split(self, text: str) -> Iterator[Chunk]: + paragraphs = _split_spans(text, _PARAGRAPH_BOUNDARY) + index = 0 + group: list[Span] = [] + for span in paragraphs: + if group and span[1] - group[0][0] > self.max_chars: + yield self._emit(text, group, index) + index += 1 + group = [] + group.append(span) + if group: + yield self._emit(text, group, index) + + @staticmethod + def _emit(text: str, group: Sequence[Span], index: int) -> Chunk: + start, end = group[0][0], group[-1][1] + return Chunk( + text=text[start:end], + index=index, + locator_type="char", + locator_start=start, + locator_end=end, + ) + + +class RecursiveSplitter: + """Split on the coarsest separator that fits, then merge back up. + + Tries each separator in turn — paragraphs, then lines, then sentences, then + words — recursing into any piece still over budget, and finally coalescing + adjacent pieces so chunks land near `max_chars` rather than far under it. + Unlike `ParagraphSplitter` this always respects the ceiling, falling back to + a hard character cut if no separator gets a piece small enough. + """ + + DEFAULT_SEPARATORS: tuple[str, ...] = ("\n\n", "\n", ". ", " ") + + def __init__( + self, + max_chars: int = 1800, + separators: Sequence[str] | None = None, + ) -> None: + if max_chars < 1: + raise ValueError(f"max_chars must be >= 1, got {max_chars}") + self.max_chars = max_chars + self.separators = tuple(separators) if separators is not None else self.DEFAULT_SEPARATORS + + def split(self, text: str) -> Iterator[Chunk]: + spans = self._recurse(text, 0, self.separators) + for index, (start, end) in enumerate(self._merge(spans)): + yield Chunk( + text=text[start:end], + index=index, + locator_type="char", + locator_start=start, + locator_end=end, + ) + + def _recurse(self, text: str, base: int, separators: Sequence[str]) -> list[Span]: + if len(text) <= self.max_chars: + return [(base + start, base + end) for start, end in _trim(text, [(0, len(text))])] + if not separators: + hard = _hard_spans(text, self.max_chars) + return [(base + start, base + end) for start, end in hard] + + head, tail = separators[0], separators[1:] + pieces = _split_spans(text, re.compile(re.escape(head))) + if len(pieces) <= 1: + # This separator bought us nothing; try the next one. + return self._recurse(text, base, tail) + + spans: list[Span] = [] + for start, end in pieces: + piece = text[start:end] + if len(piece) <= self.max_chars: + spans.append((base + start, base + end)) + else: + spans.extend(self._recurse(piece, base + start, tail)) + return spans + + def _merge(self, spans: Sequence[Span]) -> list[Span]: + """Coalesce neighbours while the combined span stays within budget.""" + merged: list[Span] = [] + for span in spans: + if merged and span[1] - merged[-1][0] <= self.max_chars: + merged[-1] = (merged[-1][0], span[1]) + else: + merged.append(span) + return merged + + +def chunk_document( + text: str, + source: str, + strategy: ChunkingStrategy, + extra: Mapping[str, str] | None = None, +) -> list[DocumentInfo]: + """Run `strategy` over `text` and render the chunks under the contract. + + `extra` is merged into every chunk's metadata — the place for source-level + facts like `extension` or `modified_at` that the splitter cannot know. + """ + documents: list[DocumentInfo] = [] + for chunk in strategy.split(text): + if extra: + chunk = replace(chunk, extra={**chunk.extra, **extra}) + documents.append(chunk.to_document(source)) + return documents diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py new file mode 100644 index 00000000..2daf5d20 --- /dev/null +++ b/packages/moss-chunking/tests/test_chunk.py @@ -0,0 +1,65 @@ +"""Contract tests: IDs, validation, and the `DocumentInfo` rendering.""" + +from __future__ import annotations + +import pytest +from moss_chunking import Chunk, chunk_id + + +def test_chunk_id_is_zero_padded_so_it_sorts_in_cut_order(): + ids = [chunk_id("notes.md", i) for i in (0, 1, 9, 10, 100)] + assert ids[0] == "notes.md#chunk-0000" + assert ids[3] == "notes.md#chunk-0010" + assert sorted(ids) == ids + + +def test_chunk_id_rejects_empty_source(): + with pytest.raises(ValueError, match="non-empty"): + chunk_id("", 0) + + +def test_chunk_id_rejects_negative_index(): + with pytest.raises(ValueError, match="index"): + chunk_id("notes.md", -1) + + +def test_to_document_stringifies_every_metadata_value(): + doc = Chunk("body", 3, "char", 10, 20).to_document("notes.md") + assert doc.id == "notes.md#chunk-0003" + assert doc.text == "body" + assert doc.metadata == { + "source": "notes.md", + "chunk_index": "3", + "locator_type": "char", + "locator_start": "10", + "locator_end": "20", + } + assert all(isinstance(value, str) for value in doc.metadata.values()) + + +def test_extra_metadata_is_merged_and_stringified(): + chunk = Chunk("body", 0, "page", 4, 4, extra={"page_label": "iv", "words": "12"}) + doc = chunk.to_document("paper.pdf") + assert doc.metadata["page_label"] == "iv" + assert doc.metadata["locator_type"] == "page" + + +def test_extra_may_not_shadow_reserved_keys(): + with pytest.raises(ValueError, match="reserved"): + Chunk("body", 0, "char", 0, 4, extra={"source": "elsewhere.md"}) + + +def test_unknown_locator_type_is_rejected(): + with pytest.raises(ValueError, match="locator_type"): + Chunk("body", 0, "byte", 0, 4) # type: ignore[arg-type] + + +def test_backwards_locator_is_rejected(): + with pytest.raises(ValueError, match="locator_end"): + Chunk("body", 0, "char", 20, 10) + + +def test_locator_may_be_a_single_point(): + """A page-located chunk starts and ends on the same page.""" + chunk = Chunk("body", 0, "page", 4, 4) + assert chunk.to_document("paper.pdf").metadata["locator_start"] == "4" diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py new file mode 100644 index 00000000..5142c0f8 --- /dev/null +++ b/packages/moss-chunking/tests/test_strategies.py @@ -0,0 +1,203 @@ +"""Splitter tests. + +The invariant every strategy must hold is the roundtrip: a chunk's offsets point +into the original text and slice back to exactly its own text. Get that wrong and +the position metadata is decorative — you can address a chunk but not find it. +""" + +from __future__ import annotations + +import pytest +from moss_chunking import ( + CharSplitter, + ParagraphSplitter, + RecursiveSplitter, + SentenceSplitter, + chunk_document, + prepend_source_context, +) + +PROSE = ( + "Moss is a semantic search runtime. It runs on device. " + "Queries return in under ten milliseconds.\n\n" + "Chunking decides what a query can match. Cut too coarsely and a hit drags " + "in noise. Cut too finely and the surrounding context is lost.\n\n" + "So the strategy has to suit the content." +) + +ALL_STRATEGIES = [ + CharSplitter(chunk_chars=120, overlap=20), + SentenceSplitter(max_words=12, overlap_sentences=1), + ParagraphSplitter(max_chars=120), + RecursiveSplitter(max_chars=120), +] + + +def roundtrips(text: str, chunks) -> bool: + return all(text[c.locator_start : c.locator_end] == c.text for c in chunks) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_offsets_slice_back_to_the_chunk_text(strategy): + chunks = list(strategy.split(PROSE)) + assert chunks + assert roundtrips(PROSE, chunks) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_indices_are_sequential_from_zero(strategy): + chunks = list(strategy.split(PROSE)) + assert [c.index for c in chunks] == list(range(len(chunks))) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_empty_and_blank_input_yields_nothing(strategy): + assert list(strategy.split("")) == [] + assert list(strategy.split(" \n\n\t ")) == [] + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_chunks_are_never_blank(strategy): + assert all(c.text.strip() for c in strategy.split(PROSE)) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_chunks_advance_through_the_document(strategy): + starts = [c.locator_start for c in strategy.split(PROSE)] + assert starts == sorted(starts) + + +# --- CharSplitter ------------------------------------------------------------ + + +def test_char_splitter_respects_its_ceiling(): + chunks = list(CharSplitter(chunk_chars=50, overlap=10).split(PROSE)) + assert all(len(c.text) <= 50 for c in chunks) + + +def test_char_splitter_overlaps_by_the_requested_amount(): + text = "a" * 250 + chunks = list(CharSplitter(chunk_chars=100, overlap=30).split(text)) + assert chunks[1].locator_start == chunks[0].locator_end - 30 + + +def test_char_splitter_covers_the_whole_document(): + chunks = list(CharSplitter(chunk_chars=40, overlap=5).split(PROSE)) + assert chunks[0].locator_start == 0 + assert chunks[-1].locator_end == len(PROSE) + + +def test_overlap_at_or_above_chunk_size_is_rejected(): + with pytest.raises(ValueError, match="never advances"): + CharSplitter(chunk_chars=100, overlap=100) + + +def test_short_input_is_a_single_chunk(): + chunks = list(CharSplitter().split("one short line")) + assert len(chunks) == 1 + assert chunks[0].text == "one short line" + + +# --- SentenceSplitter -------------------------------------------------------- + + +def test_sentence_splitter_does_not_break_mid_sentence(): + chunks = list(SentenceSplitter(max_words=12, overlap_sentences=1).split(PROSE)) + assert all(c.text.rstrip().endswith((".", "!", "?")) for c in chunks) + + +def test_sentence_longer_than_the_budget_is_still_emitted(): + text = "This one sentence is deliberately longer than the tiny word budget allows." + chunks = list(SentenceSplitter(max_words=3).split(text)) + assert len(chunks) == 1 + assert chunks[0].text == text + + +def test_sentence_splitter_handles_closing_quotes(): + text = 'He said "it works." Then he left. She agreed.' + chunks = list(SentenceSplitter(max_words=4, overlap_sentences=0).split(text)) + assert roundtrips(text, chunks) + assert chunks[0].text.startswith("He said") + + +def test_overlap_is_clamped_so_progress_is_guaranteed(): + splitter = SentenceSplitter(max_words=50, overlap_sentences=99) + assert splitter.overlap_sentences == 1 + assert list(splitter.split(PROSE)) + + +# --- ParagraphSplitter ------------------------------------------------------- + + +def test_oversized_paragraph_is_emitted_rather_than_cut(): + text = "x" * 500 + chunks = list(ParagraphSplitter(max_chars=100).split(text)) + assert len(chunks) == 1 + assert len(chunks[0].text) == 500 + + +def test_paragraphs_are_packed_together_when_they_fit(): + text = "one.\n\ntwo.\n\nthree." + assert len(list(ParagraphSplitter(max_chars=1000).split(text))) == 1 + + +def test_paragraphs_are_split_apart_when_they_do_not_fit(): + text = "one.\n\ntwo.\n\nthree." + assert len(list(ParagraphSplitter(max_chars=6).split(text))) == 3 + + +# --- RecursiveSplitter ------------------------------------------------------- + + +def test_recursive_splitter_always_respects_the_ceiling(): + chunks = list(RecursiveSplitter(max_chars=80).split(PROSE)) + assert all(len(c.text) <= 80 for c in chunks) + + +def test_recursive_splitter_hard_cuts_unseparated_text(): + text = "x" * 300 + chunks = list(RecursiveSplitter(max_chars=100).split(text)) + assert all(len(c.text) <= 100 for c in chunks) + assert "".join(c.text for c in chunks) == text + + +def test_recursive_splitter_prefers_paragraph_boundaries(): + text = "alpha beta.\n\ngamma delta." + chunks = list(RecursiveSplitter(max_chars=12).split(text)) + assert [c.text for c in chunks] == ["alpha beta.", "gamma delta."] + + +# --- chunk_document + enrich ------------------------------------------------- + + +def test_chunk_document_renders_documents_under_the_contract(): + docs = chunk_document(PROSE, "notes.md", CharSplitter(chunk_chars=100, overlap=10)) + assert docs + assert docs[0].id == "notes.md#chunk-0000" + assert docs[0].metadata["source"] == "notes.md" + assert docs[0].metadata["locator_type"] == "char" + + +def test_chunk_document_merges_source_level_extra_metadata(): + docs = chunk_document( + PROSE, + "notes.md", + CharSplitter(chunk_chars=100, overlap=10), + extra={"extension": "md", "modified_at": "2026-07-28T00:00:00Z"}, + ) + assert all(d.metadata["extension"] == "md" for d in docs) + assert all(d.metadata["source"] == "notes.md" for d in docs) + + +def test_chunk_document_rejects_extra_that_shadows_the_contract(): + with pytest.raises(ValueError, match="reserved"): + chunk_document(PROSE, "notes.md", CharSplitter(), extra={"chunk_index": "9"}) + + +def test_prepend_context_changes_text_but_not_addressing(): + doc = chunk_document(PROSE, "notes.md", CharSplitter())[0] + enriched = prepend_source_context(doc, filename="notes.md", path="/docs/notes.md") + assert enriched.id == doc.id + assert enriched.metadata == doc.metadata + assert enriched.text.startswith("Filename: notes.md\nPath: /docs/notes.md\n\n") + assert enriched.text.endswith(doc.text) diff --git a/packages/moss-chunking/uv.lock b/packages/moss-chunking/uv.lock new file mode 100644 index 00000000..3e5da749 --- /dev/null +++ b/packages/moss-chunking/uv.lock @@ -0,0 +1,318 @@ +version = 1 +revision = 3 +requires-python = ">=3.10, <3.15" + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "backports-asyncio-runner" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "inferedge-moss-core" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/02/b9f6bc352bd8e26c9006c579948444795177df86f55bceeebc301342ccc6/inferedge_moss_core-0.21.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:4ec09b432a08cd19a5c2c68c40ccc7ff06da0e5429a0461aa1b34a7973f4e313", size = 3958992, upload-time = "2026-07-21T05:06:05.281Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a6/21f74c3f239cc93a8a17e53aef8cb168b93a1ae3b48d7c13fdfec1b24e58/inferedge_moss_core-0.21.0-cp310-abi3-manylinux_2_35_x86_64.whl", hash = "sha256:ddbabce5503be4840db6c2a54f4feeb8e426baf5ed06b1182847e7115f717be4", size = 4915489, upload-time = "2026-07-21T06:27:28.13Z" }, + { url = "https://files.pythonhosted.org/packages/53/d7/9ac7b869c767928dcbcf49089ee8e9dbf18a83a6827ae08d0b94bf1b9d8f/inferedge_moss_core-0.21.0-cp310-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:bbf51735ad8a13e94fc38f1a3a147294c92b4e198fd711b02cd0b026eced7ac3", size = 4813300, upload-time = "2026-07-21T05:06:11.614Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e8/11262d2284491b858f0a84dae7f007734813e5bd91aa9da72234fd95e029/inferedge_moss_core-0.21.0-cp310-abi3-win_amd64.whl", hash = "sha256:e5d142b5c6069b8e5fb30f4ce6a1df40d7f756242de29e74c4b6f5a770b4795e", size = 5350984, upload-time = "2026-07-21T05:10:26.373Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "moss" +version = "1.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "inferedge-moss-core" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/66/8595da68c9d61128c4faa16e082b53231550c00a40a0cfabb12460b204cd/moss-1.7.2.tar.gz", hash = "sha256:adadef3c8006ac773b1665764a6e7ed0390ed5d9d28e3389890a52202e8ceb98", size = 55466, upload-time = "2026-07-21T06:34:43.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/d5/4a7ad22817bf20fa4eb29d7ae2349ad037e516e99cfa62cd19662ed52e37/moss-1.7.2-py3-none-any.whl", hash = "sha256:1f3bcf7fff29b0dcb5d78c14b9e547fed527dd3eabe5c2f5db598f60515784ad", size = 22914, upload-time = "2026-07-21T06:34:42.689Z" }, +] + +[[package]] +name = "moss-chunking" +version = "0.0.1" +source = { editable = "." } +dependencies = [ + { name = "moss" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "python-dotenv" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "moss", specifier = ">=1.1.1" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, + { name = "python-dotenv", marker = "extra == 'dev'", specifier = ">=1.0.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, + { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, + { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, + { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, + { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From 8851290c10723594acc2054b1dfdaf2e6a991862 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Tue, 4 Aug 2026 20:39:20 +0530 Subject: [PATCH 02/14] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20pre?= =?UTF-8?q?serve=20separators,=20harden=20extra,=20fix=20enrich=20reconstr?= =?UTF-8?q?uction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RecursiveSplitter dropped the non-whitespace part of any separator it cut on. Splitting "abc. def." at max_chars=4 returned "abc" and "def.", losing the first period: _split_spans discards whatever the boundary matches, which is right for "\n\n" or " " but wrong for ". ", where the period ends the sentence and only the space separates. A separator's non-whitespace head now goes into a lookbehind so the cut lands after the punctuation instead of through it. The roundtrip invariant did not catch this. Offsets stayed self-consistent — they just stopped covering the text between them. Added a test asserting the gap between adjacent chunks is only ever whitespace, which is the property that was actually missing. frozen=True freezes Chunk.extra the field, not the dict behind it, so a reserved key could be written in after __post_init__ had already validated it, and to_document would render it over the contract's own metadata. extra is now copied on construction, and merged first at render so the five reserved keys win regardless. prepend_context rewrote doc.text while carrying the old embedding forward, pairing a vector with content it no longer describes and quietly skewing the dense half of hybrid queries. It now drops the embedding, forcing a recompute downstream — the alternative is a rule that enrichment must precede embedding, which nothing can enforce. The same reconstruction was also dropping payload, which unlike the embedding has nothing to do with the text — silent data loss rather than a decision. It is now carried through. 57 tests. --- packages/moss-chunking/README.md | 5 +- packages/moss-chunking/src/chunk.py | 12 +++- packages/moss-chunking/src/enrich.py | 14 ++++- packages/moss-chunking/src/strategies.py | 18 +++++- packages/moss-chunking/tests/test_chunk.py | 25 ++++++++ .../moss-chunking/tests/test_strategies.py | 57 +++++++++++++++++++ 6 files changed, 127 insertions(+), 4 deletions(-) diff --git a/packages/moss-chunking/README.md b/packages/moss-chunking/README.md index 0a211bee..9096cd58 100644 --- a/packages/moss-chunking/README.md +++ b/packages/moss-chunking/README.md @@ -125,7 +125,10 @@ from moss_chunking import prepend_source_context doc = prepend_source_context(doc, filename="notes.md", path="/docs/notes.md") ``` -ID and metadata are untouched, so an enriched chunk stays addressable. +ID and metadata are untouched, so an enriched chunk stays addressable. Any +embedding is dropped: it was computed from the text this rewrites, and a vector +that no longer describes its chunk skews the dense half of every hybrid query +without ever announcing itself. Enrich first, embed after. ## Tests diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 49e0dcf2..1d15d690 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -79,6 +79,10 @@ def __post_init__(self) -> None: clashes = RESERVED_KEYS & self.extra.keys() if clashes: raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") + # `frozen=True` freezes the field, not the dict behind it. Without a copy + # the caller keeps a live handle on validated state and can add a + # reserved key after the check has already passed. + object.__setattr__(self, "extra", dict(self.extra)) def to_document(self, source: str) -> DocumentInfo: """Render this chunk as a Moss `DocumentInfo`. @@ -86,16 +90,22 @@ def to_document(self, source: str) -> DocumentInfo: Every metadata value is stringified because Moss types metadata as `Dict[str, str]`. An int left in there would fail at the SDK boundary, which is a worse place to discover it than here. + + `extra` is merged *first* so the contract's own keys always win. The + constructor already rejects a clashing `extra`, so this only matters if + one was introduced afterwards — but the whole point of the contract is + that these five keys mean the same thing on every chunk, and a render + step is the last place that can still guarantee it. """ return DocumentInfo( id=chunk_id(source, self.index), text=self.text, metadata={ + **self.extra, "source": source, "chunk_index": str(self.index), "locator_type": self.locator_type, "locator_start": str(self.locator_start), "locator_end": str(self.locator_end), - **self.extra, }, ) diff --git a/packages/moss-chunking/src/enrich.py b/packages/moss-chunking/src/enrich.py index 4266b3eb..e92a1bae 100644 --- a/packages/moss-chunking/src/enrich.py +++ b/packages/moss-chunking/src/enrich.py @@ -24,6 +24,17 @@ def prepend_context(doc: DocumentInfo, fields: Mapping[str, str]) -> DocumentInf The ID and metadata are untouched, so an enriched chunk is still addressable exactly as the contract says it is. + + Any embedding is dropped. It was computed from the text this function just + rewrote, so carrying it over would pair a vector with content it does not + describe — and the mismatch is invisible, quietly skewing the dense half of + every hybrid query. Dropping it forces a recompute downstream, which is the + safe direction to fail; the alternative is a rule that enrichment must + happen before embedding, which nothing can enforce. + + `payload` is carried through. Unlike the embedding it has nothing to do with + the text, so rebuilding the document without it would be silent data loss + rather than a decision. """ if not fields: return doc @@ -32,7 +43,8 @@ def prepend_context(doc: DocumentInfo, fields: Mapping[str, str]) -> DocumentInf id=doc.id, text=f"{header}\n\n{doc.text}", metadata=doc.metadata, - embedding=getattr(doc, "embedding", None), + embedding=None, + payload=getattr(doc, "payload", None), ) diff --git a/packages/moss-chunking/src/strategies.py b/packages/moss-chunking/src/strategies.py index e8d3d118..040cef78 100644 --- a/packages/moss-chunking/src/strategies.py +++ b/packages/moss-chunking/src/strategies.py @@ -57,6 +57,22 @@ def _trim(text: str, spans: Iterable[Span]) -> list[Span]: return trimmed +def _separator_pattern(separator: str) -> re.Pattern[str]: + """Boundary regex for `separator` that keeps its non-whitespace part. + + `_split_spans` discards whatever the boundary matches. That is right for + `"\\n\\n"` or `" "` — whitespace between chunks is not content. It is wrong + for `". "`, where the period belongs to the sentence it ends and only the + space separates. So a separator's non-whitespace head goes into a lookbehind + (fixed width, since it is a literal) and only its trailing whitespace is + consumed, putting the cut after the punctuation rather than through it. + """ + kept = separator.rstrip() + if not kept: + return re.compile(re.escape(separator)) + return re.compile(f"(?<={re.escape(kept)}){re.escape(separator[len(kept) :])}") + + def _split_spans(text: str, boundary: re.Pattern[str]) -> list[Span]: """Spans of `text` between matches of `boundary`, whitespace trimmed.""" spans: list[Span] = [] @@ -247,7 +263,7 @@ def _recurse(self, text: str, base: int, separators: Sequence[str]) -> list[Span return [(base + start, base + end) for start, end in hard] head, tail = separators[0], separators[1:] - pieces = _split_spans(text, re.compile(re.escape(head))) + pieces = _split_spans(text, _separator_pattern(head)) if len(pieces) <= 1: # This separator bought us nothing; try the next one. return self._recurse(text, base, tail) diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index 2daf5d20..aa9e0254 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -49,6 +49,31 @@ def test_extra_may_not_shadow_reserved_keys(): Chunk("body", 0, "char", 0, 4, extra={"source": "elsewhere.md"}) +def test_reserved_keys_win_even_if_extra_is_mutated_after_construction(): + """The check at construction is not the last line of defence. + + `frozen=True` freezes the field, not the dict behind it, so a reserved key + can still be written into `extra` after validation has passed. Rendering + merges `extra` first, so the contract's own keys overwrite it either way. + """ + chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) + chunk.extra["source"] = "elsewhere.md" + chunk.extra["chunk_index"] = "99" + + doc = chunk.to_document("notes.md") + assert doc.metadata["source"] == "notes.md" + assert doc.metadata["chunk_index"] == "0" + assert doc.metadata["extension"] == "md" + + +def test_extra_is_copied_so_the_caller_cannot_mutate_validated_state(): + supplied = {"extension": "md"} + chunk = Chunk("body", 0, "char", 0, 4, extra=supplied) + supplied["source"] = "elsewhere.md" + assert "source" not in chunk.extra + assert chunk.to_document("notes.md").metadata["source"] == "notes.md" + + def test_unknown_locator_type_is_rejected(): with pytest.raises(ValueError, match="locator_type"): Chunk("body", 0, "byte", 0, 4) # type: ignore[arg-type] diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py index 5142c0f8..6b9ab8b0 100644 --- a/packages/moss-chunking/tests/test_strategies.py +++ b/packages/moss-chunking/tests/test_strategies.py @@ -8,12 +8,14 @@ from __future__ import annotations import pytest +from moss import DocumentInfo from moss_chunking import ( CharSplitter, ParagraphSplitter, RecursiveSplitter, SentenceSplitter, chunk_document, + prepend_context, prepend_source_context, ) @@ -161,6 +163,41 @@ def test_recursive_splitter_hard_cuts_unseparated_text(): assert "".join(c.text for c in chunks) == text +def test_recursive_splitter_keeps_the_punctuation_it_splits_on(): + """A separator's non-whitespace part is content, not a delimiter. + + Splitting on `". "` used to consume the period along with the space, so + "abc. def." came back as "abc" + "def." — the chunk text no longer said what + the source said. The roundtrip invariant did not catch it: offsets stayed + self-consistent, they just stopped covering the text between them. + """ + text = "abc. def." + chunks = list(RecursiveSplitter(max_chars=4).split(text)) + assert [c.text for c in chunks] == ["abc.", "def."] + assert "".join(c.text for c in chunks) == text.replace(" ", "") + + +def test_no_strategy_loses_non_whitespace_characters(): + """The gap between two adjacent chunks may only ever be whitespace.""" + text = "one. two. three. four. five. six." + for splitter in (RecursiveSplitter(max_chars=10), RecursiveSplitter(max_chars=6)): + chunks = list(splitter.split(text)) + for earlier, later in zip(chunks, chunks[1:], strict=False): + assert not text[earlier.locator_end : later.locator_start].strip() + + +def test_whitespace_separators_are_still_consumed(): + """The fix must not start emitting the blank line between paragraphs.""" + chunks = list(RecursiveSplitter(max_chars=10).split("alpha\n\nbeta")) + assert all(not c.text.startswith(("\n", " ")) for c in chunks) + assert all(not c.text.endswith(("\n", " ")) for c in chunks) + + +def test_custom_separator_without_trailing_whitespace_is_preserved(): + chunks = list(RecursiveSplitter(max_chars=3, separators=(";",)).split("ab;cd;ef")) + assert "".join(c.text for c in chunks) == "ab;cd;ef" + + def test_recursive_splitter_prefers_paragraph_boundaries(): text = "alpha beta.\n\ngamma delta." chunks = list(RecursiveSplitter(max_chars=12).split(text)) @@ -201,3 +238,23 @@ def test_prepend_context_changes_text_but_not_addressing(): assert enriched.metadata == doc.metadata assert enriched.text.startswith("Filename: notes.md\nPath: /docs/notes.md\n\n") assert enriched.text.endswith(doc.text) + + +def test_enrichment_drops_an_embedding_computed_from_the_old_text(): + doc = chunk_document(PROSE, "notes.md", CharSplitter())[0] + stale = DocumentInfo(id=doc.id, text=doc.text, metadata=doc.metadata, embedding=[0.1, 0.2]) + enriched = prepend_source_context(stale, filename="notes.md", path="/docs/notes.md") + assert enriched.text != stale.text + assert getattr(enriched, "embedding", None) is None + + +def test_enrichment_carries_the_payload_through(): + """Unlike the embedding, `payload` has nothing to do with the text.""" + doc = DocumentInfo(id="a#chunk-0000", text="body", metadata={"source": "a"}, payload='{"p":1}') + enriched = prepend_source_context(doc, filename="a.md", path="/a.md") + assert enriched.payload == '{"p":1}' + + +def test_enrichment_with_no_fields_is_a_no_op(): + doc = chunk_document(PROSE, "notes.md", CharSplitter())[0] + assert prepend_context(doc, {}) is doc From d4d169cff8292ac3ee97371da090033b0650fb1b Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 09:07:32 +0530 Subject: [PATCH 03/14] =?UTF-8?q?fix:=20address=20bot=20review=20=E2=80=94?= =?UTF-8?q?=20metadata=20coercion,=20ID=20bound,=20CRLF,=20hashability,=20?= =?UTF-8?q?floor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-string extra values (3 reviewers). to_document promised all-string metadata but passed extra through unchanged, so extra={"page": 3} failed at the SDK boundary instead of rendering "3". Coercion is back and the field is typed dict[str, object], since {"page": 3} is the natural thing to write. Chunk index bound (3 reviewers). :04d stops being fixed width at 10000, so chunk-10000 sorted before chunk-9999 and the ID format's only promise broke silently. chunk_id now rejects an index above MAX_CHUNK_INDEX rather than emitting an unsortable ID; four digits stays for pikachu parity. Chunk was not hashable. frozen=True has the dataclass advertise hashability, but the dict field made hash(chunk) raise TypeError, so chunks could not go in a set or key a dict. extra is excluded from the hash and still compared for equality. CRLF paragraph boundaries. _PARAGRAPH_BOUNDARY matched only LF, so a Windows-authored file had no paragraph breaks at all and ParagraphSplitter emitted one chunk far past max_chars. Dependency floor. moss>=1.1.1 cannot resolve: every release before 1.7.2 pins an inferedge-moss-core that is no longer on PyPI. 1.7.2 is the earliest installable version, and the earliest whose DocumentInfo is verified to accept payload — which is what the reported TypeError was really about. Release gate. The matrix only proved the wheel imported; it now runs the suite against the installed wheel and asserts every __all__ name resolves, so a build that corrupts offsets or metadata cannot reach PyPI. Version parsing uses tomllib instead of a regex, the tag guard matches refs/tags exactly, and twine gets --skip-existing so a partial upload can be rerun. The SentenceSplitter overlap clamp is kept for llamaindex parity, but its comment claimed it was what stopped a large overlap stalling progress. It is not — the group_start + 1 floor in split() is, for any overlap. Comment corrected and a test pins the real guarantee. Also: 3.14 classifier to match requires-python and the test matrix, README imports for the custom-strategy example, and per-chunk roundtrip assertions so a failure names the offending chunk. 64 tests. --- .github/workflows/publish-moss-chunking.yml | 43 +++++++++++++------ packages/moss-chunking/README.md | 19 ++++++-- packages/moss-chunking/pyproject.toml | 7 ++- packages/moss-chunking/src/__init__.py | 10 ++++- packages/moss-chunking/src/chunk.py | 25 ++++++++++- packages/moss-chunking/src/strategies.py | 12 ++++-- packages/moss-chunking/tests/test_chunk.py | 41 +++++++++++++++++- .../moss-chunking/tests/test_strategies.py | 27 ++++++++++-- packages/moss-chunking/uv.lock | 2 +- 9 files changed, 155 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish-moss-chunking.yml b/.github/workflows/publish-moss-chunking.yml index 13b5f4ed..f3705f22 100644 --- a/.github/workflows/publish-moss-chunking.yml +++ b/.github/workflows/publish-moss-chunking.yml @@ -26,14 +26,13 @@ jobs: id: compute shell: python run: | - import os, pathlib, re, sys + import os, pathlib, sys, tomllib - text = pathlib.Path("packages/moss-chunking/pyproject.toml").read_text(encoding="utf-8") - match = re.search(r'(?m)^version\s*=\s*"([^"]+)"', text) - if not match: - print("Could not find version in pyproject", file=sys.stderr) + raw = pathlib.Path("packages/moss-chunking/pyproject.toml").read_bytes() + version = tomllib.loads(raw.decode("utf-8")).get("project", {}).get("version") + if not version: + print("Could not find project.version in pyproject", file=sys.stderr) sys.exit(1) - version = match.group(1) out = pathlib.Path(os.environ["GITHUB_OUTPUT"]) with out.open("a", encoding="utf-8") as fh: @@ -61,7 +60,7 @@ jobs: - name: Install build tooling run: | python -m pip install --upgrade pip - pip install build + pip install build pytest - name: Build distributions working-directory: packages/moss-chunking @@ -69,7 +68,7 @@ jobs: rm -rf dist python -m build - - name: Smoke test import + - name: Install the built wheel shell: python run: | import glob, subprocess, sys @@ -78,12 +77,26 @@ jobs: if not wheels: raise SystemExit("No wheel found in dist/") - wheel = wheels[0] - subprocess.check_call([sys.executable, "-m", "pip", "install", "--force-reinstall", wheel]) + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--force-reinstall", wheels[0]] + ) + - name: Smoke test the public surface + shell: python + run: | import importlib + module = importlib.import_module("moss_chunking") - print("import ok; sample attrs:", dir(module)[:5]) + missing = [name for name in module.__all__ if not hasattr(module, name)] + if missing: + raise SystemExit(f"__all__ names missing from the built wheel: {missing}") + print(f"import ok; {len(module.__all__)} public names resolve") + + # Run the suite against the installed wheel, not the source tree. A wheel + # that imports cleanly can still ship broken offsets or metadata, and that + # is precisely what these tests assert. + - name: Test the installed wheel + run: python -m pytest -q packages/moss-chunking/tests # Only runs once every build-test matrix leg has passed. publish: @@ -99,7 +112,9 @@ jobs: - name: Fail if this version was already released run: | git fetch --tags --force - if git rev-parse "moss-chunking-v${VERSION}" >/dev/null 2>&1; then + # Match the tag ref exactly: `git rev-parse` alone resolves anything + # that happens to parse as a revision, not just a tag by this name. + if git rev-parse --verify "refs/tags/moss-chunking-v${VERSION}" >/dev/null 2>&1; then echo "::error::moss-chunking v${VERSION} is already tagged/released; bump the version in pyproject.toml before releasing." exit 1 fi @@ -124,7 +139,9 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | - twine upload packages/moss-chunking/dist/* + # --skip-existing so a rerun after a partial upload can still reach the + # tag step instead of failing on files PyPI already has. + twine upload --skip-existing packages/moss-chunking/dist/* - name: Tag release run: | diff --git a/packages/moss-chunking/README.md b/packages/moss-chunking/README.md index 9096cd58..ff1c8493 100644 --- a/packages/moss-chunking/README.md +++ b/packages/moss-chunking/README.md @@ -62,6 +62,11 @@ stable across runs so re-chunking an unchanged document replaces its chunks rather than duplicating them. Values are all strings, because Moss types metadata as `Dict[str, str]`. +The sort only holds while the padding is fixed width, so `chunk_id` rejects an +index above `MAX_CHUNK_INDEX` (9999) rather than emitting `chunk-10000`, which +sorts before `chunk-9999`. A document that cuts into more than 10,000 chunks +wants a coarser strategy or a per-section `source`. + That stability is only worth something if you keep it on the way in, which is why `ingest` drops the one option the connector template it mirrors does offer: `auto_id`. Random UUIDs defeat the contract — re-indexing an unchanged document @@ -75,11 +80,14 @@ assuming one. That's the one real design call in the package. Pass source-level facts a splitter can't know via `extra`: ```python -chunk_document(text, "notes.md", CharSplitter(), extra={"extension": "md"}) +chunk_document(text, "notes.md", CharSplitter(), extra={"extension": "md", "page": 3}) ``` -`extra` cannot shadow the reserved keys above — that's an error, not a silent -overwrite. +Values are stringified on the way out, so passing an `int` is fine — Moss types +metadata as `Dict[str, str]`, and coercing here beats failing at the SDK +boundary. `extra` cannot shadow the reserved keys above: that's an error, not a +silent overwrite, and the reserved keys win at render even if one is added to +`extra` afterwards. ## Strategies @@ -96,6 +104,11 @@ than cut. `RecursiveSplitter` is the one to reach for when the ceiling must hold Write your own by implementing `split(text) -> Iterable[Chunk]`: ```python +from collections.abc import Iterator + +from moss_chunking import Chunk + + class MyStrategy: def split(self, text: str) -> Iterator[Chunk]: yield Chunk(text=..., index=..., locator_type="line", locator_start=..., locator_end=...) diff --git a/packages/moss-chunking/pyproject.toml b/packages/moss-chunking/pyproject.toml index 6481df38..8784b624 100644 --- a/packages/moss-chunking/pyproject.toml +++ b/packages/moss-chunking/pyproject.toml @@ -16,10 +16,15 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Text Processing :: Linguistic", ] +# Most packages here declare `moss>=1.1.1`, but that floor cannot resolve: every +# moss release before 1.7.2 pins an `inferedge-moss-core` that is no longer on +# PyPI. 1.7.2 is the earliest version that actually installs, and the earliest +# whose `DocumentInfo` is verified to accept `payload`. dependencies = [ - "moss>=1.1.1", + "moss>=1.7.2", ] [project.optional-dependencies] diff --git a/packages/moss-chunking/src/__init__.py b/packages/moss-chunking/src/__init__.py index c225dfff..e585af3e 100644 --- a/packages/moss-chunking/src/__init__.py +++ b/packages/moss-chunking/src/__init__.py @@ -11,7 +11,14 @@ await ingest(docs, project_id, project_key, "my-index") """ -from .chunk import LOCATOR_TYPES, RESERVED_KEYS, Chunk, LocatorType, chunk_id +from .chunk import ( + LOCATOR_TYPES, + MAX_CHUNK_INDEX, + RESERVED_KEYS, + Chunk, + LocatorType, + chunk_id, +) from .enrich import prepend_context, prepend_source_context from .ingest import ingest from .strategies import ( @@ -25,6 +32,7 @@ __all__ = [ "LOCATOR_TYPES", + "MAX_CHUNK_INDEX", "RESERVED_KEYS", "CharSplitter", "Chunk", diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 1d15d690..4eff3475 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -31,6 +31,11 @@ #: Metadata keys the contract owns. `Chunk.extra` may not shadow them. RESERVED_KEYS = frozenset({"source", "chunk_index", "locator_type", "locator_start", "locator_end"}) +#: Highest index the ID format can hold. Four digits is pikachu's width, kept for +#: parity; past it the padding stops being fixed-width and `chunk-10000` sorts +#: before `chunk-9999`, which is the one promise the format makes. +MAX_CHUNK_INDEX = 9999 + def chunk_id(source: str, index: int) -> str: """Build a chunk's stable ID. @@ -40,11 +45,20 @@ def chunk_id(source: str, index: int) -> str: original — a file path, a URL, a document name — and must be stable across runs: re-chunking an unchanged document has to reproduce the same IDs, or every chunk gets re-added instead of replaced. + + Rejects indices above `MAX_CHUNK_INDEX` rather than emitting an ID that + breaks the sort. A document that cuts into more than 10,000 chunks wants a + coarser strategy or a per-section `source`, not a silently unsortable ID. """ if not source: raise ValueError("source must be a non-empty string") if index < 0: raise ValueError(f"index must be >= 0, got {index}") + if index > MAX_CHUNK_INDEX: + raise ValueError( + f"index must be <= {MAX_CHUNK_INDEX}, got {index}: past that the " + "zero-padding is no longer fixed width and IDs stop sorting in cut order" + ) return f"{source}#chunk-{index:04d}" @@ -61,7 +75,14 @@ class Chunk: locator_type: LocatorType locator_start: int locator_end: int - extra: dict[str, str] = field(default_factory=dict) + #: Values are stringified at render, so the natural thing to pass — `{"page": + #: 3}`, `{"words": 12}` — is accepted rather than failing at the SDK boundary. + #: + #: Excluded from the hash: `frozen=True` has the dataclass advertise + #: hashability, and a dict field would make `hash(chunk)` raise instead. + #: Chunks still compare on `extra`; two that differ only there collide, which + #: is a legal hash, unlike a `Chunk` that cannot go in a set at all. + extra: dict[str, object] = field(default_factory=dict, hash=False) def __post_init__(self) -> None: if self.index < 0: @@ -101,7 +122,7 @@ def to_document(self, source: str) -> DocumentInfo: id=chunk_id(source, self.index), text=self.text, metadata={ - **self.extra, + **{key: str(value) for key, value in self.extra.items()}, "source": source, "chunk_index": str(self.index), "locator_type": self.locator_type, diff --git a/packages/moss-chunking/src/strategies.py b/packages/moss-chunking/src/strategies.py index 040cef78..efa0b2e2 100644 --- a/packages/moss-chunking/src/strategies.py +++ b/packages/moss-chunking/src/strategies.py @@ -26,8 +26,10 @@ # width, which is what Python's `re` requires. _SENTENCE_BOUNDARY = re.compile(r"(?:(?<=[.!?][\"'”’)\]])|(?<=[.!?]))\s+") -# A blank line, plus any whitespace padding around it. -_PARAGRAPH_BOUNDARY = re.compile(r"\n[ \t]*\n\s*") +# A blank line, plus any whitespace padding around it. CRLF is matched too, or +# every paragraph in a Windows-authored file would run into the next one and +# `ParagraphSplitter` would silently emit a single oversized chunk. +_PARAGRAPH_BOUNDARY = re.compile(r"\r?\n[ \t]*\r?\n\s*") Span = tuple[int, int] @@ -145,7 +147,9 @@ def __init__(self, max_words: int = 400, overlap_sentences: int = 2) -> None: if overlap_sentences < 0: raise ValueError(f"overlap_sentences must be >= 0, got {overlap_sentences}") self.max_words = max_words - # Clamp, as llamaindex does, so a large overlap cannot stall progress. + # Clamped as llamaindex clamps it, to keep the parity this splitter is + # for. Not a safety net: progress is guaranteed by the `group_start + 1` + # floor in `split`, which holds for any overlap. self.overlap_sentences = min(overlap_sentences, max(1, max_words // 100)) def split(self, text: str) -> Iterator[Chunk]: @@ -292,7 +296,7 @@ def chunk_document( text: str, source: str, strategy: ChunkingStrategy, - extra: Mapping[str, str] | None = None, + extra: Mapping[str, object] | None = None, ) -> list[DocumentInfo]: """Run `strategy` over `text` and render the chunks under the contract. diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index aa9e0254..02ec458f 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest -from moss_chunking import Chunk, chunk_id +from moss_chunking import MAX_CHUNK_INDEX, Chunk, chunk_id def test_chunk_id_is_zero_padded_so_it_sorts_in_cut_order(): @@ -18,6 +18,17 @@ def test_chunk_id_rejects_empty_source(): chunk_id("", 0) +def test_chunk_id_sorts_in_cut_order_across_the_whole_supported_range(): + ids = [chunk_id("notes.md", i) for i in (0, 1, 9, 10, 99, 100, 999, 1000, MAX_CHUNK_INDEX)] + assert ids == sorted(ids) + + +def test_chunk_id_rejects_an_index_the_padding_cannot_hold(): + """Past 9999 the width stops being fixed and chunk-10000 sorts before -9999.""" + with pytest.raises(ValueError, match="sorting in cut order|<="): + chunk_id("notes.md", MAX_CHUNK_INDEX + 1) + + def test_chunk_id_rejects_negative_index(): with pytest.raises(ValueError, match="index"): chunk_id("notes.md", -1) @@ -37,13 +48,27 @@ def test_to_document_stringifies_every_metadata_value(): assert all(isinstance(value, str) for value in doc.metadata.values()) -def test_extra_metadata_is_merged_and_stringified(): +def test_extra_metadata_is_merged(): chunk = Chunk("body", 0, "page", 4, 4, extra={"page_label": "iv", "words": "12"}) doc = chunk.to_document("paper.pdf") assert doc.metadata["page_label"] == "iv" assert doc.metadata["locator_type"] == "page" +def test_non_string_extra_values_are_stringified(): + """`{"page": 3}` is the natural thing to write, so accept and coerce it. + + Moss types metadata as `Dict[str, str]`; an int left in there fails at the + SDK boundary, which is a worse place to find out than here. + """ + chunk = Chunk("body", 0, "page", 4, 4, extra={"page": 3, "words": 12, "ok": True}) + metadata = chunk.to_document("paper.pdf").metadata + assert metadata["page"] == "3" + assert metadata["words"] == "12" + assert metadata["ok"] == "True" + assert all(isinstance(value, str) for value in metadata.values()) + + def test_extra_may_not_shadow_reserved_keys(): with pytest.raises(ValueError, match="reserved"): Chunk("body", 0, "char", 0, 4, extra={"source": "elsewhere.md"}) @@ -74,6 +99,18 @@ def test_extra_is_copied_so_the_caller_cannot_mutate_validated_state(): assert chunk.to_document("notes.md").metadata["source"] == "notes.md" +def test_a_frozen_chunk_is_actually_hashable(): + """`frozen=True` advertises hashability; a dict field would break it.""" + chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) + assert hash(chunk) == hash(Chunk("body", 0, "char", 0, 4, extra={"other": "x"})) + assert len({chunk, Chunk("body", 1, "char", 4, 8)}) == 2 + + +def test_chunks_still_compare_on_extra(): + """Excluding `extra` from the hash must not exclude it from equality.""" + assert Chunk("b", 0, "char", 0, 1, extra={"a": "1"}) != Chunk("b", 0, "char", 0, 1) + + def test_unknown_locator_type_is_rejected(): with pytest.raises(ValueError, match="locator_type"): Chunk("body", 0, "byte", 0, 4) # type: ignore[arg-type] diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py index 6b9ab8b0..e3f97f86 100644 --- a/packages/moss-chunking/tests/test_strategies.py +++ b/packages/moss-chunking/tests/test_strategies.py @@ -35,15 +35,17 @@ ] -def roundtrips(text: str, chunks) -> bool: - return all(text[c.locator_start : c.locator_end] == c.text for c in chunks) +def assert_roundtrips(text: str, chunks) -> None: + """Assert per chunk, so a failure names the offending one and its offsets.""" + for chunk in chunks: + assert text[chunk.locator_start : chunk.locator_end] == chunk.text @pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) def test_offsets_slice_back_to_the_chunk_text(strategy): chunks = list(strategy.split(PROSE)) assert chunks - assert roundtrips(PROSE, chunks) + assert_roundtrips(PROSE, chunks) @pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) @@ -118,10 +120,19 @@ def test_sentence_longer_than_the_budget_is_still_emitted(): def test_sentence_splitter_handles_closing_quotes(): text = 'He said "it works." Then he left. She agreed.' chunks = list(SentenceSplitter(max_words=4, overlap_sentences=0).split(text)) - assert roundtrips(text, chunks) + assert_roundtrips(text, chunks) assert chunks[0].text.startswith("He said") +def test_splitting_terminates_for_any_overlap(): + """The `group_start + 1` floor, not the clamp, is what guarantees progress.""" + splitter = SentenceSplitter(max_words=3) + splitter.overlap_sentences = 999 # defeat the clamp, keep the loop + chunks = list(splitter.split("One. Two. Three. Four. Five. Six.")) + assert chunks + assert [c.index for c in chunks] == list(range(len(chunks))) + + def test_overlap_is_clamped_so_progress_is_guaranteed(): splitter = SentenceSplitter(max_words=50, overlap_sentences=99) assert splitter.overlap_sentences == 1 @@ -148,6 +159,14 @@ def test_paragraphs_are_split_apart_when_they_do_not_fit(): assert len(list(ParagraphSplitter(max_chars=6).split(text))) == 3 +def test_crlf_blank_lines_are_paragraph_boundaries_too(): + """A Windows-authored file must not collapse into one oversized chunk.""" + text = "one.\r\n\r\ntwo.\r\n\r\nthree." + chunks = list(ParagraphSplitter(max_chars=6).split(text)) + assert len(chunks) == 3 + assert_roundtrips(text, chunks) + + # --- RecursiveSplitter ------------------------------------------------------- diff --git a/packages/moss-chunking/uv.lock b/packages/moss-chunking/uv.lock index 3e5da749..22e918d8 100644 --- a/packages/moss-chunking/uv.lock +++ b/packages/moss-chunking/uv.lock @@ -153,7 +153,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "moss", specifier = ">=1.1.1" }, + { name = "moss", specifier = ">=1.7.2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "python-dotenv", marker = "extra == 'dev'", specifier = ">=1.0.0" }, From 2eb65bf8c500943157cfa54cf21c32ff5773d032 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 09:28:11 +0530 Subject: [PATCH 04/14] =?UTF-8?q?fix:=20second=20bot=20round=20=E2=80=94?= =?UTF-8?q?=20duplicate=20spans,=20immutable=20extra,=20metadata=20precede?= =?UTF-8?q?nce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CharSplitter could emit the same span twice. Windows advance along the raw text but chunks are the trimmed span inside them, so on padded input two consecutive windows trim to the same piece: chunk_chars=10, overlap=5 over " abcde fghij" yielded "abcde" at (5,10) as both chunk 0 and chunk 1 — identical content indexed twice under two IDs. Emission now skips any span that ends at or before the last one, so every chunk adds new content. extra is now a read-only view on a copy, not just a copy. The previous fix left it mutable while it still counted towards equality, so a chunk already in a set could change what it equals while its hash stayed put and the set could no longer find its own member. Immutability removes the divergence rather than papering over it, and answers the reviewers who asked for an immutable mapping. Source-level extra no longer overwrites a chunk's own metadata. chunk_document merged the document-wide dict last, so a splitter that knows the page or section number had it silently replaced by a constant. The chunk's value wins now. ingest promised something it did not do: it always calls create_index, so the stable-ID re-indexing it described could never happen through it. The docstring now says it is the create path and points at MossClient.add_docs, which is where stable IDs actually replace rather than append. chunk_id rejects a non-string source. chunk_id(123, 0) formatted happily and put a non-string into metadata, which is the boundary failure this module exists to catch. Reverted twine --skip-existing. Added last round at one reviewer's suggestion, another then pointed out it lets the tag step run against artifacts PyPI already has from a different commit. Plain upload is the template's behaviour and fails loudly instead. 69 tests. --- .github/workflows/publish-moss-chunking.yml | 4 +- packages/moss-chunking/src/chunk.py | 23 ++++++----- packages/moss-chunking/src/ingest.py | 13 ++++-- packages/moss-chunking/src/strategies.py | 16 +++++++- packages/moss-chunking/tests/test_chunk.py | 40 +++++++++++++++---- .../moss-chunking/tests/test_strategies.py | 28 +++++++++++++ 6 files changed, 99 insertions(+), 25 deletions(-) diff --git a/.github/workflows/publish-moss-chunking.yml b/.github/workflows/publish-moss-chunking.yml index f3705f22..7c11e5ba 100644 --- a/.github/workflows/publish-moss-chunking.yml +++ b/.github/workflows/publish-moss-chunking.yml @@ -139,9 +139,7 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | - # --skip-existing so a rerun after a partial upload can still reach the - # tag step instead of failing on files PyPI already has. - twine upload --skip-existing packages/moss-chunking/dist/* + twine upload packages/moss-chunking/dist/* - name: Tag release run: | diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 4eff3475..767fa974 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -18,7 +18,9 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass, field +from types import MappingProxyType from typing import Literal, get_args from moss import DocumentInfo @@ -50,6 +52,8 @@ def chunk_id(source: str, index: int) -> str: breaks the sort. A document that cuts into more than 10,000 chunks wants a coarser strategy or a per-section `source`, not a silently unsortable ID. """ + if not isinstance(source, str): + raise TypeError(f"source must be a str, got {type(source).__name__}") if not source: raise ValueError("source must be a non-empty string") if index < 0: @@ -78,11 +82,12 @@ class Chunk: #: Values are stringified at render, so the natural thing to pass — `{"page": #: 3}`, `{"words": 12}` — is accepted rather than failing at the SDK boundary. #: - #: Excluded from the hash: `frozen=True` has the dataclass advertise - #: hashability, and a dict field would make `hash(chunk)` raise instead. - #: Chunks still compare on `extra`; two that differ only there collide, which - #: is a legal hash, unlike a `Chunk` that cannot go in a set at all. - extra: dict[str, object] = field(default_factory=dict, hash=False) + #: Pass a plain dict; it is replaced with a read-only view on a copy. It has + #: to be immutable, not merely copied: `extra` counts towards equality, so a + #: mutable one lets a chunk already sitting in a set change what it equals + #: while its hash stays put, and the set stops being able to find it. + #: `hash=False` because a mapping cannot itself be hashed. + extra: Mapping[str, object] = field(default_factory=dict, hash=False) def __post_init__(self) -> None: if self.index < 0: @@ -100,10 +105,10 @@ def __post_init__(self) -> None: clashes = RESERVED_KEYS & self.extra.keys() if clashes: raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") - # `frozen=True` freezes the field, not the dict behind it. Without a copy - # the caller keeps a live handle on validated state and can add a - # reserved key after the check has already passed. - object.__setattr__(self, "extra", dict(self.extra)) + # `frozen=True` freezes the field, not the dict behind it. Copy so the + # caller's handle cannot add a reserved key after the check has passed, + # and wrap it read-only so nothing can reach through `chunk.extra` either. + object.__setattr__(self, "extra", MappingProxyType(dict(self.extra))) def to_document(self, source: str) -> DocumentInfo: """Render this chunk as a Moss `DocumentInfo`. diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py index da91cfe2..6e221777 100644 --- a/packages/moss-chunking/src/ingest.py +++ b/packages/moss-chunking/src/ingest.py @@ -14,11 +14,16 @@ async def ingest( index_name: str, model_id: str | None = None, ) -> MutationResult | None: - """Copy every chunk into a fresh Moss index. + """Create a Moss index holding exactly these chunks. - Deliberately without the connector template's `auto_id` option: random UUIDs - would defeat the contract's stable IDs, and re-indexing an unchanged document - would append duplicates instead of replacing what is already there. + This is the create path only: it builds a fresh index every time, so it is + not what you reach for to refresh one document inside an existing one — that + is `MossClient.add_docs`, and it is where the contract's stable IDs earn + their keep, because re-chunking an unchanged document reproduces the same + IDs and replaces those chunks rather than appending a second copy. + + Deliberately without the connector template's `auto_id` option, for the same + reason: random UUIDs would make every chunk look new on the way back in. """ docs = list(documents) if not docs: diff --git a/packages/moss-chunking/src/strategies.py b/packages/moss-chunking/src/strategies.py index efa0b2e2..e55213c2 100644 --- a/packages/moss-chunking/src/strategies.py +++ b/packages/moss-chunking/src/strategies.py @@ -117,9 +117,17 @@ def split(self, text: str) -> Iterator[Chunk]: length = len(text) start = 0 index = 0 + # Windows advance along the raw text, but what gets emitted is the + # trimmed span inside them. On padded input two consecutive windows can + # trim down to the same span, so track where the last chunk ended and + # skip anything that adds no new content — otherwise the same text is + # indexed twice under two IDs. + last_end = -1 while start < length: end = min(start + self.chunk_chars, length) for piece_start, piece_end in _trim(text, [(start, end)]): + if piece_end <= last_end: + continue yield Chunk( text=text[piece_start:piece_end], index=index, @@ -128,6 +136,7 @@ def split(self, text: str) -> Iterator[Chunk]: locator_end=piece_end, ) index += 1 + last_end = piece_end if end >= length: break start = max(end - self.overlap, start + 1) @@ -302,10 +311,15 @@ def chunk_document( `extra` is merged into every chunk's metadata — the place for source-level facts like `extension` or `modified_at` that the splitter cannot know. + + Where the two collide the chunk's own value wins. `extra` is by definition + the same for every chunk, so letting it overwrite would mean a document-wide + constant silently replacing the page or section number that only the + splitter was in a position to know. """ documents: list[DocumentInfo] = [] for chunk in strategy.split(text): if extra: - chunk = replace(chunk, extra={**chunk.extra, **extra}) + chunk = replace(chunk, extra={**extra, **chunk.extra}) documents.append(chunk.to_document(source)) return documents diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index 02ec458f..a06c785a 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -74,21 +74,19 @@ def test_extra_may_not_shadow_reserved_keys(): Chunk("body", 0, "char", 0, 4, extra={"source": "elsewhere.md"}) -def test_reserved_keys_win_even_if_extra_is_mutated_after_construction(): - """The check at construction is not the last line of defence. +def test_reserved_keys_win_at_render_even_if_validation_is_bypassed(): + """Belt and suspenders: the constructor check is not the last defence. - `frozen=True` freezes the field, not the dict behind it, so a reserved key - can still be written into `extra` after validation has passed. Rendering - merges `extra` first, so the contract's own keys overwrite it either way. + Nothing should be able to get a reserved key into `extra` — the constructor + rejects one and the stored mapping is read-only. Forced past both, rendering + still merges `extra` first so the contract's own keys overwrite it. """ chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) - chunk.extra["source"] = "elsewhere.md" - chunk.extra["chunk_index"] = "99" + object.__setattr__(chunk, "extra", {"source": "elsewhere.md", "chunk_index": "99"}) doc = chunk.to_document("notes.md") assert doc.metadata["source"] == "notes.md" assert doc.metadata["chunk_index"] == "0" - assert doc.metadata["extension"] == "md" def test_extra_is_copied_so_the_caller_cannot_mutate_validated_state(): @@ -111,6 +109,32 @@ def test_chunks_still_compare_on_extra(): assert Chunk("b", 0, "char", 0, 1, extra={"a": "1"}) != Chunk("b", 0, "char", 0, 1) +def test_extra_cannot_be_mutated_through_the_chunk(): + """`extra` counts towards equality, so it must not be able to change. + + A mutable one lets a chunk already in a set change what it equals while its + hash stays put — the set can then no longer find its own member. + """ + chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) + with pytest.raises(TypeError): + chunk.extra["extension"] = "txt" # type: ignore[index] + with pytest.raises(TypeError): + chunk.extra["source"] = "elsewhere.md" # type: ignore[index] + + +def test_a_chunk_in_a_set_stays_findable(): + chunk = Chunk("body", 0, "char", 0, 4, extra={"k": "v"}) + members = {chunk} + assert chunk in members + assert Chunk("body", 0, "char", 0, 4, extra={"k": "v"}) in members + + +def test_chunk_id_rejects_a_non_string_source(): + """`chunk_id(123, 0)` would otherwise emit a non-string `source`.""" + with pytest.raises(TypeError, match="source must be a str"): + chunk_id(123, 0) # type: ignore[arg-type] + + def test_unknown_locator_type_is_rejected(): with pytest.raises(ValueError, match="locator_type"): Chunk("body", 0, "byte", 0, 4) # type: ignore[arg-type] diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py index e3f97f86..12f59610 100644 --- a/packages/moss-chunking/tests/test_strategies.py +++ b/packages/moss-chunking/tests/test_strategies.py @@ -11,6 +11,7 @@ from moss import DocumentInfo from moss_chunking import ( CharSplitter, + Chunk, ParagraphSplitter, RecursiveSplitter, SentenceSplitter, @@ -91,6 +92,20 @@ def test_char_splitter_covers_the_whole_document(): assert chunks[-1].locator_end == len(PROSE) +def test_padded_input_does_not_emit_the_same_span_twice(): + """Windows advance along raw text, but chunks are the trimmed span inside. + + Two consecutive windows over padded text can trim to the same span, which + would index identical content twice under two different chunk IDs. + """ + text = " abcde fghij" + chunks = list(CharSplitter(chunk_chars=10, overlap=5).split(text)) + spans = [(c.locator_start, c.locator_end) for c in chunks] + assert len(spans) == len(set(spans)) + assert [c.text for c in chunks] == ["abcde", "fghij"] + assert [c.index for c in chunks] == [0, 1] + + def test_overlap_at_or_above_chunk_size_is_rejected(): with pytest.raises(ValueError, match="never advances"): CharSplitter(chunk_chars=100, overlap=100) @@ -245,6 +260,19 @@ def test_chunk_document_merges_source_level_extra_metadata(): assert all(d.metadata["source"] == "notes.md" for d in docs) +def test_a_chunks_own_metadata_beats_source_level_extra(): + """`extra` is document-wide; the splitter knows the per-chunk truth.""" + + class Paged: + def split(self, text: str): + yield Chunk(text[:4], 0, "page", 1, 1, extra={"page": 1}) + yield Chunk(text[4:8], 1, "page", 2, 2, extra={"page": 2}) + + docs = chunk_document("aaaabbbb", "paper.pdf", Paged(), extra={"page": 0, "ext": "pdf"}) + assert [d.metadata["page"] for d in docs] == ["1", "2"] + assert all(d.metadata["ext"] == "pdf" for d in docs) + + def test_chunk_document_rejects_extra_that_shadows_the_contract(): with pytest.raises(ValueError, match="reserved"): chunk_document(PROSE, "notes.md", CharSplitter(), extra={"chunk_index": "9"}) From 6f5603bacd4fe3ac4545fa5be185437f7c846c00 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 09:30:23 +0530 Subject: [PATCH 05/14] fix: reject an unsortable chunk index at construction, not at render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MAX_CHUNK_INDEX was enforced only in chunk_id, which runs at to_document time, so a chunk past the bound was accepted at construction and failed later — mid-iteration inside chunk_document, far from where the index was set. The same check now runs in __post_init__, so the two validation sites agree. 70 tests. --- packages/moss-chunking/src/chunk.py | 8 ++++++++ packages/moss-chunking/tests/test_chunk.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 767fa974..f7f6d85c 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -92,6 +92,14 @@ class Chunk: def __post_init__(self) -> None: if self.index < 0: raise ValueError(f"index must be >= 0, got {self.index}") + # Same bound `chunk_id` enforces, checked here too so an unsortable chunk + # is rejected where its index was set rather than later, mid-render. + if self.index > MAX_CHUNK_INDEX: + raise ValueError( + f"index must be <= {MAX_CHUNK_INDEX}, got {self.index}: past that " + "the zero-padding is no longer fixed width and IDs stop sorting " + "in cut order" + ) if self.locator_type not in LOCATOR_TYPES: raise ValueError( f"locator_type must be one of {LOCATOR_TYPES}, got {self.locator_type!r}" diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index a06c785a..615369b8 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -129,6 +129,13 @@ def test_a_chunk_in_a_set_stays_findable(): assert Chunk("body", 0, "char", 0, 4, extra={"k": "v"}) in members +def test_chunk_rejects_an_unsortable_index_at_construction(): + """Rejected where the index is set, not later inside `to_document`.""" + with pytest.raises(ValueError, match="sorting in cut order|<="): + Chunk("body", MAX_CHUNK_INDEX + 1, "char", 0, 4) + assert Chunk("body", MAX_CHUNK_INDEX, "char", 0, 4).index == MAX_CHUNK_INDEX + + def test_chunk_id_rejects_a_non_string_source(): """`chunk_id(123, 0)` would otherwise emit a non-string `source`.""" with pytest.raises(TypeError, match="source must be a str"): From 6a6c1311760ef3ba9b6a9df7428f6c3c54858e18 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 09:38:29 +0530 Subject: [PATCH 06/14] fix: reject a non-integer chunk index in both validation sites bool subclasses int, so Chunk(text, True, ...) formatted as a perfectly valid chunk-0001 and nothing complained. A float was accepted at construction and then died at the :04d format with "Unknown format code 'd'", a long way from whatever set the index. Both checks now live in one _require_index helper called from chunk_id and __post_init__, which also removes the bound check that was duplicated across the two after the last fix. 75 tests. --- packages/moss-chunking/src/chunk.py | 40 +++++++++++++--------- packages/moss-chunking/tests/test_chunk.py | 9 +++++ 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index f7f6d85c..89b171e2 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -39,6 +39,25 @@ MAX_CHUNK_INDEX = 9999 +def _require_index(index: int) -> None: + """Reject anything that is not a plain in-range int. + + `bool` is excluded explicitly because it subclasses `int`, so `True` would + otherwise format as a perfectly valid `chunk-0001`. A float gets caught here + rather than at the `:04d` format, which fails with `Unknown format code 'd'` + a long way from whatever set the index. + """ + if not isinstance(index, int) or isinstance(index, bool): + raise TypeError(f"index must be an int, got {type(index).__name__}") + if index < 0: + raise ValueError(f"index must be >= 0, got {index}") + if index > MAX_CHUNK_INDEX: + raise ValueError( + f"index must be <= {MAX_CHUNK_INDEX}, got {index}: past that the " + "zero-padding is no longer fixed width and IDs stop sorting in cut order" + ) + + def chunk_id(source: str, index: int) -> str: """Build a chunk's stable ID. @@ -56,13 +75,7 @@ def chunk_id(source: str, index: int) -> str: raise TypeError(f"source must be a str, got {type(source).__name__}") if not source: raise ValueError("source must be a non-empty string") - if index < 0: - raise ValueError(f"index must be >= 0, got {index}") - if index > MAX_CHUNK_INDEX: - raise ValueError( - f"index must be <= {MAX_CHUNK_INDEX}, got {index}: past that the " - "zero-padding is no longer fixed width and IDs stop sorting in cut order" - ) + _require_index(index) return f"{source}#chunk-{index:04d}" @@ -90,16 +103,9 @@ class Chunk: extra: Mapping[str, object] = field(default_factory=dict, hash=False) def __post_init__(self) -> None: - if self.index < 0: - raise ValueError(f"index must be >= 0, got {self.index}") - # Same bound `chunk_id` enforces, checked here too so an unsortable chunk - # is rejected where its index was set rather than later, mid-render. - if self.index > MAX_CHUNK_INDEX: - raise ValueError( - f"index must be <= {MAX_CHUNK_INDEX}, got {self.index}: past that " - "the zero-padding is no longer fixed width and IDs stop sorting " - "in cut order" - ) + # The same check `chunk_id` runs, so a chunk that could never render a + # valid ID is rejected where its index was set rather than mid-render. + _require_index(self.index) if self.locator_type not in LOCATOR_TYPES: raise ValueError( f"locator_type must be one of {LOCATOR_TYPES}, got {self.locator_type!r}" diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index 615369b8..64bbefa1 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -136,6 +136,15 @@ def test_chunk_rejects_an_unsortable_index_at_construction(): assert Chunk("body", MAX_CHUNK_INDEX, "char", 0, 4).index == MAX_CHUNK_INDEX +@pytest.mark.parametrize("bad", [1.5, True, False, "0", None]) +def test_a_non_integer_index_is_rejected(bad): + """`bool` subclasses `int`, so True would otherwise format as chunk-0001.""" + with pytest.raises(TypeError, match="index must be an int"): + chunk_id("notes.md", bad) + with pytest.raises(TypeError, match="index must be an int"): + Chunk("body", bad, "char", 0, 4) + + def test_chunk_id_rejects_a_non_string_source(): """`chunk_id(123, 0)` would otherwise emit a non-string `source`.""" with pytest.raises(TypeError, match="source must be a str"): From c3cf58995e4761700c1e34ba2379f14f99cb6b96 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 09:44:22 +0530 Subject: [PATCH 07/14] fix: guard the publish ref, and make Chunk copyable again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish job holds PYPI_API_TOKEN and pushes a release tag, and workflow_dispatch accepts any ref — so an unmerged branch could be published under the package name and tagged as a release. Guarded on refs/heads/main. None of the repo's 20 publish workflows has this guard; not fixing the others here, but a new one should not ship without it. MappingProxyType was the wrong way to make extra immutable. It is not picklable, which took deepcopy and dataclasses.asdict down with it, and it still could not stop a mutable *value* — a list held in extra — from changing a chunk's equality after it went into a set. Both come from treating metadata as part of identity. A chunk is its text and its position; what is recorded about it does not change which cut of the document it is. extra is now a plain dict copy excluded from eq and hash, so the obvious dataclass paths work again and nothing extra holds can make a chunk in a set unfindable, by rebinding or by mutation. 75 tests. --- .github/workflows/publish-moss-chunking.yml | 4 +++ packages/moss-chunking/src/chunk.py | 24 +++++++------ packages/moss-chunking/tests/test_chunk.py | 39 +++++++++++---------- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/.github/workflows/publish-moss-chunking.yml b/.github/workflows/publish-moss-chunking.yml index 7c11e5ba..d8dbffe9 100644 --- a/.github/workflows/publish-moss-chunking.yml +++ b/.github/workflows/publish-moss-chunking.yml @@ -101,6 +101,10 @@ jobs: # Only runs once every build-test matrix leg has passed. publish: needs: [determine-version, build-test] + # This job holds PYPI_API_TOKEN and pushes a release tag. `workflow_dispatch` + # accepts any ref, so without this guard an unmerged branch could be + # published under the package name and tagged as a release. + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: contents: write diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 89b171e2..60d20c80 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -18,9 +18,7 @@ from __future__ import annotations -from collections.abc import Mapping from dataclasses import dataclass, field -from types import MappingProxyType from typing import Literal, get_args from moss import DocumentInfo @@ -95,12 +93,14 @@ class Chunk: #: Values are stringified at render, so the natural thing to pass — `{"page": #: 3}`, `{"words": 12}` — is accepted rather than failing at the SDK boundary. #: - #: Pass a plain dict; it is replaced with a read-only view on a copy. It has - #: to be immutable, not merely copied: `extra` counts towards equality, so a - #: mutable one lets a chunk already sitting in a set change what it equals - #: while its hash stays put, and the set stops being able to find it. - #: `hash=False` because a mapping cannot itself be hashed. - extra: Mapping[str, object] = field(default_factory=dict, hash=False) + #: Copied on construction, so a caller's handle cannot reach back in. + #: + #: Excluded from equality and hashing: a chunk *is* its text and position, + #: and metadata about it does not change which cut of the document it is. + #: That is also what keeps it safe to leave mutable — nothing here can make a + #: chunk already sitting in a set change what it equals, whether by rebinding + #: a key or by mutating a list held as a value. + extra: dict[str, object] = field(default_factory=dict, compare=False) def __post_init__(self) -> None: # The same check `chunk_id` runs, so a chunk that could never render a @@ -120,9 +120,11 @@ def __post_init__(self) -> None: if clashes: raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") # `frozen=True` freezes the field, not the dict behind it. Copy so the - # caller's handle cannot add a reserved key after the check has passed, - # and wrap it read-only so nothing can reach through `chunk.extra` either. - object.__setattr__(self, "extra", MappingProxyType(dict(self.extra))) + # caller's handle cannot add a reserved key after the check has passed. + # A plain dict rather than a read-only view, because a `MappingProxyType` + # is not picklable and takes `deepcopy` and `dataclasses.asdict` down + # with it. Excluding `extra` from equality is what makes the copy enough. + object.__setattr__(self, "extra", dict(self.extra)) def to_document(self, source: str) -> DocumentInfo: """Render this chunk as a Moss `DocumentInfo`. diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index 64bbefa1..c339bf11 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -2,6 +2,10 @@ from __future__ import annotations +import copy +import dataclasses +import pickle + import pytest from moss_chunking import MAX_CHUNK_INDEX, Chunk, chunk_id @@ -104,29 +108,28 @@ def test_a_frozen_chunk_is_actually_hashable(): assert len({chunk, Chunk("body", 1, "char", 4, 8)}) == 2 -def test_chunks_still_compare_on_extra(): - """Excluding `extra` from the hash must not exclude it from equality.""" - assert Chunk("b", 0, "char", 0, 1, extra={"a": "1"}) != Chunk("b", 0, "char", 0, 1) +def test_identity_is_text_and_position_not_metadata(): + """`extra` describes a chunk; it does not decide which chunk it is.""" + assert Chunk("b", 0, "char", 0, 1, extra={"a": "1"}) == Chunk("b", 0, "char", 0, 1) + assert Chunk("b", 0, "char", 0, 1) != Chunk("b", 1, "char", 0, 1) -def test_extra_cannot_be_mutated_through_the_chunk(): - """`extra` counts towards equality, so it must not be able to change. +def test_a_chunk_in_a_set_stays_findable_however_extra_changes(): + """Equality cannot drift out from under a set, by rebinding or mutation.""" + chunk = Chunk("body", 0, "char", 0, 4, extra={"tags": ["a"]}) + members = {chunk} - A mutable one lets a chunk already in a set change what it equals while its - hash stays put — the set can then no longer find its own member. - """ - chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) - with pytest.raises(TypeError): - chunk.extra["extension"] = "txt" # type: ignore[index] - with pytest.raises(TypeError): - chunk.extra["source"] = "elsewhere.md" # type: ignore[index] + chunk.extra["tags"].append("b") # type: ignore[attr-defined] + chunk.extra["added"] = "later" + assert chunk in members -def test_a_chunk_in_a_set_stays_findable(): - chunk = Chunk("body", 0, "char", 0, 4, extra={"k": "v"}) - members = {chunk} - assert chunk in members - assert Chunk("body", 0, "char", 0, 4, extra={"k": "v"}) in members +def test_a_chunk_survives_pickle_deepcopy_and_asdict(): + """The obvious dataclass paths must keep working.""" + chunk = Chunk("body", 0, "char", 0, 4, extra={"page": 3}) + assert pickle.loads(pickle.dumps(chunk)) == chunk + assert copy.deepcopy(chunk) == chunk + assert dataclasses.asdict(chunk)["extra"] == {"page": 3} def test_chunk_rejects_an_unsortable_index_at_construction(): From 95920e1ad20d570ee019e2f619e335c4a1d31260 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 09:55:46 +0530 Subject: [PATCH 08/14] fix: pass payload only where the installed DocumentInfo has it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit moss 1.7.2's runtime DocumentInfo does accept payload — its signature is (id, text, metadata=None, embedding=None, payload=None) and the value round trips — but the SDK's shipped __init__.pyi stub declares only the first four. Code that passes the kwarg unconditionally is therefore correct at runtime and wrong to a type checker reading the stub. Passing it conditionally keeps the package right against both, and degrades to dropping payload only on a build that has no such field to lose. The test skips rather than fails in that case, so the package is not pinned to one runtime shape of DocumentInfo. 75 tests. --- packages/moss-chunking/src/enrich.py | 13 +++++++++---- packages/moss-chunking/tests/test_strategies.py | 10 +++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/moss-chunking/src/enrich.py b/packages/moss-chunking/src/enrich.py index e92a1bae..99b2c619 100644 --- a/packages/moss-chunking/src/enrich.py +++ b/packages/moss-chunking/src/enrich.py @@ -32,19 +32,24 @@ def prepend_context(doc: DocumentInfo, fields: Mapping[str, str]) -> DocumentInf safe direction to fail; the alternative is a rule that enrichment must happen before embedding, which nothing can enforce. - `payload` is carried through. Unlike the embedding it has nothing to do with - the text, so rebuilding the document without it would be silent data loss - rather than a decision. + `payload` is carried through where the installed SDK has it. Unlike the + embedding it has nothing to do with the text, so rebuilding the document + without it would be silent data loss rather than a decision. It is passed + conditionally because `moss`'s runtime `DocumentInfo` accepts it while the + shipped `__init__.pyi` stub does not yet declare it — this keeps the package + correct against both, and degrades to dropping it only on a build that has + no such field to lose. """ if not fields: return doc header = "\n".join(f"{key}: {value}" for key, value in fields.items()) + carried = {"payload": doc.payload} if hasattr(doc, "payload") else {} return DocumentInfo( id=doc.id, text=f"{header}\n\n{doc.text}", metadata=doc.metadata, embedding=None, - payload=getattr(doc, "payload", None), + **carried, ) diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py index 12f59610..53495c7c 100644 --- a/packages/moss-chunking/tests/test_strategies.py +++ b/packages/moss-chunking/tests/test_strategies.py @@ -7,6 +7,8 @@ from __future__ import annotations +import inspect + import pytest from moss import DocumentInfo from moss_chunking import ( @@ -296,7 +298,13 @@ def test_enrichment_drops_an_embedding_computed_from_the_old_text(): def test_enrichment_carries_the_payload_through(): - """Unlike the embedding, `payload` has nothing to do with the text.""" + """Unlike the embedding, `payload` has nothing to do with the text. + + Skipped rather than failed on an SDK build without the field, so the package + does not pin itself to one runtime shape of `DocumentInfo`. + """ + if "payload" not in inspect.signature(DocumentInfo).parameters: + pytest.skip("installed moss DocumentInfo has no payload field") doc = DocumentInfo(id="a#chunk-0000", text="body", metadata={"source": "a"}, payload='{"p":1}') enriched = prepend_source_context(doc, filename="a.md", path="/a.md") assert enriched.payload == '{"p":1}' From 110f9484cb2ef3cfe76c929f4bf4f8b6cdfc42e5 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 13:53:16 +0530 Subject: [PATCH 09/14] =?UTF-8?q?fix:=20third=20bot=20round=20=E2=80=94=20?= =?UTF-8?q?non-string=20extra=20keys,=20non-sequential=20chunk=20indices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reject non-string keys in `Chunk.extra` at construction rather than coercing them: `str(1)` and "1" are the same metadata key, so coercion would let one entry overwrite another, and an uncoerced key slips past the reserved-key check to fail at the SDK boundary. Validate in `chunk_document` that a strategy numbers its chunks 0, 1, 2, …. A repeat renders the same ID twice and the second chunk overwrites the first on ingest — silent data loss. Renumbering would hide that just as quietly and rewrite the addressing of a splitter that meant something by its index. Co-Authored-By: Claude Opus 5 --- packages/moss-chunking/src/chunk.py | 13 ++++++++++++- packages/moss-chunking/src/strategies.py | 16 +++++++++++++++- packages/moss-chunking/tests/test_chunk.py | 13 ++++++++++++- packages/moss-chunking/tests/test_strategies.py | 17 +++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 60d20c80..1a5b0eb1 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -92,6 +92,8 @@ class Chunk: locator_end: int #: Values are stringified at render, so the natural thing to pass — `{"page": #: 3}`, `{"words": 12}` — is accepted rather than failing at the SDK boundary. + #: Keys must already be strings; see `__post_init__` for why they are not + #: coerced the same way. #: #: Copied on construction, so a caller's handle cannot reach back in. #: @@ -116,6 +118,14 @@ def __post_init__(self) -> None: raise ValueError( f"locator_end ({self.locator_end}) must be >= locator_start ({self.locator_start})" ) + # Values are coerced at render because `{"page": 3}` is the natural thing + # to pass and `"3"` is unambiguously what was meant. Keys are rejected + # instead: `str(1)` and `"1"` are the same metadata key, so coercing them + # would let one entry silently overwrite another, and a non-string key + # cannot collide with a reserved one and so slips past the check below. + non_string = sorted(repr(key) for key in self.extra if not isinstance(key, str)) + if non_string: + raise TypeError(f"extra keys must be str, got {', '.join(non_string)}") clashes = RESERVED_KEYS & self.extra.keys() if clashes: raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") @@ -131,7 +141,8 @@ def to_document(self, source: str) -> DocumentInfo: Every metadata value is stringified because Moss types metadata as `Dict[str, str]`. An int left in there would fail at the SDK boundary, - which is a worse place to discover it than here. + which is a worse place to discover it than here. Keys are already known + to be strings — the constructor rejects any that are not. `extra` is merged *first* so the contract's own keys always win. The constructor already rejects a clashing `extra`, so this only matters if diff --git a/packages/moss-chunking/src/strategies.py b/packages/moss-chunking/src/strategies.py index e55213c2..dc0c32ef 100644 --- a/packages/moss-chunking/src/strategies.py +++ b/packages/moss-chunking/src/strategies.py @@ -316,9 +316,23 @@ def chunk_document( the same for every chunk, so letting it overwrite would mean a document-wide constant silently replacing the page or section number that only the splitter was in a position to know. + + A strategy that does not number its chunks `0, 1, 2, …` is rejected here. + The index is what the ID is built from, so a repeat means two chunks render + the same ID and the second overwrites the first on ingest — silent data + loss, discovered later as a document with missing content. Renumbering them + would paper over that just as quietly, and would rewrite the addressing of a + splitter that meant something by its index, so the mismatch is raised + instead. """ documents: list[DocumentInfo] = [] - for chunk in strategy.split(text): + for position, chunk in enumerate(strategy.split(text)): + if chunk.index != position: + raise ValueError( + f"{type(strategy).__name__} yielded chunk.index={chunk.index} at " + f"position {position}: indices must run 0, 1, 2, … or chunk IDs " + "collide and stop sorting in cut order" + ) if extra: chunk = replace(chunk, extra={**extra, **chunk.extra}) documents.append(chunk.to_document(source)) diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index c339bf11..ad5d58f6 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -73,6 +73,17 @@ def test_non_string_extra_values_are_stringified(): assert all(isinstance(value, str) for value in metadata.values()) +def test_non_string_extra_keys_are_rejected(): + """Keys are not coerced the way values are, and must fail in this package. + + `str(1)` and `"1"` are the same metadata key, so coercing would let one + entry quietly overwrite another; left alone, a non-string key sails past the + reserved-key check and fails at the SDK boundary instead. + """ + with pytest.raises(TypeError, match="extra keys must be str"): + Chunk("body", 0, "char", 0, 4, extra={1: "one"}) + + def test_extra_may_not_shadow_reserved_keys(): with pytest.raises(ValueError, match="reserved"): Chunk("body", 0, "char", 0, 4, extra={"source": "elsewhere.md"}) @@ -82,7 +93,7 @@ def test_reserved_keys_win_at_render_even_if_validation_is_bypassed(): """Belt and suspenders: the constructor check is not the last defence. Nothing should be able to get a reserved key into `extra` — the constructor - rejects one and the stored mapping is read-only. Forced past both, rendering + rejects one and stores a copy the caller cannot reach. Forced past both, rendering still merges `extra` first so the contract's own keys overwrite it. """ chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py index 53495c7c..0b5cc8f1 100644 --- a/packages/moss-chunking/tests/test_strategies.py +++ b/packages/moss-chunking/tests/test_strategies.py @@ -275,6 +275,23 @@ def split(self, text: str): assert all(d.metadata["ext"] == "pdf" for d in docs) +@pytest.mark.parametrize("indices", [(0, 0), (0, 2), (1, 2)]) +def test_chunk_document_rejects_indices_that_are_not_sequential(indices): + """A repeat renders the same ID twice, and the second chunk wins on ingest. + + That is silent data loss, so a strategy that does not number its chunks from + zero is caught at the render boundary rather than trusted. + """ + + class Misnumbering: + def split(self, text: str): + for index in indices: + yield Chunk(text[:4], index, "char", 0, 4) + + with pytest.raises(ValueError, match="Misnumbering yielded chunk.index="): + chunk_document("aaaabbbb", "notes.md", Misnumbering()) + + def test_chunk_document_rejects_extra_that_shadows_the_contract(): with pytest.raises(ValueError, match="reserved"): chunk_document(PROSE, "notes.md", CharSplitter(), extra={"chunk_index": "9"}) From 2ff07e22c3a1e4a58b9667d73ee7e66480e4801b Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 15:56:25 +0530 Subject: [PATCH 10/14] =?UTF-8?q?fix:=20add=20refresh=5Fsource=20=E2=80=94?= =?UTF-8?q?=20upsert=20alone=20leaves=20a=20shrunken=20document's=20tail?= =?UTF-8?q?=20behind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `add_docs` upserts but does not reconcile: a document re-cut from 21 chunks into 6 leaves #chunk-0006..0020 in the index, still searchable, holding text the document no longer contains. Nothing errors and the stale hits look real. `refresh_source` deletes that tail before the new chunks go in. Finding it is cheap because the contract guarantees contiguous indices, so leftovers sit in one run above the new chunk count and a single window of candidate IDs past the end either finds it or proves it absent. Passing no documents removes the source entirely, which is how a deleted file leaves the index. Co-Authored-By: Claude Opus 5 --- packages/moss-chunking/README.md | 19 ++- packages/moss-chunking/src/__init__.py | 3 +- packages/moss-chunking/src/ingest.py | 80 ++++++++++- packages/moss-chunking/tests/test_ingest.py | 140 ++++++++++++++++++++ 4 files changed, 232 insertions(+), 10 deletions(-) create mode 100644 packages/moss-chunking/tests/test_ingest.py diff --git a/packages/moss-chunking/README.md b/packages/moss-chunking/README.md index ff1c8493..aa77c5ff 100644 --- a/packages/moss-chunking/README.md +++ b/packages/moss-chunking/README.md @@ -46,6 +46,19 @@ await ingest(docs, project_id, project_key, "my-index") `docs` are ordinary `DocumentInfo`s — they go anywhere the SDK takes documents; `ingest` is just the connector template's one-call shortcut into a fresh index. +To re-chunk one document inside an index that already exists, use +`refresh_source` rather than `add_docs`: + +```python +await refresh_source(client, "my-index", "notes.md", docs) +``` + +`add_docs` upserts, which replaces the chunks the new cut still produces — but a +document that shrinks from 21 chunks to 6 leaves `#chunk-0006` through +`#chunk-0020` in the index, still searchable, holding text the document no longer +contains. `refresh_source` deletes that tail first. Passing no documents removes +the source entirely, which is how a deleted file leaves the index. + ## The contract Every chunk, from every strategy, carries: @@ -59,8 +72,10 @@ Every chunk, from every strategy, carries: IDs are `{source}#chunk-{index:04d}` — zero-padded so they sort in cut order, and stable across runs so re-chunking an unchanged document replaces its chunks -rather than duplicating them. Values are all strings, because Moss types metadata -as `Dict[str, str]`. +rather than duplicating them. Indices run `0, 1, 2, …`; `chunk_document` rejects a +strategy that skips or repeats one, because a repeat renders the same ID twice and +the second chunk silently overwrites the first. Values are all strings, because +Moss types metadata as `Dict[str, str]`. The sort only holds while the padding is fixed width, so `chunk_id` rejects an index above `MAX_CHUNK_INDEX` (9999) rather than emitting `chunk-10000`, which diff --git a/packages/moss-chunking/src/__init__.py b/packages/moss-chunking/src/__init__.py index e585af3e..c01639de 100644 --- a/packages/moss-chunking/src/__init__.py +++ b/packages/moss-chunking/src/__init__.py @@ -20,7 +20,7 @@ chunk_id, ) from .enrich import prepend_context, prepend_source_context -from .ingest import ingest +from .ingest import ingest, refresh_source from .strategies import ( CharSplitter, ChunkingStrategy, @@ -46,4 +46,5 @@ "ingest", "prepend_context", "prepend_source_context", + "refresh_source", ] diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py index 6e221777..64620919 100644 --- a/packages/moss-chunking/src/ingest.py +++ b/packages/moss-chunking/src/ingest.py @@ -4,7 +4,20 @@ from collections.abc import Iterable -from moss import DocumentInfo, MossClient, MutationResult +from moss import ( + DocumentInfo, + GetDocumentsOptions, + MossClient, + MutationOptions, + MutationResult, +) + +from .chunk import MAX_CHUNK_INDEX, chunk_id + +#: How many candidate IDs to look up at once when hunting for a source's leftover +#: chunks. One round trip covers a document that shrank by up to this many chunks, +#: which is nearly all of them. +_PROBE_WINDOW = 256 async def ingest( @@ -17,16 +30,69 @@ async def ingest( """Create a Moss index holding exactly these chunks. This is the create path only: it builds a fresh index every time, so it is - not what you reach for to refresh one document inside an existing one — that - is `MossClient.add_docs`, and it is where the contract's stable IDs earn - their keep, because re-chunking an unchanged document reproduces the same - IDs and replaces those chunks rather than appending a second copy. + not what you reach for to refresh one document inside an existing one — use + `refresh_source` for that. - Deliberately without the connector template's `auto_id` option, for the same - reason: random UUIDs would make every chunk look new on the way back in. + Deliberately without the connector template's `auto_id` option: random UUIDs + would make every chunk look new on the way back in, where the contract's + stable IDs let an unchanged document reproduce exactly the IDs it had. """ docs = list(documents) if not docs: return None client = MossClient(project_id, project_key) return await client.create_index(index_name, docs, model_id=model_id) + + +async def refresh_source( + client: MossClient, + index_name: str, + source: str, + documents: Iterable[DocumentInfo], +) -> MutationResult | None: + """Replace everything `index_name` holds for `source` with `documents`. + + Stable IDs make re-chunking an unchanged document a no-op and a rewritten one + an overwrite — but only for the chunks that still exist. `add_docs` upserts, + it does not reconcile: if `notes.md` used to cut into 21 chunks and now cuts + into 6, `#chunk-0006` through `#chunk-0020` stay in the index, still + searchable, holding text that is no longer in the document. Nothing errors, + and the stale hits look exactly like real ones. + + So the tail is deleted before the new chunks go in. What makes finding it + cheap is the contract itself: `chunk_document` guarantees indices run + `0, 1, 2, …`, so anything left over sits in a contiguous run above the new + chunk count, and looking up one window of candidate IDs past the end is + enough to find it or prove it is not there. + + Passing no documents deletes every chunk for `source`, which is how a deleted + file is removed from the index. + + `upsert` is set explicitly rather than left to the server's default, since + replacing a chunk in place is the entire premise of the ID contract. + """ + docs = list(documents) + + stale: list[str] = [] + start = len(docs) + while start <= MAX_CHUNK_INDEX: + window = [ + chunk_id(source, index) + for index in range(start, min(start + _PROBE_WINDOW, MAX_CHUNK_INDEX + 1)) + ] + found = await client.get_docs(index_name, GetDocumentsOptions(doc_ids=window)) + if not found: + break + stale.extend(doc.id for doc in found) + start += len(window) + + if stale: + # Batched, because the stale run can be thousands of IDs long and a + # single request carrying all of them is a request that can fail all of + # them. + for offset in range(0, len(stale), _PROBE_WINDOW): + await client.delete_docs(index_name, stale[offset : offset + _PROBE_WINDOW]) + + if not docs: + return None + return await client.add_docs(index_name, docs, MutationOptions(upsert=True)) diff --git a/packages/moss-chunking/tests/test_ingest.py b/packages/moss-chunking/tests/test_ingest.py new file mode 100644 index 00000000..bcfdb82a --- /dev/null +++ b/packages/moss-chunking/tests/test_ingest.py @@ -0,0 +1,140 @@ +"""`refresh_source` tests. + +Upsert alone cannot reconcile: a document that cuts into fewer chunks than it +used to leaves the old tail behind, still searchable, holding text that is no +longer in the document. These tests are about that tail — that it is found, that +it is deleted before the new chunks land, and that a document which did not +shrink pays one lookup for the privilege and nothing more. + +The client is a fake. Everything under test is which calls get made and in what +order, which is exactly what a fake can answer and a live index cannot cheaply. +""" + +from __future__ import annotations + +import pytest +from moss import DocumentInfo +from moss_chunking import CharSplitter, chunk_document, chunk_id, refresh_source + +PROSE = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi" + + +class FakeClient: + """Enough of `MossClient` for the refresh path, recording what it was asked.""" + + def __init__(self, existing: list[str] | None = None) -> None: + self.existing = set(existing or []) + self.calls: list[tuple[str, object]] = [] + + async def get_docs(self, name, options=None): + self.calls.append(("get_docs", list(options.doc_ids))) + return [ + DocumentInfo(id=doc_id, text="stale") + for doc_id in options.doc_ids + if doc_id in self.existing + ] + + async def delete_docs(self, name, doc_ids): + self.calls.append(("delete_docs", list(doc_ids))) + self.existing -= set(doc_ids) + return None + + async def add_docs(self, name, docs, options=None): + self.calls.append(("add_docs", [d.id for d in docs])) + self.existing |= {d.id for d in docs} + return "added" + + +def ids_for(source: str, count: int) -> list[str]: + return [chunk_id(source, index) for index in range(count)] + + +def deleted(client: FakeClient) -> list[str]: + return [doc_id for kind, arg in client.calls if kind == "delete_docs" for doc_id in arg] + + +async def test_a_shrunken_document_loses_its_stale_tail(): + """The case upsert cannot handle: 21 chunks re-cut into 6.""" + client = FakeClient(existing=ids_for("notes.md", 21)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 6)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert deleted(client) == ids_for("notes.md", 21)[6:] + assert client.existing == set(ids_for("notes.md", 6)) + + +async def test_the_tail_is_deleted_before_the_new_chunks_land(): + """Ordering matters: an add that fails must not leave the stale tail behind.""" + client = FakeClient(existing=ids_for("notes.md", 9)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 2)] + + await refresh_source(client, "idx", "notes.md", docs) + + kinds = [kind for kind, _ in client.calls] + assert kinds.index("delete_docs") < kinds.index("add_docs") + + +async def test_a_document_that_did_not_shrink_deletes_nothing(): + client = FakeClient(existing=ids_for("notes.md", 4)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 4)] + + result = await refresh_source(client, "idx", "notes.md", docs) + + assert result == "added" + assert [kind for kind, _ in client.calls] == ["get_docs", "add_docs"] + + +async def test_a_grown_document_costs_a_single_lookup(): + """One probe past the end proves there is no tail; no scan of the index.""" + client = FakeClient(existing=ids_for("notes.md", 2)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 5)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert [kind for kind, _ in client.calls].count("get_docs") == 1 + + +async def test_passing_no_documents_removes_the_source_entirely(): + """How a deleted file leaves the index.""" + client = FakeClient(existing=ids_for("notes.md", 3) + ids_for("other.md", 2)) + + result = await refresh_source(client, "idx", "notes.md", []) + + assert result is None + assert client.existing == set(ids_for("other.md", 2)) + + +async def test_another_sources_chunks_are_never_touched(): + client = FakeClient(existing=ids_for("notes.md", 8) + ids_for("other.md", 8)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert set(ids_for("other.md", 8)) <= client.existing + assert not any(doc_id.startswith("other.md") for doc_id in deleted(client)) + + +async def test_a_tail_longer_than_one_probe_window_is_still_cleared(): + """The window is a batch size, not a ceiling: probing continues while it hits.""" + client = FakeClient(existing=ids_for("notes.md", 600)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert client.existing == set(ids_for("notes.md", 1)) + assert len(deleted(client)) == 599 + + +@pytest.mark.parametrize("cut", [40, 12]) +async def test_refresh_is_idempotent_for_real_chunked_output(cut): + """Twice through with the same text changes nothing the second time.""" + client = FakeClient() + docs = chunk_document(PROSE, "notes.md", CharSplitter(chunk_chars=cut, overlap=5)) + + await refresh_source(client, "idx", "notes.md", docs) + after_first = set(client.existing) + await refresh_source(client, "idx", "notes.md", docs) + + assert client.existing == after_first + assert not deleted(client) From a7b33059f5af0cdee8335e82cf87fd9c76a08b82 Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 16:10:18 +0530 Subject: [PATCH 11/14] =?UTF-8?q?fix:=20fourth=20bot=20round=20=E2=80=94?= =?UTF-8?q?=20recoverable=20deletes,=20waited=20jobs,=20async=20CI=20plugi?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete the stale tail highest ID first. Lowest-first punched a hole under the survivors of a failed batch: the next probe read the emptied low window as proof nothing was left and stopped, stranding every ID above it in the index with no run that would ever reach them. Deleting a suffix instead leaves a contiguous prefix, which is exactly what the next refresh walks. Wait on every deletion job. `delete_docs` returns when the job is accepted, not when it has run, so an unwaited deletion that failed afterwards was invisible — refresh_source returned a successful add while the stale chunks stayed searchable. Failures now raise rather than being reported as a refresh. Sort probe results before deleting, since `get_docs` promises no order. Install pytest-asyncio in the publish workflow's test job: the suite's async tests rely on asyncio_mode = "auto", and plain pytest fails them outright. Co-Authored-By: Claude Opus 5 --- .github/workflows/publish-moss-chunking.yml | 6 +- packages/moss-chunking/src/ingest.py | 43 +++++++- packages/moss-chunking/tests/test_ingest.py | 115 ++++++++++++++++++-- 3 files changed, 149 insertions(+), 15 deletions(-) diff --git a/.github/workflows/publish-moss-chunking.yml b/.github/workflows/publish-moss-chunking.yml index d8dbffe9..84f522ce 100644 --- a/.github/workflows/publish-moss-chunking.yml +++ b/.github/workflows/publish-moss-chunking.yml @@ -57,10 +57,14 @@ jobs: with: python-version: ${{ matrix.python }} + # `pytest-asyncio` is not optional here: the suite's async tests rely on + # the `asyncio_mode = "auto"` setting in pyproject.toml, and plain pytest + # fails them outright with "async def functions are not natively + # supported", which would block every release rather than skip a test. - name: Install build tooling run: | python -m pip install --upgrade pip - pip install build pytest + pip install build pytest pytest-asyncio - name: Build distributions working-directory: packages/moss-chunking diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py index 64620919..981e2851 100644 --- a/packages/moss-chunking/src/ingest.py +++ b/packages/moss-chunking/src/ingest.py @@ -65,6 +65,19 @@ async def refresh_source( chunk count, and looking up one window of candidate IDs past the end is enough to find it or prove it is not there. + That contiguity is the one assumption here, so it is also maintained here: + deletions run highest ID first and each one is waited on, which keeps what + survives a failure an unbroken run that the next refresh will find. A source + whose IDs were punched full of holes by something other than this package is + outside the guarantee — reconciling that would mean scanning the whole + `MAX_CHUNK_INDEX` space on every refresh, tens of round trips per document, + to defend against a writer that is not honouring the contract anyway. + + Raises whatever `wait_for_job` raises if a deletion fails, rather than + returning a result that implies a replacement which did not happen. Returns + the add's `MutationResult`, which the caller can wait on in turn — or `None` + when there was nothing to add. + Passing no documents deletes every chunk for `source`, which is how a deleted file is removed from the index. @@ -86,12 +99,30 @@ async def refresh_source( stale.extend(doc.id for doc in found) start += len(window) - if stale: - # Batched, because the stale run can be thousands of IDs long and a - # single request carrying all of them is a request that can fail all of - # them. - for offset in range(0, len(stale), _PROBE_WINDOW): - await client.delete_docs(index_name, stale[offset : offset + _PROBE_WINDOW]) + # `get_docs` does not promise an order, and the deletion below depends on + # one. Sorting the IDs is sorting by index, which is what the zero-padding + # in `chunk_id` is for. + stale.sort() + + # Batched, because the stale run can be thousands of IDs long and one + # request carrying all of them is one request that can fail all of them. + # + # Highest IDs first, which is what makes a failed batch recoverable. The + # survivors of a partial delete are then still one unbroken run from + # `len(docs)` upward — the shape the probe above relies on — so the next + # refresh finds them and finishes the job. Deleting lowest-first would punch + # a hole underneath them instead: the next probe would read the emptied low + # window as proof that nothing was left and stop, stranding every ID above + # it in the index, searchable, with no run that will ever reach them. + for end in range(len(stale), 0, -_PROBE_WINDOW): + batch = stale[max(end - _PROBE_WINDOW, 0) : end] + deletion = await client.delete_docs(index_name, batch) + # `delete_docs` returns when the job is accepted, not when it has run, + # so an unwaited deletion that fails afterwards is invisible: this + # function would return successfully with the stale chunks still + # searchable. The add below is handed back to the caller, who can wait + # on it; nobody outside this function ever sees these job IDs. + await client.wait_for_job(deletion.job_id) if not docs: return None diff --git a/packages/moss-chunking/tests/test_ingest.py b/packages/moss-chunking/tests/test_ingest.py index bcfdb82a..ed72a814 100644 --- a/packages/moss-chunking/tests/test_ingest.py +++ b/packages/moss-chunking/tests/test_ingest.py @@ -19,24 +19,53 @@ PROSE = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi" -class FakeClient: - """Enough of `MossClient` for the refresh path, recording what it was asked.""" +class FakeResult: + """`MutationResult` is a native type; only its `job_id` is used here.""" + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + - def __init__(self, existing: list[str] | None = None) -> None: +class FakeClient: + """Enough of `MossClient` for the refresh path, recording what it was asked. + + `fail_delete_after` fails the nth deletion the way the SDK does — after the + job has been accepted, when it is waited on — which is the case that decides + whether a half-finished refresh can be recovered by the next one. + """ + + def __init__( + self, + existing: list[str] | None = None, + fail_delete_after: int | None = None, + shuffled: bool = False, + ) -> None: self.existing = set(existing or []) self.calls: list[tuple[str, object]] = [] + self.fail_delete_after = fail_delete_after + self.shuffled = shuffled + self.deletions = 0 async def get_docs(self, name, options=None): self.calls.append(("get_docs", list(options.doc_ids))) - return [ - DocumentInfo(id=doc_id, text="stale") - for doc_id in options.doc_ids - if doc_id in self.existing - ] + found = [doc_id for doc_id in options.doc_ids if doc_id in self.existing] + if self.shuffled: + found.reverse() + return [DocumentInfo(id=doc_id, text="stale") for doc_id in found] async def delete_docs(self, name, doc_ids): self.calls.append(("delete_docs", list(doc_ids))) + self.deletions += 1 + if self.fail_delete_after is not None and self.deletions > self.fail_delete_after: + # Accepted, but the job will fail; the documents stay put. + return FakeResult(f"delete-{self.deletions}-doomed") self.existing -= set(doc_ids) + return FakeResult(f"delete-{self.deletions}") + + async def wait_for_job(self, job_id, **kwargs): + self.calls.append(("wait_for_job", job_id)) + if job_id.endswith("doomed"): + raise RuntimeError(f"job {job_id} failed") return None async def add_docs(self, name, docs, options=None): @@ -126,6 +155,76 @@ async def test_a_tail_longer_than_one_probe_window_is_still_cleared(): assert len(deleted(client)) == 599 +async def test_the_tail_is_deleted_from_the_top_down(): + """Descending order is what makes a half-finished refresh recoverable.""" + client = FakeClient(existing=ids_for("notes.md", 600)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + batches = [arg for kind, arg in client.calls if kind == "delete_docs"] + assert len(batches) > 1 + assert [batch[0] for batch in batches] == sorted((b[0] for b in batches), reverse=True) + + +async def test_every_deletion_is_waited_on(): + """`delete_docs` returns when the job is accepted, not when it has run.""" + client = FakeClient(existing=ids_for("notes.md", 600)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + kinds = [kind for kind, _ in client.calls] + for position, kind in enumerate(kinds): + if kind == "delete_docs": + assert kinds[position + 1] == "wait_for_job" + + +async def test_a_failed_deletion_is_raised_rather_than_reported_as_a_refresh(): + """Returning normally here would claim a replacement that did not happen.""" + client = FakeClient(existing=ids_for("notes.md", 9), fail_delete_after=0) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 2)] + + with pytest.raises(RuntimeError, match="failed"): + await refresh_source(client, "idx", "notes.md", docs) + + assert not any(kind == "add_docs" for kind, _ in client.calls) + + +async def test_a_refresh_after_a_failed_deletion_finishes_the_job(): + """The regression the delete order exists for. + + 600 chunks re-cut to 1, with the deletion failing partway. Whatever survives + has to stay reachable from a probe that starts at the new chunk count — if + the surviving run had a hole punched under it, the retry would stop at the + hole and strand everything above it forever. + """ + existing = ids_for("notes.md", 600) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + failing = FakeClient(existing=existing, fail_delete_after=1) + with pytest.raises(RuntimeError): + await refresh_source(failing, "idx", "notes.md", docs) + assert len(failing.existing) > 1 # the refresh really did leave work behind + + retry = FakeClient(existing=sorted(failing.existing)) + await refresh_source(retry, "idx", "notes.md", docs) + + assert retry.existing == set(ids_for("notes.md", 1)) + + +async def test_unordered_lookup_results_do_not_break_the_delete_order(): + """`get_docs` promises no order; the zero-padded IDs are sorted before use.""" + client = FakeClient(existing=ids_for("notes.md", 600), shuffled=True) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + batches = [arg for kind, arg in client.calls if kind == "delete_docs"] + assert [batch[0] for batch in batches] == sorted((b[0] for b in batches), reverse=True) + assert client.existing == set(ids_for("notes.md", 1)) + + @pytest.mark.parametrize("cut", [40, 12]) async def test_refresh_is_idempotent_for_real_chunked_output(cut): """Twice through with the same text changes nothing the second time.""" From 0edd868723b3e71bb20b40d9928cf02e510793dc Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 16:19:36 +0530 Subject: [PATCH 12/14] test: assert the SDK surface refresh_source calls actually exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fake client in these tests cannot catch a method the real one lacks, and `wait_for_job` reads as missing from both the checked-in stub and the in-tree SDK source (1.0.0b19, old enough to predate it). It is present on the declared floor, moss==1.7.2, which is the surface that decides — so assert against the installed client rather than leaving the question to whoever reads the stub. Co-Authored-By: Claude Opus 5 --- packages/moss-chunking/tests/test_ingest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/moss-chunking/tests/test_ingest.py b/packages/moss-chunking/tests/test_ingest.py index ed72a814..4ca40f87 100644 --- a/packages/moss-chunking/tests/test_ingest.py +++ b/packages/moss-chunking/tests/test_ingest.py @@ -155,6 +155,25 @@ async def test_a_tail_longer_than_one_probe_window_is_still_cleared(): assert len(deleted(client)) == 599 +def test_the_client_surface_this_module_calls_actually_exists(): + """The fake client above cannot catch a method that the real one lacks. + + `wait_for_job` in particular is real on the declared floor (`moss==1.7.2`) + but missing from both the checked-in stub and the in-tree SDK source, which + is old enough to predate it — so reading either one suggests this module + calls something that isn't there. This asserts against the installed SDK, + which is the surface that decides. + """ + from moss import MossClient + + missing = [ + name + for name in ("get_docs", "delete_docs", "add_docs", "wait_for_job") + if not hasattr(MossClient, name) + ] + assert not missing + + async def test_the_tail_is_deleted_from_the_top_down(): """Descending order is what makes a half-finished refresh recoverable.""" client = FakeClient(existing=ids_for("notes.md", 600)) From 551d4b6219f795d53200b67ee7cd84465cf767cc Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 16:23:23 +0530 Subject: [PATCH 13/14] fix: refuse documents that are not the source's own cut MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refresh_source reads len(documents) as the first index the source no longer uses. That is only true when the documents are that source's entire cut, so another source's documents — or a filtered slice of this one's — would delete live chunks and then add documents belonging to something else. IDs are now checked against chunk_id(source, position) before anything is deleted. Co-Authored-By: Claude Opus 5 --- packages/moss-chunking/src/ingest.py | 20 +++++++++++++++++ packages/moss-chunking/tests/test_ingest.py | 24 +++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py index 981e2851..ce7442ae 100644 --- a/packages/moss-chunking/src/ingest.py +++ b/packages/moss-chunking/src/ingest.py @@ -78,6 +78,11 @@ async def refresh_source( the add's `MutationResult`, which the caller can wait on in turn — or `None` when there was nothing to add. + `documents` must be `source`'s entire cut, in order — `chunk_document`'s + output for that source and nothing else. A mismatched ID raises before + anything is deleted, since the whole reconciliation is arithmetic on + `len(documents)` and the wrong list would delete live chunks. + Passing no documents deletes every chunk for `source`, which is how a deleted file is removed from the index. @@ -86,6 +91,21 @@ async def refresh_source( """ docs = list(documents) + # Everything below reads `len(docs)` as "the first index this source no + # longer uses", which is only true if these documents really are this + # source's whole cut. Handed another source's documents, or a filtered slice + # of this one's, that arithmetic would delete live chunks and then add + # documents that do not belong to `source` — a destructive way to discover a + # mistaken argument. `chunk_document` output passes this by construction. + for position, doc in enumerate(docs): + expected = chunk_id(source, position) + if doc.id != expected: + raise ValueError( + f"documents[{position}] has id {doc.id!r}, expected {expected!r}: " + f"refresh_source replaces everything under {source!r}, so it needs " + "that source's chunks, all of them, in cut order" + ) + stale: list[str] = [] start = len(docs) while start <= MAX_CHUNK_INDEX: diff --git a/packages/moss-chunking/tests/test_ingest.py b/packages/moss-chunking/tests/test_ingest.py index 4ca40f87..c1f3f0e2 100644 --- a/packages/moss-chunking/tests/test_ingest.py +++ b/packages/moss-chunking/tests/test_ingest.py @@ -155,6 +155,30 @@ async def test_a_tail_longer_than_one_probe_window_is_still_cleared(): assert len(deleted(client)) == 599 +async def test_documents_from_another_source_are_refused_before_anything_is_deleted(): + """The wrong argument would otherwise be discovered destructively.""" + client = FakeClient(existing=ids_for("notes.md", 5)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("other.md", 2)] + + with pytest.raises(ValueError, match="expected 'notes.md#chunk-0000'"): + await refresh_source(client, "idx", "notes.md", docs) + + assert client.calls == [] + assert client.existing == set(ids_for("notes.md", 5)) + + +async def test_a_gapped_document_list_is_refused(): + """A filtered list would make `len(docs)` mean the wrong index.""" + client = FakeClient(existing=ids_for("notes.md", 5)) + kept = [chunk_id("notes.md", 0), chunk_id("notes.md", 2)] + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in kept] + + with pytest.raises(ValueError, match=r"documents\[1\]"): + await refresh_source(client, "idx", "notes.md", docs) + + assert client.calls == [] + + def test_the_client_surface_this_module_calls_actually_exists(): """The fake client above cannot catch a method that the real one lacks. From 76390b4723978d2a2d5e40ae4f58b8e02e83673a Mon Sep 17 00:00:00 2001 From: Aditya Chawla Date: Wed, 5 Aug 2026 16:39:55 +0530 Subject: [PATCH 14/14] =?UTF-8?q?fix:=20fifth=20bot=20round=20=E2=80=94=20?= =?UTF-8?q?locator=20typing,=20render-time=20key=20check,=20concurrency=20?= =?UTF-8?q?note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Locators get the same treatment as the index: a float or bool passed the range checks and rendered as "1.5" or "True" in metadata meant to hold a real offset. `_require_whole_number` is now shared by both. Re-check extra's keys in to_document. `extra` stays a mutable dict — that is what keeps a chunk picklable and copyable — so a key added after construction reached DocumentInfo unvalidated. The check now runs where metadata is built, not only where it was set. Document that refresh_source is not atomic. Concurrent refreshes of one source can interleave; the index is the shared resource and no client-side lock makes probe/delete/add one operation. The delete ordering does bound the damage to ordinary stale tail, which the next refresh clears. Co-Authored-By: Claude Opus 5 --- packages/moss-chunking/src/chunk.py | 64 +++++++++++++++------- packages/moss-chunking/src/ingest.py | 9 +++ packages/moss-chunking/tests/test_chunk.py | 28 ++++++++++ 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py index 1a5b0eb1..a3d1d2ef 100644 --- a/packages/moss-chunking/src/chunk.py +++ b/packages/moss-chunking/src/chunk.py @@ -37,18 +37,38 @@ MAX_CHUNK_INDEX = 9999 -def _require_index(index: int) -> None: - """Reject anything that is not a plain in-range int. +def _require_whole_number(name: str, value: int) -> None: + """Reject anything that is not a plain non-negative int. `bool` is excluded explicitly because it subclasses `int`, so `True` would - otherwise format as a perfectly valid `chunk-0001`. A float gets caught here - rather than at the `:04d` format, which fails with `Unknown format code 'd'` - a long way from whatever set the index. + otherwise pass as the number 1 — a locator of `True` rendering as `"True"`, + an index of `True` formatting as a perfectly valid `chunk-0001`. A float is + caught here rather than at the `:04d` format, which fails with `Unknown + format code 'd'` a long way from whatever set the value. + """ + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError(f"{name} must be an int, got {type(value).__name__}") + if value < 0: + raise ValueError(f"{name} must be >= 0, got {value}") + + +def _require_string_keys(extra: dict[str, object]) -> None: + """Reject non-string keys in `extra`. + + Values are coerced at render because `{"page": 3}` is the natural thing to + write and `"3"` is unambiguously what was meant. Keys are not symmetric with + that: `str(1)` and `"1"` are the same metadata key, so coercing would let one + entry silently overwrite another. A non-string key also cannot collide with a + reserved one, so it would slip past that check and fail at the SDK boundary. """ - if not isinstance(index, int) or isinstance(index, bool): - raise TypeError(f"index must be an int, got {type(index).__name__}") - if index < 0: - raise ValueError(f"index must be >= 0, got {index}") + non_string = sorted(repr(key) for key in extra if not isinstance(key, str)) + if non_string: + raise TypeError(f"extra keys must be str, got {', '.join(non_string)}") + + +def _require_index(index: int) -> None: + """Reject an index that could not render a sortable ID.""" + _require_whole_number("index", index) if index > MAX_CHUNK_INDEX: raise ValueError( f"index must be <= {MAX_CHUNK_INDEX}, got {index}: past that the " @@ -112,20 +132,17 @@ def __post_init__(self) -> None: raise ValueError( f"locator_type must be one of {LOCATOR_TYPES}, got {self.locator_type!r}" ) - if self.locator_start < 0: - raise ValueError(f"locator_start must be >= 0, got {self.locator_start}") + # Locators are positions, so they get the same treatment as the index: a + # float or a bool here renders as `"1.5"` or `"True"` in metadata that is + # supposed to hold a real offset, and nothing downstream can tell the + # difference between that and a position the splitter meant. + _require_whole_number("locator_start", self.locator_start) + _require_whole_number("locator_end", self.locator_end) if self.locator_end < self.locator_start: raise ValueError( f"locator_end ({self.locator_end}) must be >= locator_start ({self.locator_start})" ) - # Values are coerced at render because `{"page": 3}` is the natural thing - # to pass and `"3"` is unambiguously what was meant. Keys are rejected - # instead: `str(1)` and `"1"` are the same metadata key, so coercing them - # would let one entry silently overwrite another, and a non-string key - # cannot collide with a reserved one and so slips past the check below. - non_string = sorted(repr(key) for key in self.extra if not isinstance(key, str)) - if non_string: - raise TypeError(f"extra keys must be str, got {', '.join(non_string)}") + _require_string_keys(self.extra) clashes = RESERVED_KEYS & self.extra.keys() if clashes: raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") @@ -141,8 +158,12 @@ def to_document(self, source: str) -> DocumentInfo: Every metadata value is stringified because Moss types metadata as `Dict[str, str]`. An int left in there would fail at the SDK boundary, - which is a worse place to discover it than here. Keys are already known - to be strings — the constructor rejects any that are not. + which is a worse place to discover it than here. + + Keys are re-checked rather than trusted. `extra` stays a mutable dict — + that is what keeps a chunk picklable and copyable — so `chunk.extra[1] = + "one"` after construction is possible, and the constructor's verdict is + only ever a statement about the moment it ran. `extra` is merged *first* so the contract's own keys always win. The constructor already rejects a clashing `extra`, so this only matters if @@ -150,6 +171,7 @@ def to_document(self, source: str) -> DocumentInfo: that these five keys mean the same thing on every chunk, and a render step is the last place that can still guarantee it. """ + _require_string_keys(self.extra) return DocumentInfo( id=chunk_id(source, self.index), text=self.text, diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py index ce7442ae..5290d284 100644 --- a/packages/moss-chunking/src/ingest.py +++ b/packages/moss-chunking/src/ingest.py @@ -83,6 +83,15 @@ async def refresh_source( anything is deleted, since the whole reconciliation is arithmetic on `len(documents)` and the wrong list would delete live chunks. + **Not atomic, and one source at a time.** Two refreshes of the same source + running concurrently can interleave — a short cut deletes the tail, a long + cut adds it back, and the short cut's add lands last, leaving chunks from a + document that no longer exists. Serialize refreshes per source; the index is + the shared resource and nothing client-side can make probe/delete/add one + operation. What the ordering above does guarantee is that the wreckage is + ordinary stale tail, contiguous above the surviving cut, so the next refresh + of that source clears it. + Passing no documents deletes every chunk for `source`, which is how a deleted file is removed from the index. diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py index ad5d58f6..a42e96e6 100644 --- a/packages/moss-chunking/tests/test_chunk.py +++ b/packages/moss-chunking/tests/test_chunk.py @@ -104,6 +104,20 @@ def test_reserved_keys_win_at_render_even_if_validation_is_bypassed(): assert doc.metadata["chunk_index"] == "0" +def test_a_key_added_after_construction_is_still_caught_at_render(): + """`extra` stays mutable, so the constructor's verdict has a shelf life. + + Keeping it a plain dict is what makes a chunk picklable and copyable; the + cost is that validation has to be re-run where the metadata is actually + built, rather than trusted from construction time. + """ + chunk = Chunk("body", 0, "char", 0, 4) + chunk.extra[1] = "one" + + with pytest.raises(TypeError, match="extra keys must be str"): + chunk.to_document("notes.md") + + def test_extra_is_copied_so_the_caller_cannot_mutate_validated_state(): supplied = {"extension": "md"} chunk = Chunk("body", 0, "char", 0, 4, extra=supplied) @@ -170,6 +184,20 @@ def test_unknown_locator_type_is_rejected(): Chunk("body", 0, "byte", 0, 4) # type: ignore[arg-type] +@pytest.mark.parametrize("bad", [1.5, True, False, "0", None]) +@pytest.mark.parametrize("field", ["locator_start", "locator_end"]) +def test_a_non_integer_locator_is_rejected(field, bad): + """A locator is a position, so it gets the same treatment as the index. + + Left alone, `locator_start=1.5` renders as the metadata string `"1.5"` and + `True` renders as `"True"` — neither distinguishable downstream from an + offset the splitter meant. + """ + kwargs = {"locator_start": 0, "locator_end": 4, field: bad} + with pytest.raises(TypeError, match=f"{field} must be an int"): + Chunk("body", 0, "char", **kwargs) + + def test_backwards_locator_is_rejected(): with pytest.raises(ValueError, match="locator_end"): Chunk("body", 0, "char", 20, 10)