diff --git a/.github/workflows/publish-moss-chunking.yml b/.github/workflows/publish-moss-chunking.yml new file mode 100644 index 00000000..84f522ce --- /dev/null +++ b/.github/workflows/publish-moss-chunking.yml @@ -0,0 +1,157 @@ +name: Publish moss-chunking + +permissions: + contents: read + +on: + workflow_dispatch: + +concurrency: + group: moss-chunking-release + cancel-in-progress: false + +jobs: + determine-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.compute.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Read version from pyproject + id: compute + shell: python + run: | + import os, pathlib, sys, tomllib + + raw = pathlib.Path("packages/moss-chunking/pyproject.toml").read_bytes() + version = tomllib.loads(raw.decode("utf-8")).get("project", {}).get("version") + if not version: + print("Could not find project.version in pyproject", file=sys.stderr) + sys.exit(1) + + out = pathlib.Path(os.environ["GITHUB_OUTPUT"]) + with out.open("a", encoding="utf-8") as fh: + fh.write(f"version={version}\n") + + print(f"Publishing version: {version}") + + # Build + smoke-test on every supported Python version. Publishing waits for + # the whole matrix (see the publish job's `needs`), so a failure on any + # version blocks the upload/tag instead of racing it. + build-test: + needs: determine-version + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + # `pytest-asyncio` is not optional here: the suite's async tests rely on + # the `asyncio_mode = "auto"` setting in pyproject.toml, and plain pytest + # fails them outright with "async def functions are not natively + # supported", which would block every release rather than skip a test. + - name: Install build tooling + run: | + python -m pip install --upgrade pip + pip install build pytest pytest-asyncio + + - name: Build distributions + working-directory: packages/moss-chunking + run: | + rm -rf dist + python -m build + + - name: Install the built wheel + shell: python + run: | + import glob, subprocess, sys + + wheels = glob.glob("packages/moss-chunking/dist/*.whl") + if not wheels: + raise SystemExit("No wheel found in dist/") + + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--force-reinstall", wheels[0]] + ) + + - name: Smoke test the public surface + shell: python + run: | + import importlib + + module = importlib.import_module("moss_chunking") + missing = [name for name in module.__all__ if not hasattr(module, name)] + if missing: + raise SystemExit(f"__all__ names missing from the built wheel: {missing}") + print(f"import ok; {len(module.__all__)} public names resolve") + + # Run the suite against the installed wheel, not the source tree. A wheel + # that imports cleanly can still ship broken offsets or metadata, and that + # is precisely what these tests assert. + - name: Test the installed wheel + run: python -m pytest -q packages/moss-chunking/tests + + # Only runs once every build-test matrix leg has passed. + publish: + needs: [determine-version, build-test] + # This job holds PYPI_API_TOKEN and pushes a release tag. `workflow_dispatch` + # accepts any ref, so without this guard an unmerged branch could be + # published under the package name and tagged as a release. + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + env: + VERSION: ${{ needs.determine-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Fail if this version was already released + run: | + git fetch --tags --force + # Match the tag ref exactly: `git rev-parse` alone resolves anything + # that happens to parse as a revision, not just a tag by this name. + if git rev-parse --verify "refs/tags/moss-chunking-v${VERSION}" >/dev/null 2>&1; then + echo "::error::moss-chunking v${VERSION} is already tagged/released; bump the version in pyproject.toml before releasing." + exit 1 + fi + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build distributions + working-directory: packages/moss-chunking + run: | + rm -rf dist + python -m build + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + twine upload packages/moss-chunking/dist/* + + - name: Tag release + run: | + git config user.name "github-actions" + git config user.email "github-actions@users.noreply.github.com" + git tag "moss-chunking-v${VERSION}" + git push origin "moss-chunking-v${VERSION}" diff --git a/packages/moss-chunking/.gitignore b/packages/moss-chunking/.gitignore new file mode 100644 index 00000000..b86b0644 --- /dev/null +++ b/packages/moss-chunking/.gitignore @@ -0,0 +1,10 @@ +build/ +dist/ +*.egg-info/ +__pycache__/ +*.py[cod] +.venv/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.env diff --git a/packages/moss-chunking/README.md b/packages/moss-chunking/README.md new file mode 100644 index 00000000..aa77c5ff --- /dev/null +++ b/packages/moss-chunking/README.md @@ -0,0 +1,165 @@ +# moss-chunking + +Pluggable text chunking for Moss. Splitters are swappable; the shape of what they +emit is not. + +## Why this exists + +Chunking isn't in any Moss SDK, so every app rolls its own. Two of them, in this +repo, both already wrap chunks in the SDK's `DocumentInfo` — and still can't be +treated uniformly, because they agree on nothing inside it: + +| | `examples/moss-pikachu` | `apps/moss-llamaindex` | +| --- | --- | --- | +| id | `{path}#chunk-0001` | `{filename}-p{page}-c{idx}` | +| metadata | `path`, `filename`, `chunk`, `extension`, `modified_at` | `source`, `page` | +| split | 1800 chars / 300 overlap | 400 words / 2-sentence overlap | + +Zero metadata keys in common. The envelope is shared; the contract is missing. + +Splitting strategy is genuinely contested and content-dependent — code, markdown +and transcripts all want different cuts — which is why it stays pluggable, and +why this is a standalone package rather than something frozen into five language +runtimes. But the *output* isn't contested. Nobody wants a bespoke ID scheme; +they wrote one because none was written down. + +So this package pins the output and leaves the cutting open. + +## Install + +```bash +uv pip install -e ".[dev]" # from packages/moss-chunking +``` + +Depends only on `moss`. Sentence detection is regex-based rather than nltk-backed +so there is no model download or corpus to provision. + +## Use + +```python +from moss_chunking import SentenceSplitter, chunk_document, ingest + +docs = chunk_document(text, source="notes.md", strategy=SentenceSplitter()) +await ingest(docs, project_id, project_key, "my-index") +``` + +`docs` are ordinary `DocumentInfo`s — they go anywhere the SDK takes documents; +`ingest` is just the connector template's one-call shortcut into a fresh index. + +To re-chunk one document inside an index that already exists, use +`refresh_source` rather than `add_docs`: + +```python +await refresh_source(client, "my-index", "notes.md", docs) +``` + +`add_docs` upserts, which replaces the chunks the new cut still produces — but a +document that shrinks from 21 chunks to 6 leaves `#chunk-0006` through +`#chunk-0020` in the index, still searchable, holding text the document no longer +contains. `refresh_source` deletes that tail first. Passing no documents removes +the source entirely, which is how a deleted file leaves the index. + +## The contract + +Every chunk, from every strategy, carries: + +| key | meaning | +| --- | --- | +| `source` | what was chunked — path, URL, document name | +| `chunk_index` | position in the sequence, from `0` | +| `locator_type` | `char`, `line` or `page` | +| `locator_start` / `locator_end` | position, in that unit | + +IDs are `{source}#chunk-{index:04d}` — zero-padded so they sort in cut order, and +stable across runs so re-chunking an unchanged document replaces its chunks +rather than duplicating them. Indices run `0, 1, 2, …`; `chunk_document` rejects a +strategy that skips or repeats one, because a repeat renders the same ID twice and +the second chunk silently overwrites the first. Values are all strings, because +Moss types metadata as `Dict[str, str]`. + +The sort only holds while the padding is fixed width, so `chunk_id` rejects an +index above `MAX_CHUNK_INDEX` (9999) rather than emitting `chunk-10000`, which +sorts before `chunk-9999`. A document that cuts into more than 10,000 chunks +wants a coarser strategy or a per-section `source`. + +That stability is only worth something if you keep it on the way in, which is why +`ingest` drops the one option the connector template it mirrors does offer: +`auto_id`. Random UUIDs defeat the contract — re-indexing an unchanged document +appends a second copy of every chunk instead of replacing what is already there. + +**Position is not a fixed field list.** Plain text is located by character offset, +PDFs by page, code by line; character offsets are meaningless for a PDF and page +numbers are meaningless for a source file. So a chunk declares its unit instead of +assuming one. That's the one real design call in the package. + +Pass source-level facts a splitter can't know via `extra`: + +```python +chunk_document(text, "notes.md", CharSplitter(), extra={"extension": "md", "page": 3}) +``` + +Values are stringified on the way out, so passing an `int` is fine — Moss types +metadata as `Dict[str, str]`, and coercing here beats failing at the SDK +boundary. `extra` cannot shadow the reserved keys above: that's an error, not a +silent overwrite, and the reserved keys win at render even if one is added to +`extra` afterwards. + +## Strategies + +| | cuts on | ceiling | notes | +| --- | --- | --- | --- | +| `CharSplitter` | fixed character windows | hard | pikachu's 1800/300 are the defaults | +| `SentenceSplitter` | sentence boundaries | soft | llamaindex's 400 words / 2 sentences are the defaults | +| `ParagraphSplitter` | blank lines | soft | never breaks a paragraph, even an oversized one | +| `RecursiveSplitter` | paragraphs → lines → sentences → words | hard | falls back to a hard cut if nothing fits | + +Soft ceiling means a single unit larger than the budget is emitted whole rather +than cut. `RecursiveSplitter` is the one to reach for when the ceiling must hold. + +Write your own by implementing `split(text) -> Iterable[Chunk]`: + +```python +from collections.abc import Iterator + +from moss_chunking import Chunk + + +class MyStrategy: + def split(self, text: str) -> Iterator[Chunk]: + yield Chunk(text=..., index=..., locator_type="line", locator_start=..., locator_end=...) +``` + +A strategy never builds a `DocumentInfo` — that's the contract's job, and handing +it to callers is exactly how pikachu and llamaindex drifted apart. + +The invariant the tests enforce for every strategy: +`text[chunk.locator_start:chunk.locator_end] == chunk.text`. Offsets point into +the original string, never a normalized copy. Break it and position metadata +becomes decorative — you can address a chunk but not find it again. + +Semantic chunking is not here yet: it needs embeddings, which makes it a +different shape of dependency. It's the obvious next strategy. + +## Enrichment + +Moss scores BM25 over chunk text, so facts that live only in metadata — the +filename, the folder — are invisible to the keyword half of a hybrid query. +Restating them in the text makes them matchable. Pikachu already does this by +hand; here it's a composable post-step that works with any strategy: + +```python +from moss_chunking import prepend_source_context + +doc = prepend_source_context(doc, filename="notes.md", path="/docs/notes.md") +``` + +ID and metadata are untouched, so an enriched chunk stays addressable. Any +embedding is dropped: it was computed from the text this rewrites, and a vector +that no longer describes its chunk skews the dense half of every hybrid query +without ever announcing itself. Enrich first, embed after. + +## Tests + +```bash +.venv/bin/python -m pytest -q +``` diff --git a/packages/moss-chunking/pyproject.toml b/packages/moss-chunking/pyproject.toml new file mode 100644 index 00000000..8784b624 --- /dev/null +++ b/packages/moss-chunking/pyproject.toml @@ -0,0 +1,60 @@ +[project] +name = "moss-chunking" +version = "0.0.1" +description = "Pluggable text chunking strategies that emit Moss DocumentInfo under a shared ID and metadata contract." +readme = "README.md" +requires-python = ">=3.10,<3.15" +license = { text = "BSD-2-Clause" } +authors = [{ name = "InferEdge Inc.", email = "contact@moss.dev" }] +keywords = ["moss", "chunking", "splitter", "retrieval", "rag"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Text Processing :: Linguistic", +] +# Most packages here declare `moss>=1.1.1`, but that floor cannot resolve: every +# moss release before 1.7.2 pins an `inferedge-moss-core` that is no longer on +# PyPI. 1.7.2 is the earliest version that actually installs, and the earliest +# whose `DocumentInfo` is verified to accept `payload`. +dependencies = [ + "moss>=1.7.2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "python-dotenv>=1.0.0", + "ruff>=0.5.0", +] + +[project.urls] +Homepage = "https://github.com/usemoss/moss" +Repository = "https://github.com/usemoss/moss" +Source = "https://github.com/usemoss/moss/tree/main/packages/moss-chunking" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +# Flat layout: src/ itself IS the package. +[tool.setuptools] +packages = ["moss_chunking"] +package-dir = { "moss_chunking" = "src" } + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "B", "UP"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/packages/moss-chunking/src/__init__.py b/packages/moss-chunking/src/__init__.py new file mode 100644 index 00000000..c01639de --- /dev/null +++ b/packages/moss-chunking/src/__init__.py @@ -0,0 +1,50 @@ +"""Pluggable text chunking for Moss. + +Splitters are swappable; the shape of what they emit is not. Every strategy in +here yields chunks that carry a stable ID, declared position metadata, and the +SDK's own `DocumentInfo` — see `chunk.py` for why that contract is the point of +the package. + + from moss_chunking import SentenceSplitter, chunk_document, ingest + + docs = chunk_document(text, source="notes.md", strategy=SentenceSplitter()) + await ingest(docs, project_id, project_key, "my-index") +""" + +from .chunk import ( + LOCATOR_TYPES, + MAX_CHUNK_INDEX, + RESERVED_KEYS, + Chunk, + LocatorType, + chunk_id, +) +from .enrich import prepend_context, prepend_source_context +from .ingest import ingest, refresh_source +from .strategies import ( + CharSplitter, + ChunkingStrategy, + ParagraphSplitter, + RecursiveSplitter, + SentenceSplitter, + chunk_document, +) + +__all__ = [ + "LOCATOR_TYPES", + "MAX_CHUNK_INDEX", + "RESERVED_KEYS", + "CharSplitter", + "Chunk", + "ChunkingStrategy", + "LocatorType", + "ParagraphSplitter", + "RecursiveSplitter", + "SentenceSplitter", + "chunk_document", + "chunk_id", + "ingest", + "prepend_context", + "prepend_source_context", + "refresh_source", +] diff --git a/packages/moss-chunking/src/chunk.py b/packages/moss-chunking/src/chunk.py new file mode 100644 index 00000000..a3d1d2ef --- /dev/null +++ b/packages/moss-chunking/src/chunk.py @@ -0,0 +1,186 @@ +"""The chunk contract: stable IDs, position metadata, and Moss `DocumentInfo`. + +Splitters disagree about *how* to cut text, and that is fine — they are meant to +be swappable. What they must not disagree about is what a chunk looks like once +it comes out. Today they do: `moss-pikachu` emits `{path}#chunk-0001` with +path/filename/chunk/extension/modified_at, while `moss-llamaindex` emits +`{filename}-p{page}-c{idx}` with source/page. Both already wrap chunks in +`DocumentInfo`, so the envelope is shared — but they have no metadata key in +common, so nothing downstream can treat their chunks uniformly. + +This module pins down the inside of that envelope. + +Position is deliberately *not* a fixed field list. Plain text is located by +character offset, paginated documents by page, code by line; character offsets +are meaningless for a PDF and page numbers are meaningless for a source file. So +every chunk carries a locator whose unit is declared rather than assumed. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal, get_args + +from moss import DocumentInfo + +#: Unit a chunk's position is measured in. +LocatorType = Literal["char", "line", "page"] + +LOCATOR_TYPES: tuple[str, ...] = get_args(LocatorType) + +#: Metadata keys the contract owns. `Chunk.extra` may not shadow them. +RESERVED_KEYS = frozenset({"source", "chunk_index", "locator_type", "locator_start", "locator_end"}) + +#: Highest index the ID format can hold. Four digits is pikachu's width, kept for +#: parity; past it the padding stops being fixed-width and `chunk-10000` sorts +#: before `chunk-9999`, which is the one promise the format makes. +MAX_CHUNK_INDEX = 9999 + + +def _require_whole_number(name: str, value: int) -> None: + """Reject anything that is not a plain non-negative int. + + `bool` is excluded explicitly because it subclasses `int`, so `True` would + otherwise pass as the number 1 — a locator of `True` rendering as `"True"`, + an index of `True` formatting as a perfectly valid `chunk-0001`. A float is + caught here rather than at the `:04d` format, which fails with `Unknown + format code 'd'` a long way from whatever set the value. + """ + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError(f"{name} must be an int, got {type(value).__name__}") + if value < 0: + raise ValueError(f"{name} must be >= 0, got {value}") + + +def _require_string_keys(extra: dict[str, object]) -> None: + """Reject non-string keys in `extra`. + + Values are coerced at render because `{"page": 3}` is the natural thing to + write and `"3"` is unambiguously what was meant. Keys are not symmetric with + that: `str(1)` and `"1"` are the same metadata key, so coercing would let one + entry silently overwrite another. A non-string key also cannot collide with a + reserved one, so it would slip past that check and fail at the SDK boundary. + """ + non_string = sorted(repr(key) for key in extra if not isinstance(key, str)) + if non_string: + raise TypeError(f"extra keys must be str, got {', '.join(non_string)}") + + +def _require_index(index: int) -> None: + """Reject an index that could not render a sortable ID.""" + _require_whole_number("index", index) + if index > MAX_CHUNK_INDEX: + raise ValueError( + f"index must be <= {MAX_CHUNK_INDEX}, got {index}: past that the " + "zero-padding is no longer fixed width and IDs stop sorting in cut order" + ) + + +def chunk_id(source: str, index: int) -> str: + """Build a chunk's stable ID. + + Zero-padded so IDs sort lexicographically in the order they were cut, which + is what pikachu arrived at independently. `source` is whatever identifies the + original — a file path, a URL, a document name — and must be stable across + runs: re-chunking an unchanged document has to reproduce the same IDs, or + every chunk gets re-added instead of replaced. + + Rejects indices above `MAX_CHUNK_INDEX` rather than emitting an ID that + breaks the sort. A document that cuts into more than 10,000 chunks wants a + coarser strategy or a per-section `source`, not a silently unsortable ID. + """ + if not isinstance(source, str): + raise TypeError(f"source must be a str, got {type(source).__name__}") + if not source: + raise ValueError("source must be a non-empty string") + _require_index(index) + return f"{source}#chunk-{index:04d}" + + +@dataclass(frozen=True) +class Chunk: + """One cut of a document, before it becomes a `DocumentInfo`. + + Splitters yield these. Positions stay real integers here; the conversion to + Moss's string-only metadata happens once, in `to_document`. + """ + + text: str + index: int + locator_type: LocatorType + locator_start: int + locator_end: int + #: Values are stringified at render, so the natural thing to pass — `{"page": + #: 3}`, `{"words": 12}` — is accepted rather than failing at the SDK boundary. + #: Keys must already be strings; see `__post_init__` for why they are not + #: coerced the same way. + #: + #: Copied on construction, so a caller's handle cannot reach back in. + #: + #: Excluded from equality and hashing: a chunk *is* its text and position, + #: and metadata about it does not change which cut of the document it is. + #: That is also what keeps it safe to leave mutable — nothing here can make a + #: chunk already sitting in a set change what it equals, whether by rebinding + #: a key or by mutating a list held as a value. + extra: dict[str, object] = field(default_factory=dict, compare=False) + + def __post_init__(self) -> None: + # The same check `chunk_id` runs, so a chunk that could never render a + # valid ID is rejected where its index was set rather than mid-render. + _require_index(self.index) + if self.locator_type not in LOCATOR_TYPES: + raise ValueError( + f"locator_type must be one of {LOCATOR_TYPES}, got {self.locator_type!r}" + ) + # Locators are positions, so they get the same treatment as the index: a + # float or a bool here renders as `"1.5"` or `"True"` in metadata that is + # supposed to hold a real offset, and nothing downstream can tell the + # difference between that and a position the splitter meant. + _require_whole_number("locator_start", self.locator_start) + _require_whole_number("locator_end", self.locator_end) + if self.locator_end < self.locator_start: + raise ValueError( + f"locator_end ({self.locator_end}) must be >= locator_start ({self.locator_start})" + ) + _require_string_keys(self.extra) + clashes = RESERVED_KEYS & self.extra.keys() + if clashes: + raise ValueError(f"extra may not override reserved keys: {sorted(clashes)}") + # `frozen=True` freezes the field, not the dict behind it. Copy so the + # caller's handle cannot add a reserved key after the check has passed. + # A plain dict rather than a read-only view, because a `MappingProxyType` + # is not picklable and takes `deepcopy` and `dataclasses.asdict` down + # with it. Excluding `extra` from equality is what makes the copy enough. + object.__setattr__(self, "extra", dict(self.extra)) + + def to_document(self, source: str) -> DocumentInfo: + """Render this chunk as a Moss `DocumentInfo`. + + Every metadata value is stringified because Moss types metadata as + `Dict[str, str]`. An int left in there would fail at the SDK boundary, + which is a worse place to discover it than here. + + Keys are re-checked rather than trusted. `extra` stays a mutable dict — + that is what keeps a chunk picklable and copyable — so `chunk.extra[1] = + "one"` after construction is possible, and the constructor's verdict is + only ever a statement about the moment it ran. + + `extra` is merged *first* so the contract's own keys always win. The + constructor already rejects a clashing `extra`, so this only matters if + one was introduced afterwards — but the whole point of the contract is + that these five keys mean the same thing on every chunk, and a render + step is the last place that can still guarantee it. + """ + _require_string_keys(self.extra) + return DocumentInfo( + id=chunk_id(source, self.index), + text=self.text, + metadata={ + **{key: str(value) for key, value in self.extra.items()}, + "source": source, + "chunk_index": str(self.index), + "locator_type": self.locator_type, + "locator_start": str(self.locator_start), + "locator_end": str(self.locator_end), + }, + ) diff --git a/packages/moss-chunking/src/enrich.py b/packages/moss-chunking/src/enrich.py new file mode 100644 index 00000000..99b2c619 --- /dev/null +++ b/packages/moss-chunking/src/enrich.py @@ -0,0 +1,58 @@ +"""Optional post-steps that rewrite a chunk's text after it is cut. + +These are retrieval-quality tricks, not splitting tricks, so they live outside +`ChunkingStrategy` and compose with any of them. Keeping them separate is also +what makes them adoptable: pikachu already does the trick below by hand, and it +would keep its own copy if the only way to get it were to switch splitters. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +from moss import DocumentInfo + + +def prepend_context(doc: DocumentInfo, fields: Mapping[str, str]) -> DocumentInfo: + """Return `doc` with `fields` written into the top of its text. + + Moss's hybrid search scores BM25 over chunk text, so facts that live only in + metadata — the filename, the folder it sits in — are invisible to the keyword + half of the query. Restating them in the text makes them matchable. This is + pikachu's `enrich_chunk_body`, generalised: it prepends `Filename:` and + `Path:` so a search for a file by name finds it. + + The ID and metadata are untouched, so an enriched chunk is still addressable + exactly as the contract says it is. + + Any embedding is dropped. It was computed from the text this function just + rewrote, so carrying it over would pair a vector with content it does not + describe — and the mismatch is invisible, quietly skewing the dense half of + every hybrid query. Dropping it forces a recompute downstream, which is the + safe direction to fail; the alternative is a rule that enrichment must + happen before embedding, which nothing can enforce. + + `payload` is carried through where the installed SDK has it. Unlike the + embedding it has nothing to do with the text, so rebuilding the document + without it would be silent data loss rather than a decision. It is passed + conditionally because `moss`'s runtime `DocumentInfo` accepts it while the + shipped `__init__.pyi` stub does not yet declare it — this keeps the package + correct against both, and degrades to dropping it only on a build that has + no such field to lose. + """ + if not fields: + return doc + header = "\n".join(f"{key}: {value}" for key, value in fields.items()) + carried = {"payload": doc.payload} if hasattr(doc, "payload") else {} + return DocumentInfo( + id=doc.id, + text=f"{header}\n\n{doc.text}", + metadata=doc.metadata, + embedding=None, + **carried, + ) + + +def prepend_source_context(doc: DocumentInfo, filename: str, path: str) -> DocumentInfo: + """`prepend_context` with pikachu's exact field set, for file-backed chunks.""" + return prepend_context(doc, {"Filename": filename, "Path": path}) diff --git a/packages/moss-chunking/src/ingest.py b/packages/moss-chunking/src/ingest.py new file mode 100644 index 00000000..5290d284 --- /dev/null +++ b/packages/moss-chunking/src/ingest.py @@ -0,0 +1,158 @@ +"""Copy chunks into a Moss index.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from moss import ( + DocumentInfo, + GetDocumentsOptions, + MossClient, + MutationOptions, + MutationResult, +) + +from .chunk import MAX_CHUNK_INDEX, chunk_id + +#: How many candidate IDs to look up at once when hunting for a source's leftover +#: chunks. One round trip covers a document that shrank by up to this many chunks, +#: which is nearly all of them. +_PROBE_WINDOW = 256 + + +async def ingest( + documents: Iterable[DocumentInfo], + project_id: str, + project_key: str, + index_name: str, + model_id: str | None = None, +) -> MutationResult | None: + """Create a Moss index holding exactly these chunks. + + This is the create path only: it builds a fresh index every time, so it is + not what you reach for to refresh one document inside an existing one — use + `refresh_source` for that. + + Deliberately without the connector template's `auto_id` option: random UUIDs + would make every chunk look new on the way back in, where the contract's + stable IDs let an unchanged document reproduce exactly the IDs it had. + """ + docs = list(documents) + if not docs: + return None + client = MossClient(project_id, project_key) + return await client.create_index(index_name, docs, model_id=model_id) + + +async def refresh_source( + client: MossClient, + index_name: str, + source: str, + documents: Iterable[DocumentInfo], +) -> MutationResult | None: + """Replace everything `index_name` holds for `source` with `documents`. + + Stable IDs make re-chunking an unchanged document a no-op and a rewritten one + an overwrite — but only for the chunks that still exist. `add_docs` upserts, + it does not reconcile: if `notes.md` used to cut into 21 chunks and now cuts + into 6, `#chunk-0006` through `#chunk-0020` stay in the index, still + searchable, holding text that is no longer in the document. Nothing errors, + and the stale hits look exactly like real ones. + + So the tail is deleted before the new chunks go in. What makes finding it + cheap is the contract itself: `chunk_document` guarantees indices run + `0, 1, 2, …`, so anything left over sits in a contiguous run above the new + chunk count, and looking up one window of candidate IDs past the end is + enough to find it or prove it is not there. + + That contiguity is the one assumption here, so it is also maintained here: + deletions run highest ID first and each one is waited on, which keeps what + survives a failure an unbroken run that the next refresh will find. A source + whose IDs were punched full of holes by something other than this package is + outside the guarantee — reconciling that would mean scanning the whole + `MAX_CHUNK_INDEX` space on every refresh, tens of round trips per document, + to defend against a writer that is not honouring the contract anyway. + + Raises whatever `wait_for_job` raises if a deletion fails, rather than + returning a result that implies a replacement which did not happen. Returns + the add's `MutationResult`, which the caller can wait on in turn — or `None` + when there was nothing to add. + + `documents` must be `source`'s entire cut, in order — `chunk_document`'s + output for that source and nothing else. A mismatched ID raises before + anything is deleted, since the whole reconciliation is arithmetic on + `len(documents)` and the wrong list would delete live chunks. + + **Not atomic, and one source at a time.** Two refreshes of the same source + running concurrently can interleave — a short cut deletes the tail, a long + cut adds it back, and the short cut's add lands last, leaving chunks from a + document that no longer exists. Serialize refreshes per source; the index is + the shared resource and nothing client-side can make probe/delete/add one + operation. What the ordering above does guarantee is that the wreckage is + ordinary stale tail, contiguous above the surviving cut, so the next refresh + of that source clears it. + + Passing no documents deletes every chunk for `source`, which is how a deleted + file is removed from the index. + + `upsert` is set explicitly rather than left to the server's default, since + replacing a chunk in place is the entire premise of the ID contract. + """ + docs = list(documents) + + # Everything below reads `len(docs)` as "the first index this source no + # longer uses", which is only true if these documents really are this + # source's whole cut. Handed another source's documents, or a filtered slice + # of this one's, that arithmetic would delete live chunks and then add + # documents that do not belong to `source` — a destructive way to discover a + # mistaken argument. `chunk_document` output passes this by construction. + for position, doc in enumerate(docs): + expected = chunk_id(source, position) + if doc.id != expected: + raise ValueError( + f"documents[{position}] has id {doc.id!r}, expected {expected!r}: " + f"refresh_source replaces everything under {source!r}, so it needs " + "that source's chunks, all of them, in cut order" + ) + + stale: list[str] = [] + start = len(docs) + while start <= MAX_CHUNK_INDEX: + window = [ + chunk_id(source, index) + for index in range(start, min(start + _PROBE_WINDOW, MAX_CHUNK_INDEX + 1)) + ] + found = await client.get_docs(index_name, GetDocumentsOptions(doc_ids=window)) + if not found: + break + stale.extend(doc.id for doc in found) + start += len(window) + + # `get_docs` does not promise an order, and the deletion below depends on + # one. Sorting the IDs is sorting by index, which is what the zero-padding + # in `chunk_id` is for. + stale.sort() + + # Batched, because the stale run can be thousands of IDs long and one + # request carrying all of them is one request that can fail all of them. + # + # Highest IDs first, which is what makes a failed batch recoverable. The + # survivors of a partial delete are then still one unbroken run from + # `len(docs)` upward — the shape the probe above relies on — so the next + # refresh finds them and finishes the job. Deleting lowest-first would punch + # a hole underneath them instead: the next probe would read the emptied low + # window as proof that nothing was left and stop, stranding every ID above + # it in the index, searchable, with no run that will ever reach them. + for end in range(len(stale), 0, -_PROBE_WINDOW): + batch = stale[max(end - _PROBE_WINDOW, 0) : end] + deletion = await client.delete_docs(index_name, batch) + # `delete_docs` returns when the job is accepted, not when it has run, + # so an unwaited deletion that fails afterwards is invisible: this + # function would return successfully with the stale chunks still + # searchable. The add below is handed back to the caller, who can wait + # on it; nobody outside this function ever sees these job IDs. + await client.wait_for_job(deletion.job_id) + + if not docs: + return None + return await client.add_docs(index_name, docs, MutationOptions(upsert=True)) diff --git a/packages/moss-chunking/src/strategies.py b/packages/moss-chunking/src/strategies.py new file mode 100644 index 00000000..dc0c32ef --- /dev/null +++ b/packages/moss-chunking/src/strategies.py @@ -0,0 +1,339 @@ +"""Built-in splitters. One class per strategy, all yielding `Chunk`. + +Every splitter here reports `char` locators, because they all cut plain text and +character offsets are what plain text has. A splitter over paginated or line- +oriented content reports `page` or `line` instead — the contract does not care, +so long as the unit is declared. + +The shared invariant, which the tests enforce: for a chunk produced from `text`, +`text[chunk.locator_start:chunk.locator_end] == chunk.text`. Offsets point into +the original string, never into a normalized or stripped copy of it. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import replace +from typing import Protocol + +from moss import DocumentInfo + +from .chunk import Chunk + +# A sentence ends at .!? — optionally followed by a closing quote or bracket — +# and is separated from the next by whitespace. Each lookbehind branch is a fixed +# width, which is what Python's `re` requires. +_SENTENCE_BOUNDARY = re.compile(r"(?:(?<=[.!?][\"'”’)\]])|(?<=[.!?]))\s+") + +# A blank line, plus any whitespace padding around it. CRLF is matched too, or +# every paragraph in a Windows-authored file would run into the next one and +# `ParagraphSplitter` would silently emit a single oversized chunk. +_PARAGRAPH_BOUNDARY = re.compile(r"\r?\n[ \t]*\r?\n\s*") + +Span = tuple[int, int] + + +class ChunkingStrategy(Protocol): + """Cut `text` into ordered `Chunk`s. + + Implementations own only the cutting. IDs and metadata belong to the + contract (see `chunk.py`), so a strategy never builds a `DocumentInfo` + itself — that is precisely the freedom that let pikachu and llamaindex drift + apart. + """ + + def split(self, text: str) -> Iterable[Chunk]: ... + + +def _trim(text: str, spans: Iterable[Span]) -> list[Span]: + """Shrink each span past its surrounding whitespace, dropping blank ones.""" + trimmed: list[Span] = [] + for start, end in spans: + raw = text[start:end] + stripped = raw.strip() + if not stripped: + continue + lead = len(raw) - len(raw.lstrip()) + trimmed.append((start + lead, start + lead + len(stripped))) + return trimmed + + +def _separator_pattern(separator: str) -> re.Pattern[str]: + """Boundary regex for `separator` that keeps its non-whitespace part. + + `_split_spans` discards whatever the boundary matches. That is right for + `"\\n\\n"` or `" "` — whitespace between chunks is not content. It is wrong + for `". "`, where the period belongs to the sentence it ends and only the + space separates. So a separator's non-whitespace head goes into a lookbehind + (fixed width, since it is a literal) and only its trailing whitespace is + consumed, putting the cut after the punctuation rather than through it. + """ + kept = separator.rstrip() + if not kept: + return re.compile(re.escape(separator)) + return re.compile(f"(?<={re.escape(kept)}){re.escape(separator[len(kept) :])}") + + +def _split_spans(text: str, boundary: re.Pattern[str]) -> list[Span]: + """Spans of `text` between matches of `boundary`, whitespace trimmed.""" + spans: list[Span] = [] + pos = 0 + for match in boundary.finditer(text): + spans.append((pos, match.start())) + pos = match.end() + spans.append((pos, len(text))) + return _trim(text, spans) + + +def _hard_spans(text: str, max_chars: int) -> list[Span]: + """Last resort: slice at `max_chars` when no separator gets a piece small.""" + windows = [ + (offset, min(offset + max_chars, len(text))) for offset in range(0, len(text), max_chars) + ] + return _trim(text, windows) + + +class CharSplitter: + """Fixed-width character windows with a trailing overlap. + + The pikachu strategy: its 1800/300 settings are the defaults here. + """ + + def __init__(self, chunk_chars: int = 1800, overlap: int = 300) -> None: + if chunk_chars < 1: + raise ValueError(f"chunk_chars must be >= 1, got {chunk_chars}") + if overlap < 0: + raise ValueError(f"overlap must be >= 0, got {overlap}") + if overlap >= chunk_chars: + raise ValueError( + f"overlap ({overlap}) must be < chunk_chars ({chunk_chars}), " + "otherwise chunking never advances" + ) + self.chunk_chars = chunk_chars + self.overlap = overlap + + def split(self, text: str) -> Iterator[Chunk]: + length = len(text) + start = 0 + index = 0 + # Windows advance along the raw text, but what gets emitted is the + # trimmed span inside them. On padded input two consecutive windows can + # trim down to the same span, so track where the last chunk ended and + # skip anything that adds no new content — otherwise the same text is + # indexed twice under two IDs. + last_end = -1 + while start < length: + end = min(start + self.chunk_chars, length) + for piece_start, piece_end in _trim(text, [(start, end)]): + if piece_end <= last_end: + continue + yield Chunk( + text=text[piece_start:piece_end], + index=index, + locator_type="char", + locator_start=piece_start, + locator_end=piece_end, + ) + index += 1 + last_end = piece_end + if end >= length: + break + start = max(end - self.overlap, start + 1) + + +class SentenceSplitter: + """Whole sentences accumulated up to a word budget, overlapping by sentence. + + The llamaindex strategy: its 400-word / 2-sentence settings are the defaults. + Sentence detection is regex-based rather than nltk-backed, so the package + stays dependency-free apart from `moss` itself. + """ + + def __init__(self, max_words: int = 400, overlap_sentences: int = 2) -> None: + if max_words < 1: + raise ValueError(f"max_words must be >= 1, got {max_words}") + if overlap_sentences < 0: + raise ValueError(f"overlap_sentences must be >= 0, got {overlap_sentences}") + self.max_words = max_words + # Clamped as llamaindex clamps it, to keep the parity this splitter is + # for. Not a safety net: progress is guaranteed by the `group_start + 1` + # floor in `split`, which holds for any overlap. + self.overlap_sentences = min(overlap_sentences, max(1, max_words // 100)) + + def split(self, text: str) -> Iterator[Chunk]: + sentences = _split_spans(text, _SENTENCE_BOUNDARY) + if not sentences: + return + + cursor = 0 + index = 0 + while cursor < len(sentences): + group_start = cursor + words = 0 + while cursor < len(sentences): + start, end = sentences[cursor] + sentence_words = len(text[start:end].split()) + # The first sentence always goes in, even if it blows the budget + # on its own — otherwise nothing would ever be emitted for it. + if cursor > group_start and words + sentence_words > self.max_words: + break + words += sentence_words + cursor += 1 + + chunk_start = sentences[group_start][0] + chunk_end = sentences[cursor - 1][1] + yield Chunk( + text=text[chunk_start:chunk_end], + index=index, + locator_type="char", + locator_start=chunk_start, + locator_end=chunk_end, + ) + index += 1 + + if cursor < len(sentences): + cursor = max(cursor - self.overlap_sentences, group_start + 1) + + +class ParagraphSplitter: + """Whole paragraphs packed up to a character budget. + + Paragraphs are never broken apart: one longer than `max_chars` is emitted on + its own rather than cut, on the grounds that an author's paragraph break is + better evidence of a topic boundary than an arbitrary offset. Use + `RecursiveSplitter` when a hard ceiling matters more than that. + """ + + def __init__(self, max_chars: int = 1800) -> None: + if max_chars < 1: + raise ValueError(f"max_chars must be >= 1, got {max_chars}") + self.max_chars = max_chars + + def split(self, text: str) -> Iterator[Chunk]: + paragraphs = _split_spans(text, _PARAGRAPH_BOUNDARY) + index = 0 + group: list[Span] = [] + for span in paragraphs: + if group and span[1] - group[0][0] > self.max_chars: + yield self._emit(text, group, index) + index += 1 + group = [] + group.append(span) + if group: + yield self._emit(text, group, index) + + @staticmethod + def _emit(text: str, group: Sequence[Span], index: int) -> Chunk: + start, end = group[0][0], group[-1][1] + return Chunk( + text=text[start:end], + index=index, + locator_type="char", + locator_start=start, + locator_end=end, + ) + + +class RecursiveSplitter: + """Split on the coarsest separator that fits, then merge back up. + + Tries each separator in turn — paragraphs, then lines, then sentences, then + words — recursing into any piece still over budget, and finally coalescing + adjacent pieces so chunks land near `max_chars` rather than far under it. + Unlike `ParagraphSplitter` this always respects the ceiling, falling back to + a hard character cut if no separator gets a piece small enough. + """ + + DEFAULT_SEPARATORS: tuple[str, ...] = ("\n\n", "\n", ". ", " ") + + def __init__( + self, + max_chars: int = 1800, + separators: Sequence[str] | None = None, + ) -> None: + if max_chars < 1: + raise ValueError(f"max_chars must be >= 1, got {max_chars}") + self.max_chars = max_chars + self.separators = tuple(separators) if separators is not None else self.DEFAULT_SEPARATORS + + def split(self, text: str) -> Iterator[Chunk]: + spans = self._recurse(text, 0, self.separators) + for index, (start, end) in enumerate(self._merge(spans)): + yield Chunk( + text=text[start:end], + index=index, + locator_type="char", + locator_start=start, + locator_end=end, + ) + + def _recurse(self, text: str, base: int, separators: Sequence[str]) -> list[Span]: + if len(text) <= self.max_chars: + return [(base + start, base + end) for start, end in _trim(text, [(0, len(text))])] + if not separators: + hard = _hard_spans(text, self.max_chars) + return [(base + start, base + end) for start, end in hard] + + head, tail = separators[0], separators[1:] + pieces = _split_spans(text, _separator_pattern(head)) + if len(pieces) <= 1: + # This separator bought us nothing; try the next one. + return self._recurse(text, base, tail) + + spans: list[Span] = [] + for start, end in pieces: + piece = text[start:end] + if len(piece) <= self.max_chars: + spans.append((base + start, base + end)) + else: + spans.extend(self._recurse(piece, base + start, tail)) + return spans + + def _merge(self, spans: Sequence[Span]) -> list[Span]: + """Coalesce neighbours while the combined span stays within budget.""" + merged: list[Span] = [] + for span in spans: + if merged and span[1] - merged[-1][0] <= self.max_chars: + merged[-1] = (merged[-1][0], span[1]) + else: + merged.append(span) + return merged + + +def chunk_document( + text: str, + source: str, + strategy: ChunkingStrategy, + extra: Mapping[str, object] | None = None, +) -> list[DocumentInfo]: + """Run `strategy` over `text` and render the chunks under the contract. + + `extra` is merged into every chunk's metadata — the place for source-level + facts like `extension` or `modified_at` that the splitter cannot know. + + Where the two collide the chunk's own value wins. `extra` is by definition + the same for every chunk, so letting it overwrite would mean a document-wide + constant silently replacing the page or section number that only the + splitter was in a position to know. + + A strategy that does not number its chunks `0, 1, 2, …` is rejected here. + The index is what the ID is built from, so a repeat means two chunks render + the same ID and the second overwrites the first on ingest — silent data + loss, discovered later as a document with missing content. Renumbering them + would paper over that just as quietly, and would rewrite the addressing of a + splitter that meant something by its index, so the mismatch is raised + instead. + """ + documents: list[DocumentInfo] = [] + for position, chunk in enumerate(strategy.split(text)): + if chunk.index != position: + raise ValueError( + f"{type(strategy).__name__} yielded chunk.index={chunk.index} at " + f"position {position}: indices must run 0, 1, 2, … or chunk IDs " + "collide and stop sorting in cut order" + ) + if extra: + chunk = replace(chunk, extra={**extra, **chunk.extra}) + documents.append(chunk.to_document(source)) + return documents diff --git a/packages/moss-chunking/tests/test_chunk.py b/packages/moss-chunking/tests/test_chunk.py new file mode 100644 index 00000000..a42e96e6 --- /dev/null +++ b/packages/moss-chunking/tests/test_chunk.py @@ -0,0 +1,209 @@ +"""Contract tests: IDs, validation, and the `DocumentInfo` rendering.""" + +from __future__ import annotations + +import copy +import dataclasses +import pickle + +import pytest +from moss_chunking import MAX_CHUNK_INDEX, Chunk, chunk_id + + +def test_chunk_id_is_zero_padded_so_it_sorts_in_cut_order(): + ids = [chunk_id("notes.md", i) for i in (0, 1, 9, 10, 100)] + assert ids[0] == "notes.md#chunk-0000" + assert ids[3] == "notes.md#chunk-0010" + assert sorted(ids) == ids + + +def test_chunk_id_rejects_empty_source(): + with pytest.raises(ValueError, match="non-empty"): + chunk_id("", 0) + + +def test_chunk_id_sorts_in_cut_order_across_the_whole_supported_range(): + ids = [chunk_id("notes.md", i) for i in (0, 1, 9, 10, 99, 100, 999, 1000, MAX_CHUNK_INDEX)] + assert ids == sorted(ids) + + +def test_chunk_id_rejects_an_index_the_padding_cannot_hold(): + """Past 9999 the width stops being fixed and chunk-10000 sorts before -9999.""" + with pytest.raises(ValueError, match="sorting in cut order|<="): + chunk_id("notes.md", MAX_CHUNK_INDEX + 1) + + +def test_chunk_id_rejects_negative_index(): + with pytest.raises(ValueError, match="index"): + chunk_id("notes.md", -1) + + +def test_to_document_stringifies_every_metadata_value(): + doc = Chunk("body", 3, "char", 10, 20).to_document("notes.md") + assert doc.id == "notes.md#chunk-0003" + assert doc.text == "body" + assert doc.metadata == { + "source": "notes.md", + "chunk_index": "3", + "locator_type": "char", + "locator_start": "10", + "locator_end": "20", + } + assert all(isinstance(value, str) for value in doc.metadata.values()) + + +def test_extra_metadata_is_merged(): + chunk = Chunk("body", 0, "page", 4, 4, extra={"page_label": "iv", "words": "12"}) + doc = chunk.to_document("paper.pdf") + assert doc.metadata["page_label"] == "iv" + assert doc.metadata["locator_type"] == "page" + + +def test_non_string_extra_values_are_stringified(): + """`{"page": 3}` is the natural thing to write, so accept and coerce it. + + Moss types metadata as `Dict[str, str]`; an int left in there fails at the + SDK boundary, which is a worse place to find out than here. + """ + chunk = Chunk("body", 0, "page", 4, 4, extra={"page": 3, "words": 12, "ok": True}) + metadata = chunk.to_document("paper.pdf").metadata + assert metadata["page"] == "3" + assert metadata["words"] == "12" + assert metadata["ok"] == "True" + assert all(isinstance(value, str) for value in metadata.values()) + + +def test_non_string_extra_keys_are_rejected(): + """Keys are not coerced the way values are, and must fail in this package. + + `str(1)` and `"1"` are the same metadata key, so coercing would let one + entry quietly overwrite another; left alone, a non-string key sails past the + reserved-key check and fails at the SDK boundary instead. + """ + with pytest.raises(TypeError, match="extra keys must be str"): + Chunk("body", 0, "char", 0, 4, extra={1: "one"}) + + +def test_extra_may_not_shadow_reserved_keys(): + with pytest.raises(ValueError, match="reserved"): + Chunk("body", 0, "char", 0, 4, extra={"source": "elsewhere.md"}) + + +def test_reserved_keys_win_at_render_even_if_validation_is_bypassed(): + """Belt and suspenders: the constructor check is not the last defence. + + Nothing should be able to get a reserved key into `extra` — the constructor + rejects one and stores a copy the caller cannot reach. Forced past both, rendering + still merges `extra` first so the contract's own keys overwrite it. + """ + chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) + object.__setattr__(chunk, "extra", {"source": "elsewhere.md", "chunk_index": "99"}) + + doc = chunk.to_document("notes.md") + assert doc.metadata["source"] == "notes.md" + assert doc.metadata["chunk_index"] == "0" + + +def test_a_key_added_after_construction_is_still_caught_at_render(): + """`extra` stays mutable, so the constructor's verdict has a shelf life. + + Keeping it a plain dict is what makes a chunk picklable and copyable; the + cost is that validation has to be re-run where the metadata is actually + built, rather than trusted from construction time. + """ + chunk = Chunk("body", 0, "char", 0, 4) + chunk.extra[1] = "one" + + with pytest.raises(TypeError, match="extra keys must be str"): + chunk.to_document("notes.md") + + +def test_extra_is_copied_so_the_caller_cannot_mutate_validated_state(): + supplied = {"extension": "md"} + chunk = Chunk("body", 0, "char", 0, 4, extra=supplied) + supplied["source"] = "elsewhere.md" + assert "source" not in chunk.extra + assert chunk.to_document("notes.md").metadata["source"] == "notes.md" + + +def test_a_frozen_chunk_is_actually_hashable(): + """`frozen=True` advertises hashability; a dict field would break it.""" + chunk = Chunk("body", 0, "char", 0, 4, extra={"extension": "md"}) + assert hash(chunk) == hash(Chunk("body", 0, "char", 0, 4, extra={"other": "x"})) + assert len({chunk, Chunk("body", 1, "char", 4, 8)}) == 2 + + +def test_identity_is_text_and_position_not_metadata(): + """`extra` describes a chunk; it does not decide which chunk it is.""" + assert Chunk("b", 0, "char", 0, 1, extra={"a": "1"}) == Chunk("b", 0, "char", 0, 1) + assert Chunk("b", 0, "char", 0, 1) != Chunk("b", 1, "char", 0, 1) + + +def test_a_chunk_in_a_set_stays_findable_however_extra_changes(): + """Equality cannot drift out from under a set, by rebinding or mutation.""" + chunk = Chunk("body", 0, "char", 0, 4, extra={"tags": ["a"]}) + members = {chunk} + + chunk.extra["tags"].append("b") # type: ignore[attr-defined] + chunk.extra["added"] = "later" + assert chunk in members + + +def test_a_chunk_survives_pickle_deepcopy_and_asdict(): + """The obvious dataclass paths must keep working.""" + chunk = Chunk("body", 0, "char", 0, 4, extra={"page": 3}) + assert pickle.loads(pickle.dumps(chunk)) == chunk + assert copy.deepcopy(chunk) == chunk + assert dataclasses.asdict(chunk)["extra"] == {"page": 3} + + +def test_chunk_rejects_an_unsortable_index_at_construction(): + """Rejected where the index is set, not later inside `to_document`.""" + with pytest.raises(ValueError, match="sorting in cut order|<="): + Chunk("body", MAX_CHUNK_INDEX + 1, "char", 0, 4) + assert Chunk("body", MAX_CHUNK_INDEX, "char", 0, 4).index == MAX_CHUNK_INDEX + + +@pytest.mark.parametrize("bad", [1.5, True, False, "0", None]) +def test_a_non_integer_index_is_rejected(bad): + """`bool` subclasses `int`, so True would otherwise format as chunk-0001.""" + with pytest.raises(TypeError, match="index must be an int"): + chunk_id("notes.md", bad) + with pytest.raises(TypeError, match="index must be an int"): + Chunk("body", bad, "char", 0, 4) + + +def test_chunk_id_rejects_a_non_string_source(): + """`chunk_id(123, 0)` would otherwise emit a non-string `source`.""" + with pytest.raises(TypeError, match="source must be a str"): + chunk_id(123, 0) # type: ignore[arg-type] + + +def test_unknown_locator_type_is_rejected(): + with pytest.raises(ValueError, match="locator_type"): + Chunk("body", 0, "byte", 0, 4) # type: ignore[arg-type] + + +@pytest.mark.parametrize("bad", [1.5, True, False, "0", None]) +@pytest.mark.parametrize("field", ["locator_start", "locator_end"]) +def test_a_non_integer_locator_is_rejected(field, bad): + """A locator is a position, so it gets the same treatment as the index. + + Left alone, `locator_start=1.5` renders as the metadata string `"1.5"` and + `True` renders as `"True"` — neither distinguishable downstream from an + offset the splitter meant. + """ + kwargs = {"locator_start": 0, "locator_end": 4, field: bad} + with pytest.raises(TypeError, match=f"{field} must be an int"): + Chunk("body", 0, "char", **kwargs) + + +def test_backwards_locator_is_rejected(): + with pytest.raises(ValueError, match="locator_end"): + Chunk("body", 0, "char", 20, 10) + + +def test_locator_may_be_a_single_point(): + """A page-located chunk starts and ends on the same page.""" + chunk = Chunk("body", 0, "page", 4, 4) + assert chunk.to_document("paper.pdf").metadata["locator_start"] == "4" diff --git a/packages/moss-chunking/tests/test_ingest.py b/packages/moss-chunking/tests/test_ingest.py new file mode 100644 index 00000000..c1f3f0e2 --- /dev/null +++ b/packages/moss-chunking/tests/test_ingest.py @@ -0,0 +1,282 @@ +"""`refresh_source` tests. + +Upsert alone cannot reconcile: a document that cuts into fewer chunks than it +used to leaves the old tail behind, still searchable, holding text that is no +longer in the document. These tests are about that tail — that it is found, that +it is deleted before the new chunks land, and that a document which did not +shrink pays one lookup for the privilege and nothing more. + +The client is a fake. Everything under test is which calls get made and in what +order, which is exactly what a fake can answer and a live index cannot cheaply. +""" + +from __future__ import annotations + +import pytest +from moss import DocumentInfo +from moss_chunking import CharSplitter, chunk_document, chunk_id, refresh_source + +PROSE = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi" + + +class FakeResult: + """`MutationResult` is a native type; only its `job_id` is used here.""" + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + + +class FakeClient: + """Enough of `MossClient` for the refresh path, recording what it was asked. + + `fail_delete_after` fails the nth deletion the way the SDK does — after the + job has been accepted, when it is waited on — which is the case that decides + whether a half-finished refresh can be recovered by the next one. + """ + + def __init__( + self, + existing: list[str] | None = None, + fail_delete_after: int | None = None, + shuffled: bool = False, + ) -> None: + self.existing = set(existing or []) + self.calls: list[tuple[str, object]] = [] + self.fail_delete_after = fail_delete_after + self.shuffled = shuffled + self.deletions = 0 + + async def get_docs(self, name, options=None): + self.calls.append(("get_docs", list(options.doc_ids))) + found = [doc_id for doc_id in options.doc_ids if doc_id in self.existing] + if self.shuffled: + found.reverse() + return [DocumentInfo(id=doc_id, text="stale") for doc_id in found] + + async def delete_docs(self, name, doc_ids): + self.calls.append(("delete_docs", list(doc_ids))) + self.deletions += 1 + if self.fail_delete_after is not None and self.deletions > self.fail_delete_after: + # Accepted, but the job will fail; the documents stay put. + return FakeResult(f"delete-{self.deletions}-doomed") + self.existing -= set(doc_ids) + return FakeResult(f"delete-{self.deletions}") + + async def wait_for_job(self, job_id, **kwargs): + self.calls.append(("wait_for_job", job_id)) + if job_id.endswith("doomed"): + raise RuntimeError(f"job {job_id} failed") + return None + + async def add_docs(self, name, docs, options=None): + self.calls.append(("add_docs", [d.id for d in docs])) + self.existing |= {d.id for d in docs} + return "added" + + +def ids_for(source: str, count: int) -> list[str]: + return [chunk_id(source, index) for index in range(count)] + + +def deleted(client: FakeClient) -> list[str]: + return [doc_id for kind, arg in client.calls if kind == "delete_docs" for doc_id in arg] + + +async def test_a_shrunken_document_loses_its_stale_tail(): + """The case upsert cannot handle: 21 chunks re-cut into 6.""" + client = FakeClient(existing=ids_for("notes.md", 21)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 6)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert deleted(client) == ids_for("notes.md", 21)[6:] + assert client.existing == set(ids_for("notes.md", 6)) + + +async def test_the_tail_is_deleted_before_the_new_chunks_land(): + """Ordering matters: an add that fails must not leave the stale tail behind.""" + client = FakeClient(existing=ids_for("notes.md", 9)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 2)] + + await refresh_source(client, "idx", "notes.md", docs) + + kinds = [kind for kind, _ in client.calls] + assert kinds.index("delete_docs") < kinds.index("add_docs") + + +async def test_a_document_that_did_not_shrink_deletes_nothing(): + client = FakeClient(existing=ids_for("notes.md", 4)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 4)] + + result = await refresh_source(client, "idx", "notes.md", docs) + + assert result == "added" + assert [kind for kind, _ in client.calls] == ["get_docs", "add_docs"] + + +async def test_a_grown_document_costs_a_single_lookup(): + """One probe past the end proves there is no tail; no scan of the index.""" + client = FakeClient(existing=ids_for("notes.md", 2)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 5)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert [kind for kind, _ in client.calls].count("get_docs") == 1 + + +async def test_passing_no_documents_removes_the_source_entirely(): + """How a deleted file leaves the index.""" + client = FakeClient(existing=ids_for("notes.md", 3) + ids_for("other.md", 2)) + + result = await refresh_source(client, "idx", "notes.md", []) + + assert result is None + assert client.existing == set(ids_for("other.md", 2)) + + +async def test_another_sources_chunks_are_never_touched(): + client = FakeClient(existing=ids_for("notes.md", 8) + ids_for("other.md", 8)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert set(ids_for("other.md", 8)) <= client.existing + assert not any(doc_id.startswith("other.md") for doc_id in deleted(client)) + + +async def test_a_tail_longer_than_one_probe_window_is_still_cleared(): + """The window is a batch size, not a ceiling: probing continues while it hits.""" + client = FakeClient(existing=ids_for("notes.md", 600)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + assert client.existing == set(ids_for("notes.md", 1)) + assert len(deleted(client)) == 599 + + +async def test_documents_from_another_source_are_refused_before_anything_is_deleted(): + """The wrong argument would otherwise be discovered destructively.""" + client = FakeClient(existing=ids_for("notes.md", 5)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("other.md", 2)] + + with pytest.raises(ValueError, match="expected 'notes.md#chunk-0000'"): + await refresh_source(client, "idx", "notes.md", docs) + + assert client.calls == [] + assert client.existing == set(ids_for("notes.md", 5)) + + +async def test_a_gapped_document_list_is_refused(): + """A filtered list would make `len(docs)` mean the wrong index.""" + client = FakeClient(existing=ids_for("notes.md", 5)) + kept = [chunk_id("notes.md", 0), chunk_id("notes.md", 2)] + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in kept] + + with pytest.raises(ValueError, match=r"documents\[1\]"): + await refresh_source(client, "idx", "notes.md", docs) + + assert client.calls == [] + + +def test_the_client_surface_this_module_calls_actually_exists(): + """The fake client above cannot catch a method that the real one lacks. + + `wait_for_job` in particular is real on the declared floor (`moss==1.7.2`) + but missing from both the checked-in stub and the in-tree SDK source, which + is old enough to predate it — so reading either one suggests this module + calls something that isn't there. This asserts against the installed SDK, + which is the surface that decides. + """ + from moss import MossClient + + missing = [ + name + for name in ("get_docs", "delete_docs", "add_docs", "wait_for_job") + if not hasattr(MossClient, name) + ] + assert not missing + + +async def test_the_tail_is_deleted_from_the_top_down(): + """Descending order is what makes a half-finished refresh recoverable.""" + client = FakeClient(existing=ids_for("notes.md", 600)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + batches = [arg for kind, arg in client.calls if kind == "delete_docs"] + assert len(batches) > 1 + assert [batch[0] for batch in batches] == sorted((b[0] for b in batches), reverse=True) + + +async def test_every_deletion_is_waited_on(): + """`delete_docs` returns when the job is accepted, not when it has run.""" + client = FakeClient(existing=ids_for("notes.md", 600)) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + kinds = [kind for kind, _ in client.calls] + for position, kind in enumerate(kinds): + if kind == "delete_docs": + assert kinds[position + 1] == "wait_for_job" + + +async def test_a_failed_deletion_is_raised_rather_than_reported_as_a_refresh(): + """Returning normally here would claim a replacement that did not happen.""" + client = FakeClient(existing=ids_for("notes.md", 9), fail_delete_after=0) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 2)] + + with pytest.raises(RuntimeError, match="failed"): + await refresh_source(client, "idx", "notes.md", docs) + + assert not any(kind == "add_docs" for kind, _ in client.calls) + + +async def test_a_refresh_after_a_failed_deletion_finishes_the_job(): + """The regression the delete order exists for. + + 600 chunks re-cut to 1, with the deletion failing partway. Whatever survives + has to stay reachable from a probe that starts at the new chunk count — if + the surviving run had a hole punched under it, the retry would stop at the + hole and strand everything above it forever. + """ + existing = ids_for("notes.md", 600) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + failing = FakeClient(existing=existing, fail_delete_after=1) + with pytest.raises(RuntimeError): + await refresh_source(failing, "idx", "notes.md", docs) + assert len(failing.existing) > 1 # the refresh really did leave work behind + + retry = FakeClient(existing=sorted(failing.existing)) + await refresh_source(retry, "idx", "notes.md", docs) + + assert retry.existing == set(ids_for("notes.md", 1)) + + +async def test_unordered_lookup_results_do_not_break_the_delete_order(): + """`get_docs` promises no order; the zero-padded IDs are sorted before use.""" + client = FakeClient(existing=ids_for("notes.md", 600), shuffled=True) + docs = [DocumentInfo(id=doc_id, text="fresh") for doc_id in ids_for("notes.md", 1)] + + await refresh_source(client, "idx", "notes.md", docs) + + batches = [arg for kind, arg in client.calls if kind == "delete_docs"] + assert [batch[0] for batch in batches] == sorted((b[0] for b in batches), reverse=True) + assert client.existing == set(ids_for("notes.md", 1)) + + +@pytest.mark.parametrize("cut", [40, 12]) +async def test_refresh_is_idempotent_for_real_chunked_output(cut): + """Twice through with the same text changes nothing the second time.""" + client = FakeClient() + docs = chunk_document(PROSE, "notes.md", CharSplitter(chunk_chars=cut, overlap=5)) + + await refresh_source(client, "idx", "notes.md", docs) + after_first = set(client.existing) + await refresh_source(client, "idx", "notes.md", docs) + + assert client.existing == after_first + assert not deleted(client) diff --git a/packages/moss-chunking/tests/test_strategies.py b/packages/moss-chunking/tests/test_strategies.py new file mode 100644 index 00000000..0b5cc8f1 --- /dev/null +++ b/packages/moss-chunking/tests/test_strategies.py @@ -0,0 +1,332 @@ +"""Splitter tests. + +The invariant every strategy must hold is the roundtrip: a chunk's offsets point +into the original text and slice back to exactly its own text. Get that wrong and +the position metadata is decorative — you can address a chunk but not find it. +""" + +from __future__ import annotations + +import inspect + +import pytest +from moss import DocumentInfo +from moss_chunking import ( + CharSplitter, + Chunk, + ParagraphSplitter, + RecursiveSplitter, + SentenceSplitter, + chunk_document, + prepend_context, + prepend_source_context, +) + +PROSE = ( + "Moss is a semantic search runtime. It runs on device. " + "Queries return in under ten milliseconds.\n\n" + "Chunking decides what a query can match. Cut too coarsely and a hit drags " + "in noise. Cut too finely and the surrounding context is lost.\n\n" + "So the strategy has to suit the content." +) + +ALL_STRATEGIES = [ + CharSplitter(chunk_chars=120, overlap=20), + SentenceSplitter(max_words=12, overlap_sentences=1), + ParagraphSplitter(max_chars=120), + RecursiveSplitter(max_chars=120), +] + + +def assert_roundtrips(text: str, chunks) -> None: + """Assert per chunk, so a failure names the offending one and its offsets.""" + for chunk in chunks: + assert text[chunk.locator_start : chunk.locator_end] == chunk.text + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_offsets_slice_back_to_the_chunk_text(strategy): + chunks = list(strategy.split(PROSE)) + assert chunks + assert_roundtrips(PROSE, chunks) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_indices_are_sequential_from_zero(strategy): + chunks = list(strategy.split(PROSE)) + assert [c.index for c in chunks] == list(range(len(chunks))) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_empty_and_blank_input_yields_nothing(strategy): + assert list(strategy.split("")) == [] + assert list(strategy.split(" \n\n\t ")) == [] + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_chunks_are_never_blank(strategy): + assert all(c.text.strip() for c in strategy.split(PROSE)) + + +@pytest.mark.parametrize("strategy", ALL_STRATEGIES, ids=lambda s: type(s).__name__) +def test_chunks_advance_through_the_document(strategy): + starts = [c.locator_start for c in strategy.split(PROSE)] + assert starts == sorted(starts) + + +# --- CharSplitter ------------------------------------------------------------ + + +def test_char_splitter_respects_its_ceiling(): + chunks = list(CharSplitter(chunk_chars=50, overlap=10).split(PROSE)) + assert all(len(c.text) <= 50 for c in chunks) + + +def test_char_splitter_overlaps_by_the_requested_amount(): + text = "a" * 250 + chunks = list(CharSplitter(chunk_chars=100, overlap=30).split(text)) + assert chunks[1].locator_start == chunks[0].locator_end - 30 + + +def test_char_splitter_covers_the_whole_document(): + chunks = list(CharSplitter(chunk_chars=40, overlap=5).split(PROSE)) + assert chunks[0].locator_start == 0 + assert chunks[-1].locator_end == len(PROSE) + + +def test_padded_input_does_not_emit_the_same_span_twice(): + """Windows advance along raw text, but chunks are the trimmed span inside. + + Two consecutive windows over padded text can trim to the same span, which + would index identical content twice under two different chunk IDs. + """ + text = " abcde fghij" + chunks = list(CharSplitter(chunk_chars=10, overlap=5).split(text)) + spans = [(c.locator_start, c.locator_end) for c in chunks] + assert len(spans) == len(set(spans)) + assert [c.text for c in chunks] == ["abcde", "fghij"] + assert [c.index for c in chunks] == [0, 1] + + +def test_overlap_at_or_above_chunk_size_is_rejected(): + with pytest.raises(ValueError, match="never advances"): + CharSplitter(chunk_chars=100, overlap=100) + + +def test_short_input_is_a_single_chunk(): + chunks = list(CharSplitter().split("one short line")) + assert len(chunks) == 1 + assert chunks[0].text == "one short line" + + +# --- SentenceSplitter -------------------------------------------------------- + + +def test_sentence_splitter_does_not_break_mid_sentence(): + chunks = list(SentenceSplitter(max_words=12, overlap_sentences=1).split(PROSE)) + assert all(c.text.rstrip().endswith((".", "!", "?")) for c in chunks) + + +def test_sentence_longer_than_the_budget_is_still_emitted(): + text = "This one sentence is deliberately longer than the tiny word budget allows." + chunks = list(SentenceSplitter(max_words=3).split(text)) + assert len(chunks) == 1 + assert chunks[0].text == text + + +def test_sentence_splitter_handles_closing_quotes(): + text = 'He said "it works." Then he left. She agreed.' + chunks = list(SentenceSplitter(max_words=4, overlap_sentences=0).split(text)) + assert_roundtrips(text, chunks) + assert chunks[0].text.startswith("He said") + + +def test_splitting_terminates_for_any_overlap(): + """The `group_start + 1` floor, not the clamp, is what guarantees progress.""" + splitter = SentenceSplitter(max_words=3) + splitter.overlap_sentences = 999 # defeat the clamp, keep the loop + chunks = list(splitter.split("One. Two. Three. Four. Five. Six.")) + assert chunks + assert [c.index for c in chunks] == list(range(len(chunks))) + + +def test_overlap_is_clamped_so_progress_is_guaranteed(): + splitter = SentenceSplitter(max_words=50, overlap_sentences=99) + assert splitter.overlap_sentences == 1 + assert list(splitter.split(PROSE)) + + +# --- ParagraphSplitter ------------------------------------------------------- + + +def test_oversized_paragraph_is_emitted_rather_than_cut(): + text = "x" * 500 + chunks = list(ParagraphSplitter(max_chars=100).split(text)) + assert len(chunks) == 1 + assert len(chunks[0].text) == 500 + + +def test_paragraphs_are_packed_together_when_they_fit(): + text = "one.\n\ntwo.\n\nthree." + assert len(list(ParagraphSplitter(max_chars=1000).split(text))) == 1 + + +def test_paragraphs_are_split_apart_when_they_do_not_fit(): + text = "one.\n\ntwo.\n\nthree." + assert len(list(ParagraphSplitter(max_chars=6).split(text))) == 3 + + +def test_crlf_blank_lines_are_paragraph_boundaries_too(): + """A Windows-authored file must not collapse into one oversized chunk.""" + text = "one.\r\n\r\ntwo.\r\n\r\nthree." + chunks = list(ParagraphSplitter(max_chars=6).split(text)) + assert len(chunks) == 3 + assert_roundtrips(text, chunks) + + +# --- RecursiveSplitter ------------------------------------------------------- + + +def test_recursive_splitter_always_respects_the_ceiling(): + chunks = list(RecursiveSplitter(max_chars=80).split(PROSE)) + assert all(len(c.text) <= 80 for c in chunks) + + +def test_recursive_splitter_hard_cuts_unseparated_text(): + text = "x" * 300 + chunks = list(RecursiveSplitter(max_chars=100).split(text)) + assert all(len(c.text) <= 100 for c in chunks) + assert "".join(c.text for c in chunks) == text + + +def test_recursive_splitter_keeps_the_punctuation_it_splits_on(): + """A separator's non-whitespace part is content, not a delimiter. + + Splitting on `". "` used to consume the period along with the space, so + "abc. def." came back as "abc" + "def." — the chunk text no longer said what + the source said. The roundtrip invariant did not catch it: offsets stayed + self-consistent, they just stopped covering the text between them. + """ + text = "abc. def." + chunks = list(RecursiveSplitter(max_chars=4).split(text)) + assert [c.text for c in chunks] == ["abc.", "def."] + assert "".join(c.text for c in chunks) == text.replace(" ", "") + + +def test_no_strategy_loses_non_whitespace_characters(): + """The gap between two adjacent chunks may only ever be whitespace.""" + text = "one. two. three. four. five. six." + for splitter in (RecursiveSplitter(max_chars=10), RecursiveSplitter(max_chars=6)): + chunks = list(splitter.split(text)) + for earlier, later in zip(chunks, chunks[1:], strict=False): + assert not text[earlier.locator_end : later.locator_start].strip() + + +def test_whitespace_separators_are_still_consumed(): + """The fix must not start emitting the blank line between paragraphs.""" + chunks = list(RecursiveSplitter(max_chars=10).split("alpha\n\nbeta")) + assert all(not c.text.startswith(("\n", " ")) for c in chunks) + assert all(not c.text.endswith(("\n", " ")) for c in chunks) + + +def test_custom_separator_without_trailing_whitespace_is_preserved(): + chunks = list(RecursiveSplitter(max_chars=3, separators=(";",)).split("ab;cd;ef")) + assert "".join(c.text for c in chunks) == "ab;cd;ef" + + +def test_recursive_splitter_prefers_paragraph_boundaries(): + text = "alpha beta.\n\ngamma delta." + chunks = list(RecursiveSplitter(max_chars=12).split(text)) + assert [c.text for c in chunks] == ["alpha beta.", "gamma delta."] + + +# --- chunk_document + enrich ------------------------------------------------- + + +def test_chunk_document_renders_documents_under_the_contract(): + docs = chunk_document(PROSE, "notes.md", CharSplitter(chunk_chars=100, overlap=10)) + assert docs + assert docs[0].id == "notes.md#chunk-0000" + assert docs[0].metadata["source"] == "notes.md" + assert docs[0].metadata["locator_type"] == "char" + + +def test_chunk_document_merges_source_level_extra_metadata(): + docs = chunk_document( + PROSE, + "notes.md", + CharSplitter(chunk_chars=100, overlap=10), + extra={"extension": "md", "modified_at": "2026-07-28T00:00:00Z"}, + ) + assert all(d.metadata["extension"] == "md" for d in docs) + assert all(d.metadata["source"] == "notes.md" for d in docs) + + +def test_a_chunks_own_metadata_beats_source_level_extra(): + """`extra` is document-wide; the splitter knows the per-chunk truth.""" + + class Paged: + def split(self, text: str): + yield Chunk(text[:4], 0, "page", 1, 1, extra={"page": 1}) + yield Chunk(text[4:8], 1, "page", 2, 2, extra={"page": 2}) + + docs = chunk_document("aaaabbbb", "paper.pdf", Paged(), extra={"page": 0, "ext": "pdf"}) + assert [d.metadata["page"] for d in docs] == ["1", "2"] + assert all(d.metadata["ext"] == "pdf" for d in docs) + + +@pytest.mark.parametrize("indices", [(0, 0), (0, 2), (1, 2)]) +def test_chunk_document_rejects_indices_that_are_not_sequential(indices): + """A repeat renders the same ID twice, and the second chunk wins on ingest. + + That is silent data loss, so a strategy that does not number its chunks from + zero is caught at the render boundary rather than trusted. + """ + + class Misnumbering: + def split(self, text: str): + for index in indices: + yield Chunk(text[:4], index, "char", 0, 4) + + with pytest.raises(ValueError, match="Misnumbering yielded chunk.index="): + chunk_document("aaaabbbb", "notes.md", Misnumbering()) + + +def test_chunk_document_rejects_extra_that_shadows_the_contract(): + with pytest.raises(ValueError, match="reserved"): + chunk_document(PROSE, "notes.md", CharSplitter(), extra={"chunk_index": "9"}) + + +def test_prepend_context_changes_text_but_not_addressing(): + doc = chunk_document(PROSE, "notes.md", CharSplitter())[0] + enriched = prepend_source_context(doc, filename="notes.md", path="/docs/notes.md") + assert enriched.id == doc.id + assert enriched.metadata == doc.metadata + assert enriched.text.startswith("Filename: notes.md\nPath: /docs/notes.md\n\n") + assert enriched.text.endswith(doc.text) + + +def test_enrichment_drops_an_embedding_computed_from_the_old_text(): + doc = chunk_document(PROSE, "notes.md", CharSplitter())[0] + stale = DocumentInfo(id=doc.id, text=doc.text, metadata=doc.metadata, embedding=[0.1, 0.2]) + enriched = prepend_source_context(stale, filename="notes.md", path="/docs/notes.md") + assert enriched.text != stale.text + assert getattr(enriched, "embedding", None) is None + + +def test_enrichment_carries_the_payload_through(): + """Unlike the embedding, `payload` has nothing to do with the text. + + Skipped rather than failed on an SDK build without the field, so the package + does not pin itself to one runtime shape of `DocumentInfo`. + """ + if "payload" not in inspect.signature(DocumentInfo).parameters: + pytest.skip("installed moss DocumentInfo has no payload field") + doc = DocumentInfo(id="a#chunk-0000", text="body", metadata={"source": "a"}, payload='{"p":1}') + enriched = prepend_source_context(doc, filename="a.md", path="/a.md") + assert enriched.payload == '{"p":1}' + + +def test_enrichment_with_no_fields_is_a_no_op(): + doc = chunk_document(PROSE, "notes.md", CharSplitter())[0] + assert prepend_context(doc, {}) is doc diff --git a/packages/moss-chunking/uv.lock b/packages/moss-chunking/uv.lock new file mode 100644 index 00000000..22e918d8 --- /dev/null +++ b/packages/moss-chunking/uv.lock @@ -0,0 +1,318 @@ +version = 1 +revision = 3 +requires-python = ">=3.10, <3.15" + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "backports-asyncio-runner" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "inferedge-moss-core" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/02/b9f6bc352bd8e26c9006c579948444795177df86f55bceeebc301342ccc6/inferedge_moss_core-0.21.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:4ec09b432a08cd19a5c2c68c40ccc7ff06da0e5429a0461aa1b34a7973f4e313", size = 3958992, upload-time = "2026-07-21T05:06:05.281Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a6/21f74c3f239cc93a8a17e53aef8cb168b93a1ae3b48d7c13fdfec1b24e58/inferedge_moss_core-0.21.0-cp310-abi3-manylinux_2_35_x86_64.whl", hash = "sha256:ddbabce5503be4840db6c2a54f4feeb8e426baf5ed06b1182847e7115f717be4", size = 4915489, upload-time = "2026-07-21T06:27:28.13Z" }, + { url = "https://files.pythonhosted.org/packages/53/d7/9ac7b869c767928dcbcf49089ee8e9dbf18a83a6827ae08d0b94bf1b9d8f/inferedge_moss_core-0.21.0-cp310-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:bbf51735ad8a13e94fc38f1a3a147294c92b4e198fd711b02cd0b026eced7ac3", size = 4813300, upload-time = "2026-07-21T05:06:11.614Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e8/11262d2284491b858f0a84dae7f007734813e5bd91aa9da72234fd95e029/inferedge_moss_core-0.21.0-cp310-abi3-win_amd64.whl", hash = "sha256:e5d142b5c6069b8e5fb30f4ce6a1df40d7f756242de29e74c4b6f5a770b4795e", size = 5350984, upload-time = "2026-07-21T05:10:26.373Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "moss" +version = "1.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "inferedge-moss-core" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/66/8595da68c9d61128c4faa16e082b53231550c00a40a0cfabb12460b204cd/moss-1.7.2.tar.gz", hash = "sha256:adadef3c8006ac773b1665764a6e7ed0390ed5d9d28e3389890a52202e8ceb98", size = 55466, upload-time = "2026-07-21T06:34:43.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/d5/4a7ad22817bf20fa4eb29d7ae2349ad037e516e99cfa62cd19662ed52e37/moss-1.7.2-py3-none-any.whl", hash = "sha256:1f3bcf7fff29b0dcb5d78c14b9e547fed527dd3eabe5c2f5db598f60515784ad", size = 22914, upload-time = "2026-07-21T06:34:42.689Z" }, +] + +[[package]] +name = "moss-chunking" +version = "0.0.1" +source = { editable = "." } +dependencies = [ + { name = "moss" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "python-dotenv" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "moss", specifier = ">=1.7.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, + { name = "python-dotenv", marker = "extra == 'dev'", specifier = ">=1.0.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, + { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, + { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, + { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, + { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +]