From 70c8ae7dda9a45b319a04278e103ea4fd0523f3b Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 8 Nov 2026 06:55:35 +0100 Subject: [PATCH 1/2] feat(catalog): refuse a catalog newer than the supported schema version Catalog load reads catalog_schema_version from the parquet sddb_release_info metadata and raises a clear error when it exceeds SUPPORTED_CATALOG_SCHEMA=2. A legacy catalog that declares no version loads unchanged (missing == OK). --- README.md | 2 ++ src/sddb/_catalog_schema.py | 6 ++++++ src/sddb/catalog.py | 24 ++++++++++++++++++++++++ tests/test_catalog_metadata.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/README.md b/README.md index 1627617..7a49262 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ cat.query(organism="human") `from_file` accepts absolute or relative paths and `file://` URLs, and validates the file against the catalog schema. +Catalog schema version: the client loads any catalog up to schema version `2`; a catalog cut under a newer schema is refused with a clear error asking you to `pip install -U spatialdata-db-client`. A legacy catalog that declares no version loads unchanged. + Remote datasets: query the catalog, inspect with `elements()` (no download), or load it with `to_sdata()` (copies the store to the cache). Lazy/partial remote open is not supported yet (spatialdata upstream limitation); local stores open lazily. ## Releasing diff --git a/src/sddb/_catalog_schema.py b/src/sddb/_catalog_schema.py index d657fe6..d519451 100644 --- a/src/sddb/_catalog_schema.py +++ b/src/sddb/_catalog_schema.py @@ -200,6 +200,12 @@ "collections", ) +# The newest catalog schema version this client understands. A catalog that declares a higher +# ``catalog_schema_version`` (in release_info.json / the parquet metadata) is refused on load; a +# catalog that declares none is a legacy (<= 2) catalog and loads. Bump when the client learns a +# new schema's columns. +SUPPORTED_CATALOG_SCHEMA = 2 + # Columns the client derives on load (not in the published parquet): ``source`` is the lowercased # ``study_id`` prefix before ":" (geo, hest, 10x, ...), null for rows without a study_id. DERIVED_COLUMNS: dict[str, str] = {"source": "string"} diff --git a/src/sddb/catalog.py b/src/sddb/catalog.py index 3a99b66..f5c214e 100644 --- a/src/sddb/catalog.py +++ b/src/sddb/catalog.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import operator import os import re @@ -152,6 +153,23 @@ def facet_values(df: pd.DataFrame, field: str | None = None) -> list[str]: return _distinct(df[field]) +def _catalog_schema_version(path: Path) -> int | None: + """The catalog's declared ``catalog_schema_version``, or ``None`` for a legacy catalog without one. + + Read from the parquet ``sddb_release_info`` metadata — a JSON copy of release_info.json that the + producer embeds in the parquet so it travels with the file (through both remote download and the + local-copy cache, exactly like ``sddb_generated_at``). Absent or unreadable provenance is a legacy + catalog, not an error — the caller treats ``None`` as supported. + """ + try: + raw = (pq.read_metadata(path).metadata or {}).get(b"sddb_release_info") + info = json.loads(raw.decode()) if raw else {} + except Exception: # absent/corrupt provenance is legacy, never fatal + info = {} + ver = info.get("catalog_schema_version") + return int(ver) if isinstance(ver, int) and not isinstance(ver, bool) else None + + class Catalog: """The dataset catalog: fetched, cached and validated on construction.""" @@ -195,6 +213,12 @@ def __init__( self._genes: GeneIndex | None = None self._features: FeatureIndex | None = None self._path = fetch_catalog(url, cache_dir=cache_dir, refresh=refresh) + ver = _catalog_schema_version(self._path) + if ver is not None and ver > schema.SUPPORTED_CATALOG_SCHEMA: + raise ValueError( + f"catalog at {url} declares schema version {ver}, but this sddb client supports up to " + f"{schema.SUPPORTED_CATALOG_SCHEMA}. Upgrade with `pip install -U spatialdata-db-client`." + ) self._sha256 = _file_sha256(self._path) # recorded in every cohort's Source, checked by verify() # The dated release behind this file: catalog-latest is an alias of the newest one, whose date is the # parquet's sddb_generated_at. Only the default release URLs have a release. diff --git a/tests/test_catalog_metadata.py b/tests/test_catalog_metadata.py index a7b8245..f9c7d9d 100644 --- a/tests/test_catalog_metadata.py +++ b/tests/test_catalog_metadata.py @@ -2,10 +2,14 @@ from __future__ import annotations +import json + import pyarrow as pa import pyarrow.parquet as pq +import pytest from sddb import Catalog +from sddb import _catalog_schema as schema def _write(path, meta): @@ -35,3 +39,29 @@ def test_generated_at_absent_is_none(tmp_path): p = tmp_path / "catalog.parquet" _write(p, {}) assert Catalog.from_file(p).generated_at is None + + +def _release_info_meta(version): + return {b"sddb_release_info": json.dumps({"catalog_schema_version": version}).encode()} + + +def test_schema_version_supported_loads(tmp_path): + """A catalog declaring the supported schema version loads.""" + p = tmp_path / "catalog.parquet" + _write(p, _release_info_meta(schema.SUPPORTED_CATALOG_SCHEMA)) + assert len(Catalog.from_file(p)) == 1 + + +def test_schema_version_absent_is_legacy(tmp_path): + """A legacy catalog without a declared schema version loads (missing == OK).""" + p = tmp_path / "catalog.parquet" + _write(p, {}) + assert len(Catalog.from_file(p)) == 1 + + +def test_schema_version_too_new_raises(tmp_path): + """A catalog declaring a newer schema in its parquet metadata is refused with a clear error.""" + p = tmp_path / "catalog.parquet" + _write(p, _release_info_meta(schema.SUPPORTED_CATALOG_SCHEMA + 1)) + with pytest.raises(ValueError, match=f"schema version {schema.SUPPORTED_CATALOG_SCHEMA + 1}"): + Catalog.from_file(p) From fdc56e15cb17593b7103e470d9b14238833f35f8 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 8 Nov 2026 07:26:37 +0100 Subject: [PATCH 2/2] fix(catalog): fail closed on broken or non-integer schema provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: only a missing sddb_release_info or a missing catalog_schema_version key is legacy. Provenance that is present must be sound — unparseable sddb_release_info, or a non-integer catalog_schema_version, now raises ValueError (a release shipping broken provenance is a bug), as does an int above SUPPORTED_CATALOG_SCHEMA. --- src/sddb/catalog.py | 29 ++++++++++++++++++++++------- tests/test_catalog_metadata.py | 22 +++++++++++++++++++++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/sddb/catalog.py b/src/sddb/catalog.py index f5c214e..cf5b19e 100644 --- a/src/sddb/catalog.py +++ b/src/sddb/catalog.py @@ -158,16 +158,31 @@ def _catalog_schema_version(path: Path) -> int | None: Read from the parquet ``sddb_release_info`` metadata — a JSON copy of release_info.json that the producer embeds in the parquet so it travels with the file (through both remote download and the - local-copy cache, exactly like ``sddb_generated_at``). Absent or unreadable provenance is a legacy - catalog, not an error — the caller treats ``None`` as supported. + local-copy cache, exactly like ``sddb_generated_at``). + + Fail closed on broken provenance: a release that ships a corrupt ``sddb_release_info`` or a + non-integer ``catalog_schema_version`` is a bug, so both raise ``ValueError``. Only genuine absence + is legacy — no ``sddb_release_info`` at all, or provenance that declares no ``catalog_schema_version`` + — and returns ``None`` (the caller treats ``None`` as supported). """ try: raw = (pq.read_metadata(path).metadata or {}).get(b"sddb_release_info") - info = json.loads(raw.decode()) if raw else {} - except Exception: # absent/corrupt provenance is legacy, never fatal - info = {} - ver = info.get("catalog_schema_version") - return int(ver) if isinstance(ver, int) and not isinstance(ver, bool) else None + except Exception: # an unreadable parquet is reported with a clear message by read_parquet + return None + if raw is None: + return None # no provenance at all -> legacy + try: + info = json.loads(raw.decode()) + except Exception as err: + raise ValueError( + f"catalog sddb_release_info is not valid JSON; a release with broken provenance must not load: {err}" + ) from err + if "catalog_schema_version" not in info: + return None # provenance present, version absent -> legacy + ver = info["catalog_schema_version"] + if not isinstance(ver, int) or isinstance(ver, bool): + raise ValueError(f"catalog_schema_version must be an integer, got {ver!r}") + return ver class Catalog: diff --git a/tests/test_catalog_metadata.py b/tests/test_catalog_metadata.py index f9c7d9d..2f87f7a 100644 --- a/tests/test_catalog_metadata.py +++ b/tests/test_catalog_metadata.py @@ -55,8 +55,11 @@ def test_schema_version_supported_loads(tmp_path): def test_schema_version_absent_is_legacy(tmp_path): """A legacy catalog without a declared schema version loads (missing == OK).""" p = tmp_path / "catalog.parquet" - _write(p, {}) + _write(p, {}) # no sddb_release_info at all assert len(Catalog.from_file(p)) == 1 + p2 = tmp_path / "catalog2.parquet" + _write(p2, {b"sddb_release_info": json.dumps({"hub_sha": "abc"}).encode()}) # provenance, no version key + assert len(Catalog.from_file(p2)) == 1 def test_schema_version_too_new_raises(tmp_path): @@ -65,3 +68,20 @@ def test_schema_version_too_new_raises(tmp_path): _write(p, _release_info_meta(schema.SUPPORTED_CATALOG_SCHEMA + 1)) with pytest.raises(ValueError, match=f"schema version {schema.SUPPORTED_CATALOG_SCHEMA + 1}"): Catalog.from_file(p) + + +def test_schema_version_broken_provenance_raises(tmp_path): + """Corrupt sddb_release_info is a bug, not a legacy catalog — it must fail closed.""" + p = tmp_path / "catalog.parquet" + _write(p, {b"sddb_release_info": b"{not json"}) + with pytest.raises(ValueError, match="not valid JSON"): + Catalog.from_file(p) + + +def test_schema_version_non_integer_raises(tmp_path): + """A present catalog_schema_version must be an integer (a string or float fails closed).""" + for bad in ("3", 2.5): + p = tmp_path / f"catalog_{bad}.parquet" + _write(p, _release_info_meta(bad)) + with pytest.raises(ValueError, match="must be an integer"): + Catalog.from_file(p)