diff --git a/MODEL_CARD.md b/MODEL_CARD.md index e5a5ae5..bf51001 100644 --- a/MODEL_CARD.md +++ b/MODEL_CARD.md @@ -8,10 +8,23 @@ NadirClaw, the open-source router in this repo, ships **the architecture description, the bundled trained weights, the heuristics on top of them**, and the cascade rule engine (see `nadirclaw/cascade.py`, `nadirclaw/cascade_rules/`, `nadirclaw/heuristic_verifier.py`, -`nadirclaw/wide_deep_classifier.py`). The `wide_deep_asym_v3.pt` -checkpoint (and its symmetric-loss companion `wide_deep_sym_v3.pt`, -~900 KB each) lives under `nadirclaw/models/` and is loaded -automatically by `nadirclaw.wide_deep_classifier.get_wide_deep_classifier()`. +`nadirclaw/wide_deep_classifier.py`). Three ~900 KB checkpoints live under +`nadirclaw/models/`: `wide_deep_v3.pt` (the clean retrain, **the default +since 0.22**), `wide_deep_asym_v3.pt` (the original asym-loss head, which +suppresses the simple class), and `wide_deep_sym_v3.pt` (symmetric-loss +companion). They are loaded automatically by +`nadirclaw.wide_deep_classifier.get_wide_deep_classifier()`. + +**v3+gate (default since 0.22).** The `v3` head runs under a Neyman-Pearson +**complex gate**: a small logistic (`complex_gate_v1.pt`, ~12 KB) decides +complex-vs-rest at high recall (threshold τ=0.12); below the gate, the v3 +head's own `P(simple)` vs `P(medium)` splits the rest (the head split — more +accurate than the legacy companion logistic, which under-fills Medium). On a +2,479-prompt RouterArena eval this misses 8.2% of truly-complex prompts while +reducing cost ~41% vs always-premium. Disable with `NADIR_COMPLEX_GATE=0`; +tune τ with `NADIR_GATE_THRESHOLD`; revert the split with +`NADIR_MS_SPLIT=companion`. `asym`/`symmetric` keep their legacy argmax +behaviour (gate off by default). The weights and code are released under the PolyForm Noncommercial License 1.0.0 alongside the rest of the package — free for noncommercial use; commercial use requires a license via [getnadir.com](https://getnadir.com). @@ -20,10 +33,10 @@ team billing, the trained DeBERTa-v3-small cascade verifier, and closed-loop retraining over the same classifier. - **Router name**: `nadir` -- **Classifier family**: wide-and-deep asymmetric (`wide_deep_asym`) -- **Production artifact**: `wide_deep_asym_v3.pt` (bundled in NadirClaw + used in Nadir Pro) -- **Companion artifact**: `wide_deep_sym_v3.pt` (symmetric-loss variant, fixes the asym head's simple-class collapse under argmax decoding) -- **Card last updated**: 2026-05-27 +- **Classifier family**: wide-and-deep (`wide_deep`) +- **Production artifact**: `wide_deep_v3.pt` + `complex_gate_v1.pt` (v3+gate, the default since 0.22; bundled in NadirClaw + deployed in Nadir Pro) +- **Legacy artifacts**: `wide_deep_asym_v3.pt` (asym-loss, suppresses simple), `wide_deep_sym_v3.pt` (symmetric-loss companion) — both opt-in +- **Card last updated**: 2026-07-19 - **Schema version**: 1 --- diff --git a/nadirclaw/__init__.py b/nadirclaw/__init__.py index 038ef6b..3963e71 100644 --- a/nadirclaw/__init__.py +++ b/nadirclaw/__init__.py @@ -1,3 +1,3 @@ """NadirClaw — Open-source LLM router.""" -__version__ = "0.21.1" +__version__ = "0.22.0" diff --git a/nadirclaw/models/complex_gate_v1.pt b/nadirclaw/models/complex_gate_v1.pt new file mode 100644 index 0000000..e22dab8 Binary files /dev/null and b/nadirclaw/models/complex_gate_v1.pt differ diff --git a/nadirclaw/models/complex_gate_v2.pt b/nadirclaw/models/complex_gate_v2.pt new file mode 100644 index 0000000..4c0a888 Binary files /dev/null and b/nadirclaw/models/complex_gate_v2.pt differ diff --git a/nadirclaw/models/complex_gate_v3.pt b/nadirclaw/models/complex_gate_v3.pt new file mode 100644 index 0000000..273a2cd Binary files /dev/null and b/nadirclaw/models/complex_gate_v3.pt differ diff --git a/nadirclaw/models/wide_deep_v3.pt b/nadirclaw/models/wide_deep_v3.pt new file mode 100644 index 0000000..c8ad59e Binary files /dev/null and b/nadirclaw/models/wide_deep_v3.pt differ diff --git a/nadirclaw/wide_deep_classifier.py b/nadirclaw/wide_deep_classifier.py index cabee07..a7c33fa 100644 --- a/nadirclaw/wide_deep_classifier.py +++ b/nadirclaw/wide_deep_classifier.py @@ -65,6 +65,11 @@ _MODELS_DIR = os.path.join(_PKG_DIR, "models") _MODEL_PATH_ASYM = os.path.join(_MODELS_DIR, "wide_deep_asym_v3.pt") _MODEL_PATH_SYM = os.path.join(_MODELS_DIR, "wide_deep_sym_v3.pt") +# v3 = the clean retrain (the 0.22 default). Unlike asym (which suppresses the +# simple class) it keeps a real P(simple), so it can drive the below-gate split. +_MODEL_PATH_V3 = os.path.join(_MODELS_DIR, "wide_deep_v3.pt") +# Neyman-Pearson complex gate + medium/simple companion (shipped default: v1). +_GATE_PATH = os.getenv("NADIR_GATE_PATH", os.path.join(_MODELS_DIR, "complex_gate_v1.pt")) _TIER_MAP: Dict[int, str] = {0: "simple", 1: "medium", 2: "complex"} _TIER_NUM: Dict[str, int] = {"simple": 1, "medium": 2, "complex": 3} @@ -226,11 +231,12 @@ class WideDeepClassifier: Parameters ---------- checkpoint_variant - ``"asym"`` (default, matches MODEL_CARD numbers) or ``"symmetric"`` - (recovers correct simple-class behaviour under argmax decoding). + ``"v3"`` (default since 0.22 — the clean retrain, driven by the + complex gate + head medium/simple split, aka "v3+gate"), ``"asym"`` + (the original, suppresses the simple class), or ``"symmetric"``. decision_rule - ``"argmax"`` or ``"cost_sensitive"``. Pair the ``asym`` checkpoint - with ``cost_sensitive`` for production-style behaviour. + ``"argmax"`` or ``"cost_sensitive"``. Ignored when the complex gate + is active (the gate replaces the 3-class decode). cost_lambda Downgrade penalty multiplier for cost-sensitive decoding. 3 is balanced; 20 is max-safe. @@ -243,7 +249,7 @@ class WideDeepClassifier: def __init__( self, - checkpoint_variant: str = "asym", + checkpoint_variant: str = "v3", decision_rule: str = "argmax", cost_lambda: float = 3.0, model_path: Optional[str] = None, @@ -251,9 +257,9 @@ def __init__( import torch import numpy as np - if checkpoint_variant not in ("asym", "symmetric"): + if checkpoint_variant not in ("asym", "symmetric", "v3"): raise ValueError( - f"checkpoint_variant must be 'asym' or 'symmetric', got {checkpoint_variant!r}" + f"checkpoint_variant must be 'asym', 'symmetric' or 'v3', got {checkpoint_variant!r}" ) if decision_rule not in ("argmax", "cost_sensitive"): raise ValueError( @@ -268,6 +274,8 @@ def __init__( if model_path is not None: path = model_path + elif checkpoint_variant == "v3": + path = _MODEL_PATH_V3 elif checkpoint_variant == "symmetric": path = _MODEL_PATH_SYM else: @@ -294,6 +302,37 @@ def __init__( self._encoder_name = ckpt.get("encoder", "BAAI/bge-base-en-v1.5") self.model_path = path + # ---- Neyman-Pearson complex gate ("v3+gate", the 0.22 default) ------- + # The gate decides complex-vs-rest at high recall; the below-gate + # simple/medium split runs through the v3 head (NADIR_MS_SPLIT=head, + # default — more accurate than the companion LR) or the companion + # (NADIR_MS_SPLIT=companion). Threshold τ defaults to 0.12; the gate is + # ON by default for the v3 variant (disable with NADIR_COMPLEX_GATE=0) + # and OFF for asym/symmetric so their legacy argmax behaviour is intact. + self._ms_split = os.getenv("NADIR_MS_SPLIT", "head").lower() + if self._ms_split not in ("head", "companion"): + self._ms_split = "head" + self._gate = None + _gate_default = "1" if checkpoint_variant == "v3" else "0" + if os.getenv("NADIR_COMPLEX_GATE", _gate_default) == "1": + try: + g = torch.load(_GATE_PATH, map_location="cpu", weights_only=False) + thr = float(os.getenv("NADIR_GATE_THRESHOLD", "0.12")) + self._gate = { + "coef": np.asarray(g["gate_coef"], dtype=np.float32).ravel(), + "intercept": float(np.asarray(g["gate_intercept"]).ravel()[0]), + "b_coef": np.asarray(g["below_coef"], dtype=np.float32).ravel(), + "b_intercept": float(np.asarray(g["below_intercept"]).ravel()[0]), + "threshold": thr, + # medium/simple companion threshold (only used when ms_split=companion) + "b_threshold": float(g.get("below_threshold", 0.5)), + } + logger.info("complex gate ENABLED (threshold %.4f, ms_split=%s)", + thr, self._ms_split) + except Exception as e: # pragma: no cover — fall back to plain decode + logger.error("complex gate requested but failed to load (%s); " + "continuing WITHOUT the gate", e) + logger.info( "WideDeepClassifier loaded (variant=%s, encoder=%s, struct_dim=%d, " "trained_lambda=%.1f, rule=%s, lambda=%.1f)", @@ -310,7 +349,7 @@ def __init__( # ------------------------------------------------------------------ def _predict_proba( self, prompt: str, system_prompt: str = "" - ) -> Tuple["np.ndarray", int]: # type: ignore[name-defined] + ) -> Tuple["np.ndarray", int, "np.ndarray"]: # type: ignore[name-defined] import numpy as np import torch @@ -343,6 +382,11 @@ def _predict_proba( struct_s = (struct_vec - self._struct_mean) / self._struct_scale struct_s = struct_s.reshape(1, -1) + # Gate features: [bge_normalized | struct RAW nan_to_num] (NOT scaled) — + # the complex gate was trained on the raw concat, not the model's scaled + # struct vector. + gate_x = np.concatenate([emb.ravel(), np.nan_to_num(struct_vec).ravel()]) + with torch.no_grad(): logits = self._model( torch.from_numpy(emb), @@ -351,7 +395,7 @@ def _predict_proba( probs = torch.softmax(logits, dim=1).cpu().numpy()[0] latency_ms = int((time.time() - t0) * 1000) - return probs, latency_ms + return probs, latency_ms, gate_x def classify( self, prompt: str, system_prompt: str = "" @@ -359,15 +403,29 @@ def classify( """Classify a single prompt; returns a :class:`ClassificationResult`.""" import numpy as np - probs, latency_ms = self._predict_proba(prompt, system_prompt=system_prompt) - - if self.decision_rule == "cost_sensitive": + probs, latency_ms, gate_x = self._predict_proba(prompt, system_prompt=system_prompt) + + if self._gate is not None: + # Neyman-Pearson gate: keep-on-complex when P(complex) >= threshold; + # below the gate, the v3 head (default) or the companion LR splits + # medium vs simple. This REPLACES the 3-class decode. + z = float(gate_x @ self._gate["coef"] + self._gate["intercept"]) + p_complex = 1.0 / (1.0 + np.exp(-z)) + if p_complex >= self._gate["threshold"]: + pred_idx = 2 + elif self._ms_split == "companion": + zb = float(gate_x @ self._gate["b_coef"] + self._gate["b_intercept"]) + pred_idx = 1 if (1.0 / (1.0 + np.exp(-zb))) >= self._gate["b_threshold"] else 0 + else: # "head" (v3+gate default): v3 head splits medium vs simple + pred_idx = 1 if probs[1] >= probs[0] else 0 + elif self.decision_rule == "cost_sensitive": # E[cost | j] = Σ_i P(i) * C[i, j] → argmin_j expected_cost = probs @ self._cost pred_idx = int(np.argmin(expected_cost)) else: pred_idx = int(np.argmax(probs)) + rule = "complex_gate" if self._gate is not None else self.decision_rule tier = _TIER_MAP[pred_idx] # Continuous-score adapter for the N-tier selector. Inlined # rather than imported to keep this module dependency-free at @@ -386,7 +444,7 @@ def classify( "complex": float(probs[2]), }, argmax_tier=_TIER_MAP[int(np.argmax(probs))], - decision_rule=self.decision_rule, + decision_rule=rule, cost_lambda=self.cost_lambda, latency_ms=latency_ms, classifier_version=self.ANALYZER_VERSION, @@ -405,7 +463,7 @@ def classify_tuple( # Public accessors # --------------------------------------------------------------------------- def get_wide_deep_classifier( - checkpoint_variant: str = "asym", + checkpoint_variant: str = "v3", decision_rule: str = "argmax", cost_lambda: float = 3.0, model_path: Optional[str] = None, @@ -446,6 +504,8 @@ def bundled_model_paths() -> Dict[str, str]: return { "asym": _MODEL_PATH_ASYM, "symmetric": _MODEL_PATH_SYM, + "v3": _MODEL_PATH_V3, + "gate": _GATE_PATH, } diff --git a/tests/test_wide_deep_classifier.py b/tests/test_wide_deep_classifier.py index f1acc3f..6fd918e 100644 --- a/tests/test_wide_deep_classifier.py +++ b/tests/test_wide_deep_classifier.py @@ -35,15 +35,17 @@ def test_bundled_paths_exist(): """Both checkpoints must be present on disk after a normal install.""" paths = bundled_model_paths() - assert set(paths) == {"asym", "symmetric"} + assert set(paths) == {"asym", "symmetric", "v3", "gate"} for variant, path in paths.items(): assert os.path.exists(path), ( f"Bundled {variant!r} checkpoint missing at {path}. " f"Check pyproject.toml [tool.setuptools.package-data]." ) - # Sanity check: each shipped checkpoint is roughly 900 KB. + # Sanity check: the W&D checkpoints are ~900 KB; the gate is a small + # logistic (~12 KB). size = os.path.getsize(path) - assert 100_000 < size < 5_000_000, ( + lo = 1_000 if variant == "gate" else 100_000 + assert lo < size < 5_000_000, ( f"Checkpoint {variant!r} at {path} has suspicious size: {size} bytes" ) @@ -151,3 +153,69 @@ def test_invalid_args(): def test_missing_checkpoint_path(): with pytest.raises(FileNotFoundError): WideDeepClassifier(model_path="/nonexistent/no_such_checkpoint.pt") + + +# --------------------------------------------------------------------------- +# v3+gate — the 0.22 default: v3 head + Neyman-Pearson complex gate + head +# medium/simple split at τ=0.12. Gate is ON by default for the v3 variant. +# --------------------------------------------------------------------------- +def test_default_variant_is_v3_with_gate(monkeypatch): + """The shipped default is v3 with the complex gate enabled at τ=0.12.""" + monkeypatch.delenv("NADIR_COMPLEX_GATE", raising=False) + monkeypatch.delenv("NADIR_GATE_THRESHOLD", raising=False) + monkeypatch.delenv("NADIR_MS_SPLIT", raising=False) + clf = WideDeepClassifier() # no args → v3 + assert clf.checkpoint_variant == "v3" + assert clf._gate is not None + assert clf._ms_split == "head" + assert abs(clf._gate["threshold"] - 0.12) < 1e-9 + + +def test_v3gate_routes_trivial_to_simple(monkeypatch): + """The v3 head keeps a real P(simple), so a greeting routes to simple — + the whole point of v3+gate over the asym checkpoint (which can't).""" + monkeypatch.delenv("NADIR_COMPLEX_GATE", raising=False) + clf = WideDeepClassifier(checkpoint_variant="v3") + r = clf.classify("hi") + assert r.tier == "simple" + assert r.decision_rule == "complex_gate" + + +def test_v3gate_routes_hard_prompt_to_complex(monkeypatch): + monkeypatch.delenv("NADIR_COMPLEX_GATE", raising=False) + clf = WideDeepClassifier(checkpoint_variant="v3") + r = clf.classify( + "Design a horizontally-scalable, exactly-once payment ledger across " + "three regions; discuss consensus, clock skew, and idempotency in depth." + ) + assert r.tier == "complex" + + +def test_gate_can_be_disabled(monkeypatch): + """NADIR_COMPLEX_GATE=0 falls back to the plain 3-class decode.""" + monkeypatch.setenv("NADIR_COMPLEX_GATE", "0") + clf = WideDeepClassifier(checkpoint_variant="v3") + assert clf._gate is None + r = clf.classify("hi") + assert r.decision_rule == "argmax" + + +def test_gate_off_by_default_for_asym(monkeypatch): + """asym/symmetric keep their legacy argmax behaviour (gate off by default).""" + monkeypatch.delenv("NADIR_COMPLEX_GATE", raising=False) + clf = WideDeepClassifier(checkpoint_variant="asym") + assert clf._gate is None + + +def test_ms_split_companion_rollback(monkeypatch): + monkeypatch.delenv("NADIR_COMPLEX_GATE", raising=False) + monkeypatch.setenv("NADIR_MS_SPLIT", "companion") + clf = WideDeepClassifier(checkpoint_variant="v3") + assert clf._ms_split == "companion" + assert clf._gate is not None + + +def test_gate_threshold_override(monkeypatch): + monkeypatch.setenv("NADIR_GATE_THRESHOLD", "0.30") + clf = WideDeepClassifier(checkpoint_variant="v3") + assert abs(clf._gate["threshold"] - 0.30) < 1e-9