From e957b941bfd003a71ec2a2a48e1d02c9a407e771 Mon Sep 17 00:00:00 2001 From: Nadir Date: Tue, 30 Jun 2026 07:34:24 -0400 Subject: [PATCH] feat: Morph Model Router as an opt-in complexity classifier (#68) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds NADIRCLAW_COMPLEXITY_ANALYZER=morph, delegating the simple/mid/complex routing decision to Morph's hosted Model Router. Implemented per the maintainer triage on #68: - Strictly opt-in: default stays the local binary classifier; morph requires MORPH_API_KEY (missing key degrades to binary at selection time). - Fail-closed to local: on missing key, HTTP error, timeout, or unparseable response, log once and serve that request from BinaryComplexityClassifier. - Latency budget: MORPH_TIMEOUT_MS (default 200ms) + in-memory LRU on prompt hash for repeated CLI prompts. - Tier mapping reuses BinaryComplexityClassifier._score_to_tier so the same NADIRCLAW_TIER_THRESHOLDS apply and `mid` only appears when MID_MODEL is set. - Stdlib-only HTTP (urllib.request) — no new dependency; fully monkeypatched in tests (success, fail-closed, parsing, caching, dispatch). No live API in CI. Cost-tracking honesty (subtracting the per-classification Morph fee from net-saved in `nadirclaw report`) is deferred as a follow-up, noted in the README, pending Morph's published classification price. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 21 +++- nadirclaw/classifier.py | 23 ++++ nadirclaw/morph_classifier.py | 204 +++++++++++++++++++++++++++++++++ nadirclaw/settings.py | 30 +++++ tests/test_morph_classifier.py | 183 +++++++++++++++++++++++++++++ 5 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 nadirclaw/morph_classifier.py create mode 100644 tests/test_morph_classifier.py diff --git a/README.md b/README.md index 92ba87e..e5405db 100644 --- a/README.md +++ b/README.md @@ -369,12 +369,13 @@ Gemini models are called natively via the Google GenAI SDK. All other models go ### Complexity analyzer -NadirClaw ships two prompt classifiers. Pick one with `NADIRCLAW_COMPLEXITY_ANALYZER`: +NadirClaw ships two local prompt classifiers, plus an optional remote one. Pick one with `NADIRCLAW_COMPLEXITY_ANALYZER`: | Value | Model | Latency | Size | Output | |---|---|---|---|---| | `binary` *(default)* | Sentence-embedding centroid classifier | ~10ms | ~22MB | 2-class (simple / complex) — `mid` and `reasoning` come from rule overlays | | `distilbert` | Fine-tuned DistilBERT sequence classifier | ~30ms | ~256MB | 3-class (simple / mid / complex) predicted natively | +| `morph` | [Morph hosted Model Router](https://docs.morphllm.com/sdk/components/router) (remote) | ~50ms + network | — | difficulty → simple / mid / complex via the same tier thresholds | ```bash # opt into the 3-class DistilBERT classifier @@ -383,6 +384,24 @@ NADIRCLAW_COMPLEXITY_ANALYZER=distilbert The DistilBERT artifact is **not** bundled in the package — on first use it downloads (~256MB, then cached under `~/.cache/huggingface/hub/`) from the Hugging Face Hub. Override the source repo with `NADIRCLAW_DISTILBERT_REPO`. If the download fails, NadirClaw logs a warning and falls back to the binary classifier — it never crashes the router. +#### Morph router (remote, opt-in) + +`morph` delegates the routing decision to Morph's hosted Model Router instead of a local model. It is **strictly opt-in** and requires `MORPH_API_KEY`: + +```bash +NADIRCLAW_COMPLEXITY_ANALYZER=morph MORPH_API_KEY=sk-... nadirclaw serve +``` + +Morph returns a `difficulty` (`easy` / `medium` / `hard` / `needs_info`), which NadirClaw maps to a complexity score and then to a tier using the same `NADIRCLAW_TIER_THRESHOLDS` as the local classifiers (so `mid` only appears when `NADIRCLAW_MID_MODEL` is set). Tuning knobs: + +| Env var | Default | Purpose | +|---|---|---| +| `MORPH_API_KEY` | — | Required. Without it, routing falls back to `binary`. | +| `MORPH_API_BASE` | `https://api.morphllm.com/v1` | Override for self-hosted/proxy gateways. | +| `MORPH_TIMEOUT_MS` | `200` | Per-request budget; past it, fail closed to the local classifier. | + +**Fail-closed:** on a missing key, HTTP error, timeout, or unparseable response, NadirClaw logs once and serves that request from the local binary classifier — a Morph outage degrades routing quality, never availability. Repeated prompts within a session are served from a small in-memory cache. Note: the Morph call adds a small per-classification cost that NadirClaw's savings report does not yet subtract from net-saved (tracked as a follow-up). + Test how any prompt buckets with either analyzer: ```bash diff --git a/nadirclaw/classifier.py b/nadirclaw/classifier.py index 6ab8917..9bec329 100644 --- a/nadirclaw/classifier.py +++ b/nadirclaw/classifier.py @@ -356,6 +356,17 @@ def get_classifier() -> Any: "Failed to load DistilBERT classifier (%s) — falling back to binary", e, ) + elif kind == "morph": + try: + from nadirclaw.morph_classifier import MorphRouterClassifier + _active_classifier = MorphRouterClassifier() + logger.info("Using Morph router classifier (remote, fail-closed to binary)") + return _active_classifier + except Exception as e: + logger.warning( + "Failed to init Morph router classifier (%s) — falling back to binary", + e, + ) _active_classifier = get_binary_classifier() return _active_classifier @@ -378,6 +389,18 @@ def warmup() -> None: "DistilBERT warmup failed (%s) — falling back to binary classifier", e, ) + elif kind == "morph": + try: + from nadirclaw.morph_classifier import MorphRouterClassifier + logger.info("Warming up MorphRouterClassifier (remote) ...") + _active_classifier = MorphRouterClassifier() + logger.info("MorphRouterClassifier warmup complete") + return + except Exception as e: + logger.warning( + "Morph warmup failed (%s) — falling back to binary classifier", + e, + ) logger.info("Warming up BinaryComplexityClassifier ...") _singleton = BinaryComplexityClassifier() _active_classifier = _singleton diff --git a/nadirclaw/morph_classifier.py b/nadirclaw/morph_classifier.py new file mode 100644 index 0000000..01cadbd --- /dev/null +++ b/nadirclaw/morph_classifier.py @@ -0,0 +1,204 @@ +"""Morph Model Router complexity classifier. + +An optional, opt-in classifier that delegates the simple/mid/complex routing +decision to Morph's hosted Model Router (https://docs.morphllm.com/sdk/components/router) +instead of the local centroid/DistilBERT classifiers. + +Design constraints (see issue #68): + + - **Strictly opt-in.** Activated only with ``NADIRCLAW_COMPLEXITY_ANALYZER=morph`` + and a ``MORPH_API_KEY``. The default classifier stays the local binary one; + a remote round-trip is never a hard dependency for first-run UX. + - **Fail-closed to local.** On a missing key, HTTP error, timeout, or an + unparseable response, we log once and fall back to the local + ``BinaryComplexityClassifier`` for that request. A transient Morph outage + degrades quality, never availability. + - **Latency budget.** ``MORPH_TIMEOUT_MS`` (default 200ms) bounds the call. + Repeated prompts in a CLI session are served from a small in-memory LRU. + +Only the standard library is used for the HTTP call (``urllib.request``) so the +classifier adds no new dependency and is trivially monkeypatchable in tests. +""" + +import hashlib +import json +import logging +import os +import time +import urllib.error +import urllib.request +from collections import OrderedDict +from typing import Any, Dict, Optional, Tuple + +logger = logging.getLogger(__name__) + +# Difficulty bucket -> complexity score. The score (not the bucket) is mapped to +# a tier by BinaryComplexityClassifier._score_to_tier, so the same configurable +# NADIRCLAW_TIER_THRESHOLDS govern Morph and the local classifier alike, and the +# "mid" tier only appears when MID_MODEL is configured. +_DIFFICULTY_SCORE = { + "easy": 0.15, + "medium": 0.55, + "hard": 0.85, + "needs_info": 0.92, +} + +# Module-level guards so the fail-closed fallback only logs once per process. +_warned_fallback = False + + +def _warn_once(message: str, *args: Any) -> None: + global _warned_fallback + if not _warned_fallback: + logger.warning(message, *args) + _warned_fallback = True + + +class MorphRouterClassifier: + """Routes via Morph's Model Router, falling back to the local classifier.""" + + CLASSIFIER_VERSION = "1.0" + + def __init__(self): + from nadirclaw.settings import settings + + self._api_key = settings.MORPH_API_KEY + if not self._api_key: + # Raise so get_classifier() degrades to binary at selection time. + raise ValueError( + "NADIRCLAW_COMPLEXITY_ANALYZER=morph requires MORPH_API_KEY. " + "Set it or unset NADIRCLAW_COMPLEXITY_ANALYZER to use the local " + "classifier." + ) + + self._api_base = settings.MORPH_API_BASE.rstrip("/") + self._timeout_s = max(0.001, settings.MORPH_TIMEOUT_MS / 1000.0) + self._cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + self._cache_cap = 512 + self._fallback: Optional[Any] = None # lazily-built BinaryComplexityClassifier + + logger.info( + "MorphRouterClassifier ready (base=%s, timeout=%dms)", + self._api_base, + settings.MORPH_TIMEOUT_MS, + ) + + # ------------------------------------------------------------------ + # Remote call + # ------------------------------------------------------------------ + + def _call_morph(self, prompt: str) -> Dict[str, Any]: + """POST the prompt to Morph's router. Raises on any transport error.""" + url = f"{self._api_base}/router/classify" + body = json.dumps({"input": prompt}).encode("utf-8") + req = urllib.request.Request( + url, + data=body, + headers={ + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=self._timeout_s) as resp: + return json.loads(resp.read().decode("utf-8")) + + @staticmethod + def _parse_response(payload: Dict[str, Any]) -> Tuple[str, float]: + """Extract (difficulty, confidence) from a Morph response. + + Tolerant of the response being either flat or nested under a + ``classification`` key. ``confidence`` is the model's confidence when + present, otherwise ``1 - ambiguity``, otherwise 1.0. + """ + inner = payload.get("classification", payload) + + difficulty = inner.get("difficulty") + if not isinstance(difficulty, str): + raise ValueError(f"Morph response missing 'difficulty': {payload!r}") + difficulty = difficulty.strip().lower() + if difficulty not in _DIFFICULTY_SCORE: + raise ValueError(f"Unknown Morph difficulty {difficulty!r}") + + confidence = inner.get("confidence") + if not isinstance(confidence, (int, float)): + ambiguity = inner.get("ambiguity") + if isinstance(ambiguity, (int, float)): + confidence = 1.0 - float(ambiguity) + else: + confidence = 1.0 + confidence = max(0.0, min(1.0, float(confidence))) + + return difficulty, confidence + + # ------------------------------------------------------------------ + # Fail-closed local fallback + # ------------------------------------------------------------------ + + async def _local_fallback(self, text: str, system_message: str) -> Dict[str, Any]: + if self._fallback is None: + from nadirclaw.classifier import get_binary_classifier + self._fallback = get_binary_classifier() + result = await self._fallback.analyze(text=text, system_message=system_message) + result["analyzer_type"] = "morph-fallback-binary" + return result + + # ------------------------------------------------------------------ + # Public interface + # ------------------------------------------------------------------ + + async def analyze(self, text: str = "", system_message: str = "", **kwargs) -> Dict[str, Any]: + """Async-compatible interface matching the server's expected API.""" + cache_key = hashlib.sha256(text.encode("utf-8")).hexdigest() + if cache_key in self._cache: + self._cache.move_to_end(cache_key) + return dict(self._cache[cache_key]) + + start = time.time() + try: + payload = self._call_morph(text) + difficulty, confidence = self._parse_response(payload) + except (urllib.error.URLError, ValueError, OSError, json.JSONDecodeError, TimeoutError) as e: + _warn_once( + "Morph router unavailable (%s) — falling back to local binary " + "classifier. Further failures are suppressed.", + e, + ) + return await self._local_fallback(text, system_message) + + from nadirclaw.classifier import BinaryComplexityClassifier + + complexity_score = _DIFFICULTY_SCORE[difficulty] + tier_name, tier = BinaryComplexityClassifier._score_to_tier(complexity_score) + recommended_model, recommended_provider = ( + BinaryComplexityClassifier._select_model_by_tier(tier_name) + ) + latency_ms = int((time.time() - start) * 1000) + + result = { + "recommended_model": recommended_model, + "recommended_provider": recommended_provider, + "confidence": confidence, + "complexity_score": complexity_score, + "complexity_tier": tier, + "complexity_name": tier_name, + "tier": tier, + "tier_name": tier_name, + "reasoning": ( + f"Morph router: difficulty={difficulty} -> {tier_name} " + f"(score={complexity_score:.2f}, confidence={confidence:.0%})" + ), + "ranked_models": [], + "analyzer_latency_ms": latency_ms, + "analyzer_type": f"morph-v{self.CLASSIFIER_VERSION}", + "selection_method": "morph_router", + "model_type": "morph_router", + "morph_difficulty": difficulty, + } + + self._cache[cache_key] = dict(result) + self._cache.move_to_end(cache_key) + if len(self._cache) > self._cache_cap: + self._cache.popitem(last=False) + + return result diff --git a/nadirclaw/settings.py b/nadirclaw/settings.py index 604b608..66071d5 100644 --- a/nadirclaw/settings.py +++ b/nadirclaw/settings.py @@ -170,6 +170,9 @@ def COMPLEXITY_ANALYZER_TYPE(self) -> str: - "binary" (default) — fast 2-class centroid classifier (~10ms, 22MB) - "distilbert" — 3-class fine-tuned DistilBERT (~30ms, 256MB) produces simple/mid/complex tiers natively. + - "morph" — Morph hosted Model Router (remote). Requires + MORPH_API_KEY. Fails closed to "binary" on any + error/timeout/missing key. Set via NADIRCLAW_COMPLEXITY_ANALYZER. The DistilBERT artifact is not shipped in the package — install separately or train via @@ -177,6 +180,33 @@ def COMPLEXITY_ANALYZER_TYPE(self) -> str: """ return os.getenv("NADIRCLAW_COMPLEXITY_ANALYZER", "binary").strip().lower() + @property + def MORPH_API_KEY(self) -> str: + """API key for the Morph router classifier (NADIRCLAW_COMPLEXITY_ANALYZER=morph). + + Read from MORPH_API_KEY. Empty means the morph classifier is unavailable + and routing falls back to the local binary classifier. + """ + return os.getenv("MORPH_API_KEY", "").strip() + + @property + def MORPH_API_BASE(self) -> str: + """Base URL for Morph's OpenAI-style router API (no trailing /router/...).""" + return os.getenv("MORPH_API_BASE", "https://api.morphllm.com/v1").strip() + + @property + def MORPH_TIMEOUT_MS(self) -> int: + """Per-request timeout budget for the Morph router call (default 200ms). + + A CLI on residential internet sees tail latency well above Morph's ~50ms + median; past this budget we fail closed to the local classifier rather + than stall the request. + """ + try: + return max(1, int(os.getenv("MORPH_TIMEOUT_MS", "200"))) + except ValueError: + return 200 + @property def FALLBACK_CHAIN(self) -> list[str]: """Ordered fallback chain. When a model fails, try the next one. diff --git a/tests/test_morph_classifier.py b/tests/test_morph_classifier.py new file mode 100644 index 0000000..44ce23e --- /dev/null +++ b/tests/test_morph_classifier.py @@ -0,0 +1,183 @@ +"""Tests for nadirclaw.morph_classifier — Morph router classifier (issue #68). + +No test hits the live Morph API; the HTTP layer is monkeypatched. Covers the +success path, the fail-closed-to-binary path, response parsing, caching, and +the get_classifier() dispatch + degradation when no key is set. +""" + +import json +import urllib.error + +import pytest + + +def _fake_response(payload: dict): + """Build a context-manager stand-in for urllib.request.urlopen().""" + class _Resp: + def __enter__(self_): + return self_ + def __exit__(self_, *a): + return False + def read(self_): + return json.dumps(payload).encode("utf-8") + return _Resp() + + +@pytest.fixture(autouse=True) +def _reset_state(monkeypatch): + """Each test starts with a clean key, base, and warn-once guard.""" + monkeypatch.setenv("MORPH_API_KEY", "test-key") + monkeypatch.delenv("MORPH_API_BASE", raising=False) + monkeypatch.delenv("MORPH_TIMEOUT_MS", raising=False) + import nadirclaw.morph_classifier as m + m._warned_fallback = False + import nadirclaw.classifier as c + c._active_classifier = None + yield + c._active_classifier = None + + +class TestParseResponse: + def test_difficulty_and_confidence(self): + from nadirclaw.morph_classifier import MorphRouterClassifier + diff, conf = MorphRouterClassifier._parse_response( + {"difficulty": "hard", "confidence": 0.9} + ) + assert diff == "hard" + assert conf == pytest.approx(0.9) + + def test_confidence_from_ambiguity(self): + from nadirclaw.morph_classifier import MorphRouterClassifier + diff, conf = MorphRouterClassifier._parse_response( + {"difficulty": "easy", "ambiguity": 0.25} + ) + assert diff == "easy" + assert conf == pytest.approx(0.75) + + def test_nested_classification_key(self): + from nadirclaw.morph_classifier import MorphRouterClassifier + diff, conf = MorphRouterClassifier._parse_response( + {"classification": {"difficulty": "MEDIUM"}} + ) + assert diff == "medium" + assert conf == pytest.approx(1.0) + + def test_missing_difficulty_raises(self): + from nadirclaw.morph_classifier import MorphRouterClassifier + with pytest.raises(ValueError): + MorphRouterClassifier._parse_response({"domain": "code"}) + + def test_unknown_difficulty_raises(self): + from nadirclaw.morph_classifier import MorphRouterClassifier + with pytest.raises(ValueError): + MorphRouterClassifier._parse_response({"difficulty": "trivial"}) + + +class TestAnalyzeSuccess: + @pytest.mark.asyncio + async def test_hard_routes_complex(self, monkeypatch): + import nadirclaw.morph_classifier as m + monkeypatch.setattr( + m.urllib.request, "urlopen", + lambda *a, **k: _fake_response({"difficulty": "hard", "confidence": 0.95}), + ) + clf = m.MorphRouterClassifier() + result = await clf.analyze(text="Design a consensus protocol") + assert result["tier_name"] == "complex" + assert result["analyzer_type"].startswith("morph-v") + assert result["morph_difficulty"] == "hard" + assert 0.0 <= result["confidence"] <= 1.0 + + @pytest.mark.asyncio + async def test_easy_routes_simple(self, monkeypatch): + import nadirclaw.morph_classifier as m + monkeypatch.setattr( + m.urllib.request, "urlopen", + lambda *a, **k: _fake_response({"difficulty": "easy", "ambiguity": 0.1}), + ) + clf = m.MorphRouterClassifier() + result = await clf.analyze(text="What is 2+2?") + assert result["tier_name"] == "simple" + + @pytest.mark.asyncio + async def test_response_is_cached(self, monkeypatch): + import nadirclaw.morph_classifier as m + calls = {"n": 0} + + def _urlopen(*a, **k): + calls["n"] += 1 + return _fake_response({"difficulty": "hard", "confidence": 0.9}) + + monkeypatch.setattr(m.urllib.request, "urlopen", _urlopen) + clf = m.MorphRouterClassifier() + await clf.analyze(text="same prompt") + await clf.analyze(text="same prompt") + assert calls["n"] == 1 # second call served from cache + + +class TestFailClosed: + @pytest.mark.asyncio + async def test_http_error_falls_back_to_binary(self, monkeypatch): + import nadirclaw.morph_classifier as m + + def _boom(*a, **k): + raise urllib.error.URLError("connection refused") + + monkeypatch.setattr(m.urllib.request, "urlopen", _boom) + clf = m.MorphRouterClassifier() + result = await clf.analyze(text="What is Python?") + # Degrades, doesn't crash, and is clearly labelled as the fallback. + assert result["analyzer_type"] == "morph-fallback-binary" + assert result["tier_name"] in ("simple", "mid", "complex") + + @pytest.mark.asyncio + async def test_timeout_falls_back_to_binary(self, monkeypatch): + import nadirclaw.morph_classifier as m + + def _slow(*a, **k): + raise TimeoutError("timed out") + + monkeypatch.setattr(m.urllib.request, "urlopen", _slow) + clf = m.MorphRouterClassifier() + result = await clf.analyze(text="What is Python?") + assert result["analyzer_type"] == "morph-fallback-binary" + + @pytest.mark.asyncio + async def test_bad_json_falls_back_to_binary(self, monkeypatch): + import nadirclaw.morph_classifier as m + + class _BadResp: + def __enter__(self_): + return self_ + def __exit__(self_, *a): + return False + def read(self_): + return b"not json" + + monkeypatch.setattr(m.urllib.request, "urlopen", lambda *a, **k: _BadResp()) + clf = m.MorphRouterClassifier() + result = await clf.analyze(text="hi") + assert result["analyzer_type"] == "morph-fallback-binary" + + +class TestDispatch: + def test_missing_key_degrades_to_binary(self, monkeypatch): + monkeypatch.delenv("MORPH_API_KEY", raising=False) + monkeypatch.setenv("NADIRCLAW_COMPLEXITY_ANALYZER", "morph") + import nadirclaw.classifier as c + c._active_classifier = None + from nadirclaw.classifier import get_classifier, BinaryComplexityClassifier + clf = get_classifier() + assert isinstance(clf, BinaryComplexityClassifier) + c._active_classifier = None + + def test_with_key_selects_morph(self, monkeypatch): + monkeypatch.setenv("MORPH_API_KEY", "test-key") + monkeypatch.setenv("NADIRCLAW_COMPLEXITY_ANALYZER", "morph") + import nadirclaw.classifier as c + c._active_classifier = None + from nadirclaw.classifier import get_classifier + from nadirclaw.morph_classifier import MorphRouterClassifier + clf = get_classifier() + assert isinstance(clf, MorphRouterClassifier) + c._active_classifier = None