diff --git a/.env.example b/.env.example index d11b9bb..65ce05f 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,7 @@ # Default model to use (must exist in providers.json). # Ollama examples: "gemma4:latest", "qwen3:4b", "mistral:7b" # Gemini examples: "gemini-2.5-pro", "gemini-2.5-flash" +# Claude Agent SDK examples: "sonnet", "opus", "fable" (no API key; uses local Claude Code login) DEFAULT_MODEL=gemma4:latest # API keys — only needed for providers whose api_key_env is set in providers.json. diff --git a/README.md b/README.md index 83aff47..21f52ac 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ $ cp .env.example .env | `GEMINI_API_KEY` | string | Required when using a Gemini model. | | `GITHUB_TOKEN` | optional | Inherits from your shell environment, improves GitHub API rate limits. | -Provider mapping lives in `providers.json` — each provider declares its `base_url`, an optional API-key env var, and per-model parameters; `config.py` loads it and resolves the provider for a model. `config.py` also has a flag: +Provider mapping lives in `providers.json` — each provider declares its `base_url`, an optional API-key env var, and per-model parameters; `config.py` loads it and resolves the provider for a model. A provider may instead set `"provider_type": "claude_agent_sdk"` to use the local Claude Agent SDK (no `base_url`/`api_key`); the default `provider_type` is `openai_compatible`. `config.py` also has a flag: ```python # config.py @@ -312,6 +312,14 @@ What happens: - Provide `GEMINI_API_KEY` - The same `models.OpenAICompatibleProvider` wrapper is used, pointed at Gemini's OpenAI-compatible endpoint +### Claude Agent SDK + +- Set `DEFAULT_MODEL` to a Claude Agent model listed in `providers.json`, for example `sonnet`, `opus`, or `fable` +- No API key required — it uses your local [Claude Code](https://claude.com/claude-code) authentication +- Install the optional dependency: `pip install claude-agent-sdk` +- Requests go through `models.ClaudeAgentProvider`, selected via the `"provider_type": "claude_agent_sdk"` flag on the `claude_agent` entry in `providers.json` (no `base_url`/`api_key`) +- Distinct from the `anthropic` provider, which calls the Anthropic API and needs `ANTHROPIC_API_KEY` + --- ## Contributing diff --git a/config.py b/config.py index fc14cfa..bf40125 100644 --- a/config.py +++ b/config.py @@ -33,7 +33,10 @@ def provider_for(model_name: str) -> dict: """Resolve provider config for a model. - Returns {base_url, api_key, structured_output, extra_body}. + Returns {provider_type, base_url, api_key, structured_output, extra_body}. + `provider_type` selects the client: "openai_compatible" (default) hits + base_url with api_key; "claude_agent_sdk" uses the local Claude Agent SDK + and needs neither base_url nor api_key. Raises ValueError if the model is unknown or its required key is unset. """ for name, prov in _config["providers"].items(): @@ -50,8 +53,10 @@ def provider_for(model_name: str) -> dict: **prov.get("extra_body", {}), **prov["models"][model_name].get("extra_body", {}), } + base_url = prov.get("base_url") return { - "base_url": prov["base_url"].rstrip("/"), + "provider_type": prov.get("provider_type", "openai_compatible"), + "base_url": base_url.rstrip("/") if base_url else None, "api_key": api_key, "structured_output": prov.get("structured_output", "json_schema"), "extra_body": extra_body, diff --git a/evaluator.py b/evaluator.py index 3b66754..589ef92 100644 --- a/evaluator.py +++ b/evaluator.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional, Tuple, Any -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, ValidationError from models import JSONResume, EvaluationData from llm_utils import initialize_llm_provider, extract_json_from_response import logging @@ -43,47 +43,70 @@ def _load_evaluation_prompt(self, resume_text: str) -> str: raise ValueError("Failed to load resume evaluation criteria template") return criteria_template + # Retry count for when the model returns broken or off-schema JSON. + # Providers that do not enforce a schema server-side sometimes emit + # invalid JSON, and a fresh sample almost always parses. + MAX_EVALUATION_ATTEMPTS = 3 + def evaluate_resume(self, resume_text: str) -> EvaluationData: self._last_resume_text = resume_text full_prompt = self._load_evaluation_prompt(resume_text) # logger.info(f"🔤 Evaluation prompt being sent: {full_prompt}") - try: - system_message = self.template_manager.render_template( - "resume_evaluation_system_message" + + system_message = self.template_manager.render_template( + "resume_evaluation_system_message" + ) + if system_message is None: + raise ValueError( + "Failed to load resume evaluation system message template" ) - if system_message is None: - raise ValueError( - "Failed to load resume evaluation system message template" + + # Prepare chat parameters + chat_params = { + "model": self.model_name, + "messages": [ + {"role": "system", "content": system_message}, + {"role": "user", "content": full_prompt}, + ], + "options": { + "stream": False, + "temperature": self.model_params.get("temperature", 0.5), + "top_p": self.model_params.get("top_p", 0.9), + }, + } + + # Add format parameter for structured output + kwargs = {"format": EvaluationData.model_json_schema()} + + last_error: Optional[Exception] = None + for attempt in range(1, self.MAX_EVALUATION_ATTEMPTS + 1): + try: + # Use the appropriate provider to make the API call + response = self.provider.chat(**chat_params, **kwargs) + + response_text = response["message"]["content"] + response_text = extract_json_from_response(response_text) + + # Trim to the outermost JSON object so any stray prose around + # it does not break parsing. + json_start = response_text.find("{") + json_end = response_text.rfind("}") + if json_start != -1 and json_end != -1: + response_text = response_text[json_start : json_end + 1] + logger.debug(f"🔤 Prompt response: {response_text}") + + evaluation_dict = json.loads(response_text) + return EvaluationData(**evaluation_dict) + + except (json.JSONDecodeError, ValidationError) as e: + last_error = e + logger.warning( + f"🔁 Evaluation response invalid " + f"(attempt {attempt}/{self.MAX_EVALUATION_ATTEMPTS}): {e}" ) - # Prepare chat parameters - chat_params = { - "model": self.model_name, - "messages": [ - {"role": "system", "content": system_message}, - {"role": "user", "content": full_prompt}, - ], - "options": { - "stream": False, - "temperature": self.model_params.get("temperature", 0.5), - "top_p": self.model_params.get("top_p", 0.9), - }, - } - - # Add format parameter for structured output - kwargs = {"format": EvaluationData.model_json_schema()} - # Use the appropriate provider to make the API call - response = self.provider.chat(**chat_params, **kwargs) - - response_text = response["message"]["content"] - response_text = extract_json_from_response(response_text) - logger.error(f"🔤 Prompt response: {response_text}") - - evaluation_dict = json.loads(response_text) - evaluation_data = EvaluationData(**evaluation_dict) - - return evaluation_data - - except Exception as e: - logger.error(f"Error evaluating resume: {str(e)}") - raise + logger.error( + f"Error evaluating resume after " + f"{self.MAX_EVALUATION_ATTEMPTS} attempts: {last_error}" + ) + raise last_error diff --git a/llm_utils.py b/llm_utils.py index 7b00149..68d82df 100644 --- a/llm_utils.py +++ b/llm_utils.py @@ -5,7 +5,7 @@ import logging from typing import Any, Dict, Optional from config import provider_for -from models import OpenAICompatibleProvider +from models import OpenAICompatibleProvider, ClaudeAgentProvider logger = logging.getLogger(__name__) @@ -43,6 +43,12 @@ def initialize_llm_provider(model_name: str) -> Any: resolving base_url / api_key / structured-output mode from providers.json. """ cfg = provider_for(model_name) + if cfg["provider_type"] == "claude_agent_sdk": + logger.info( + f"🔄 Using model {model_name} via Claude Agent SDK " + "(local Claude Code authentication)" + ) + return ClaudeAgentProvider() logger.info(f"🔄 Using model {model_name} via {cfg['base_url']}") return OpenAICompatibleProvider( base_url=cfg["base_url"], diff --git a/models.py b/models.py index 73f64ab..5f22984 100644 --- a/models.py +++ b/models.py @@ -1,3 +1,5 @@ +import json + from typing import List, Optional, Dict, Tuple, Any, Protocol, runtime_checkable from pydantic import BaseModel, Field, field_validator @@ -359,3 +361,129 @@ def chat( except (KeyError, IndexError, TypeError): raise ValueError(f"Unexpected response shape from {url}: {data}") return {"message": {"role": "assistant", "content": content}} + + +class ClaudeAgentProvider: + """Claude Agent SDK provider (uses local Claude Code authentication). + + Unlike OpenAICompatibleProvider, this does not use an API key or base_url. + It relies on the Claude Agent SDK, which authenticates through a local + Claude Code login. Install the optional dependency with: + + pip install claude-agent-sdk + + Adapts the response to the {"message": {"content": ...}} shape the + evaluator expects. + """ + + def chat( + self, + model: str, + messages: List[Dict[str, str]], + options: Dict[str, Any] = None, + **kwargs + ) -> Dict[str, Any]: + """Send a chat request to Claude Agent. `options`/`format` are ignored; + structured output is driven by the prompt and parsed downstream.""" + import asyncio + + return asyncio.run(self._chat_claude_async(model, messages, **kwargs)) + + async def _chat_claude_async( + self, + model: str, + messages: List[Dict[str, str]], + **kwargs + ) -> Dict[str, Any]: + # Imported lazily so the base install does not require claude-agent-sdk. + try: + from claude_agent_sdk import ( + query, + ClaudeAgentOptions, + AssistantMessage, + ResultMessage, + TextBlock, + ) + except ImportError as e: + raise ImportError( + "claude_agent_sdk is required for the Claude Agent provider. " + "Install it with: pip install claude-agent-sdk" + ) from e + + system_prompt = None + transcript = [] + for m in messages: + if m["role"] == "system": + system_prompt = m["content"] + elif m["role"] == "user": + transcript.append(f"User: {m['content']}") + elif m["role"] == "assistant": + transcript.append(f"Assistant: {m['content']}") + + if len(transcript) == 1 and messages[-1]["role"] == "user": + prompt = messages[-1]["content"] + else: + prompt = ( + "Below is the conversation so far. Reply as the Assistant " + "to the last message.\n\n" + "\n\n".join(transcript) + ) + + # When the caller passes a JSON schema in `format`, set `output_format` + # so the CLI returns validated JSON in ResultMessage.structured_output + # instead of free-form JSON in the text, which the model sometimes + # gets wrong. + output_schema = kwargs.get("format") + + option_kwargs = dict( + model=model, + system_prompt=system_prompt, + tools=[], + allowed_tools=[], + # A no-tool query usually finishes in one turn, but structured + # output uses an extra internal turn, so cap at 8 rather than 1. + # No tools means the model cannot loop, so the higher bound is safe. + max_turns=8, + # Isolation mode. Keep the host project's CLAUDE.md, settings, + # skills, hooks, and MCP servers out of this subprocess. They + # pollute the prompt and break structured output. + setting_sources=[], + skills=[], + strict_mcp_config=True, + ) + if output_schema: + option_kwargs["output_format"] = { + "type": "json_schema", + "schema": output_schema, + } + agent_options = ClaudeAgentOptions(**option_kwargs) + + # The CLI may emit an in-progress assistant snapshot before the final + # one. Joining text across every AssistantMessage or TextBlock would + # glue a partial draft onto the final answer and produce two JSON + # objects in a row. Keep only the last assistant message, and prefer + # structured_output, then result. + text_parts: List[str] = [] + structured_output = None + result_text = None + async for msg in query(prompt=prompt, options=agent_options): + if isinstance(msg, AssistantMessage): + text_parts = [ + block.text + for block in msg.content + if isinstance(block, TextBlock) + ] + elif isinstance(msg, ResultMessage): + if msg.structured_output is not None: + structured_output = msg.structured_output + if msg.result: + result_text = msg.result + + if structured_output is not None: + # Serialize back to text so callers that json.loads() the body + # still work, now with valid JSON. + content = json.dumps(structured_output) + elif result_text is not None: + content = result_text + else: + content = "".join(text_parts) + return {"message": {"role": "assistant", "content": content}} diff --git a/providers.json b/providers.json index 33dc1a6..45d35f6 100644 --- a/providers.json +++ b/providers.json @@ -37,6 +37,16 @@ "claude-sonnet-5": { "temperature": 0.1, "top_p": 0.9 }, "claude-haiku-4-5": { "temperature": 0.1, "top_p": 0.9 } } + }, + "claude_agent": { + "provider_type": "claude_agent_sdk", + "api_key_env": null, + "structured_output": "none", + "models": { + "sonnet": { "temperature": 0.1, "top_p": 0.9 }, + "opus": { "temperature": 0.1, "top_p": 0.9 }, + "fable": { "temperature": 0.1, "top_p": 0.9 } + } } } } diff --git a/requirements.txt b/requirements.txt index 162f35c..3881a83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,6 @@ pymupdf4llm==0.0.27 Jinja2==3.1.6 google-generativeai==0.4.0 python-dotenv==1.2.2 -black==25.9.0 \ No newline at end of file +black==25.9.0 +# Optional: only needed for the Claude Agent SDK provider (claude_agent models). +# claude-agent-sdk==0.2.127 \ No newline at end of file diff --git a/score.py b/score.py index 6baf3e7..a67dbb4 100644 --- a/score.py +++ b/score.py @@ -43,6 +43,10 @@ format="%(asctime)s - %(name)5s - %(lineno)5d - %(funcName)33s - %(levelname)5s - %(message)s", ) +# Quiet the SDK's "Using bundled Claude Code CLI" INFO line, which fires on +# every LLM call and carries no useful signal. +logging.getLogger("claude_agent_sdk").setLevel(logging.WARNING) + def print_evaluation_results( evaluation: EvaluationData, candidate_name: str = "Candidate"