feat: Memory system improvements + Custom Agents integration

apps/backend/__init__.py

@@ -19,5 +19,5 @@
 See README.md for full documentation.
 """
 
-__version__ = "2.7.6"
+__version__ = "2.7.14"
 __author__ = "Auto Claude Team"

apps/backend/agents/coder.py

@@ -5,93 +5,94 @@
 Main autonomous agent loop that runs the coder agent to implement subtasks.
 """
 
 import asyncio
 import json
 import logging
 import os
 import re
 from datetime import datetime, timedelta
 from pathlib import Path
 
 from context.constants import SKIP_DIRS
 from core.client import create_client
 from core.file_utils import write_json_atomic
 from linear_updater import (
     LinearTaskState,
     is_linear_enabled,
     linear_build_complete,
     linear_task_started,
     linear_task_stuck,
 )
 from phase_config import (
     get_fast_mode,
     get_phase_client_thinking_kwargs,
     get_phase_model,
     get_phase_model_betas,
 )
 from phase_event import ExecutionPhase, emit_phase
 from progress import (
     count_subtasks,
     count_subtasks_detailed,
     get_current_phase,
     get_next_subtask,
     is_build_complete,
     print_build_complete_banner,
     print_progress_summary,
     print_session_header,
 )
 from prompt_generator import (
     format_context_for_prompt,
     generate_planner_prompt,
     generate_subtask_prompt,
     load_subtask_context,
 )
 from prompts import is_first_run
 from recovery import RecoveryManager
 from security.constants import PROJECT_DIR_ENV_VAR
 from task_logger import (
     LogPhase,
     get_task_logger,
 )
 from ui import (
     BuildState,
     Icons,
     StatusManager,
     bold,
     box,
     highlight,
     icon,
     muted,
     print_key_value,
     print_status,
 )
 
 from .base import (
     AUTH_FAILURE_PAUSE_FILE,
     AUTH_RESUME_CHECK_INTERVAL_SECONDS,
     AUTH_RESUME_MAX_WAIT_SECONDS,
     AUTO_CONTINUE_DELAY_SECONDS,
     HUMAN_INTERVENTION_FILE,
     INITIAL_RETRY_DELAY_SECONDS,
     MAX_CONCURRENCY_RETRIES,
     MAX_RATE_LIMIT_WAIT_SECONDS,
     MAX_RETRY_DELAY_SECONDS,
     MAX_SUBTASK_RETRIES,
     RATE_LIMIT_CHECK_INTERVAL_SECONDS,
     RATE_LIMIT_PAUSE_FILE,
     RESUME_FILE,
     sanitize_error_message,
 )
 from .memory_manager import debug_memory_system_status, get_graphiti_context
+from .custom_agents import build_agents_catalog_prompt
 from .session import post_session_processing, run_agent_session
 from .utils import (
     find_phase_for_subtask,
     find_subtask_in_plan,
     get_commit_count,
     get_latest_commit,
     load_implementation_plan,
     sync_spec_to_source,
 )
 
 logger = logging.getLogger(__name__)
 
@@ -982,6 +983,9 @@
             f"[Coder] [Fast Mode] {'ENABLED' if fast_mode else 'disabled'} for phase={current_phase}"
         )
 
+        # Build catalog of available specialist agents (loaded once, cached)
+        agents_catalog = build_agents_catalog_prompt()
+
         if first_run:
             # Create client for planning phase
             client = create_client(
@@ -991,6 +995,7 @@
                 agent_type="planner",
                 betas=phase_betas,
                 fast_mode=fast_mode,
+                agents_catalog_prompt=agents_catalog,
                 **thinking_kwargs,
             )
             prompt = generate_planner_prompt(spec_dir, project_dir)
@@ -1139,6 +1144,7 @@
                 agent_type="coder",
                 betas=phase_betas,
                 fast_mode=fast_mode,
+                agents_catalog_prompt=agents_catalog,
                 **thinking_kwargs,
             )
 

apps/backend/agents/custom_agents.py

@@ -0,0 +1,270 @@
+"""
+Custom Agents Integration
+=========================
+
+Parses custom agent .md files from ~/.claude/agents/ and provides
+their configuration (system prompt, optional tool/MCP overrides)
+for use in the Auto-Claude build pipeline.
+
+Custom agent files are markdown files that may include YAML frontmatter
+for tool and MCP server configuration:
+
+    ---
+    tools: [Read, Write, Edit, Bash, Glob, Grep]
+    mcp_servers: [context7, graphiti]
+    thinking: high
+    ---
+    You are a frontend specialist...
+
+If no frontmatter is provided, only the system prompt (markdown body)
+is used, and tool/MCP configuration comes from the base AGENT_CONFIGS.
+"""
+
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CustomAgentConfig:
+    """Parsed configuration from a custom agent .md file."""
+
+    agent_id: str
+    system_prompt: str
+    tools: list[str] | None = None  # Override base tools (None = use defaults)
+    mcp_servers: list[str] | None = None  # Override MCP servers (None = use defaults)
+    thinking: str | None = None  # Override thinking level (None = use default)
+    raw_frontmatter: dict = field(default_factory=dict)
+
+
+def get_agents_dir() -> Path:
+    """Get the custom agents directory (~/.claude/agents/)."""
+    config_dir = os.environ.get("CLAUDE_CONFIG_DIR") or os.path.join(
+        os.path.expanduser("~"), ".claude"
+    )
+    return Path(config_dir) / "agents"
+
+
+def parse_agent_file(file_path: Path) -> CustomAgentConfig | None:
+    """
+    Parse a custom agent .md file.
+
+    Extracts optional YAML frontmatter (between --- delimiters) and the
+    markdown body as the system prompt.
+
+    Args:
+        file_path: Path to the .md agent file
+
+    Returns:
+        CustomAgentConfig if file is valid, None if file doesn't exist or is invalid
+    """
+    if not file_path.exists() or not file_path.is_file():
+        logger.warning(f"Custom agent file not found: {file_path}")
+        return None
+
+    try:
+        content = file_path.read_text(encoding="utf-8")
+    except Exception as e:
+        logger.warning(f"Failed to read custom agent file {file_path}: {e}")
+        return None
+
+    agent_id = file_path.stem  # filename without .md
+    frontmatter = {}
+    body = content
+
+    # Extract YAML frontmatter if present
+    fm_match = re.match(r"^---\s*\n(.*?)\n---\s*\n(.*)", content, re.DOTALL)
+    if fm_match:
+        fm_text = fm_match.group(1)
+        body = fm_match.group(2).strip()
+
+        # Simple YAML-like parsing (avoid heavy yaml dependency)
+        frontmatter = _parse_simple_yaml(fm_text)
+
+    system_prompt = body.strip()
+    if not system_prompt:
+        logger.warning(f"Custom agent file has no content: {file_path}")
+        return None
+
+    # Extract optional overrides from frontmatter
+    tools = _parse_string_list(frontmatter.get("tools"))
+    mcp_servers = _parse_string_list(frontmatter.get("mcp_servers"))
+    thinking = frontmatter.get("thinking")
+
+    if thinking and thinking not in ("low", "medium", "high"):
+        logger.warning(
+            f"Invalid thinking level '{thinking}' in {file_path}, ignoring"
+        )
+        thinking = None
+
+    return CustomAgentConfig(
+        agent_id=agent_id,
+        system_prompt=system_prompt,
+        tools=tools,
+        mcp_servers=mcp_servers,
+        thinking=thinking,
+        raw_frontmatter=frontmatter,
+    )
+
+
+def load_custom_agent(agent_id: str) -> CustomAgentConfig | None:
+    """
+    Load a custom agent by ID.
+
+    Searches through category directories in ~/.claude/agents/ for
+    a matching agent file.
+
+    Args:
+        agent_id: Agent ID (filename without .md extension)
+
+    Returns:
+        CustomAgentConfig if found, None otherwise
+    """
+    agents_dir = get_agents_dir()
+    if not agents_dir.exists():
+        return None
+
+    # Search in all category directories
+    for category_dir in sorted(agents_dir.iterdir()):
+        if not category_dir.is_dir():
+            continue
+        agent_file = category_dir / f"{agent_id}.md"
+        if agent_file.exists():
+            return parse_agent_file(agent_file)
+
+    # Also check root agents dir (no category)
+    root_file = agents_dir / f"{agent_id}.md"
+    if root_file.exists():
+        return parse_agent_file(root_file)
+
+    logger.debug(f"Custom agent '{agent_id}' not found in {agents_dir}")
+    return None
+
+
+def load_all_agents() -> list[CustomAgentConfig]:
+    """Load all custom agents from all categories in ~/.claude/agents/."""
+    agents_dir = get_agents_dir()
+    if not agents_dir.exists():
+        return []
+
+    agents = []
+    for category_dir in sorted(agents_dir.iterdir()):
+        if not category_dir.is_dir():
+            continue
+        for agent_file in sorted(category_dir.glob("*.md")):
+            if agent_file.name == "README.md":
+                continue
+            agent = parse_agent_file(agent_file)
+            if agent:
+                agents.append(agent)
+    return agents
+
+
+def build_agents_catalog_prompt() -> str | None:
+    """
+    Build a concise catalog of all available custom agents for system prompt injection.
+
+    Returns a formatted string listing all agents by category with their descriptions,
+    or None if no agents are available.
+    """
+    agents_dir = get_agents_dir()
+    if not agents_dir.exists():
+        return None
+
+    categories: list[tuple[str, list[tuple[str, str]]]] = []
+
+    for category_dir in sorted(agents_dir.iterdir()):
+        if not category_dir.is_dir():
+            continue
+        category_name = category_dir.name.split("-", 1)[-1].replace("-", " ").title() if "-" in category_dir.name else category_dir.name
+
+        agent_entries = []
+        for agent_file in sorted(category_dir.glob("*.md")):
+            if agent_file.name == "README.md":
+                continue
+            agent = parse_agent_file(agent_file)
+            if agent:
+                # Get description from frontmatter, or first line of prompt
+                description = agent.raw_frontmatter.get("description", "")
+                if not description:
+                    # Use first sentence of system prompt as fallback
+                    first_line = agent.system_prompt.split("\n")[0].strip()
+                    description = first_line[:120]
+                elif len(description) > 150:
+                    description = description[:147] + "..."
+                agent_entries.append((agent.agent_id, description))
+
+        if agent_entries:
+            categories.append((category_name, agent_entries))
+
+    if not categories:
+        return None
+
+    total = sum(len(entries) for _, entries in categories)
+    lines = [
+        f"# Available Specialist Agents ({total} agents)",
+        "",
+        "You have access to the following specialist agents organized by category.",
+        "Use them when the task requires specialized expertise — spawn them as subagents",
+        "via the Agent tool with the appropriate subagent_type.",
+        "",
+    ]
+
+    for category_name, entries in categories:
+        lines.append(f"## {category_name}")
+        for agent_id, desc in entries:
+            lines.append(f"- **{agent_id}**: {desc}")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _parse_simple_yaml(text: str) -> dict:
+    """
+    Parse simple YAML-like frontmatter (key: value pairs).
+
+    Handles:
+    - key: value (strings)
+    - key: [item1, item2] (inline lists)
+    - key: (empty value)
+
+    Does NOT handle nested structures or multi-line values.
+    """
+    result = {}
+    for line in text.split("\n"):
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if ":" not in line:
+            continue
+        key, _, value = line.partition(":")
+        key = key.strip()
+        value = value.strip()
+
+        # Parse inline list: [item1, item2]
+        if value.startswith("[") and value.endswith("]"):
+            items = value[1:-1].split(",")
+            result[key] = [item.strip().strip("\"'") for item in items if item.strip()]
+        elif value:
+            # Strip quotes
+            result[key] = value.strip("\"'")
+        else:
+            result[key] = ""
+
+    return result
+
+
+def _parse_string_list(value) -> list[str] | None:
+    """Parse a value as a list of strings, or None if empty/invalid."""
+    if value is None:
+        return None
+    if isinstance(value, list):
+        cleaned = [str(v).strip() for v in value if v]
+        return cleaned if cleaned else None
+    if isinstance(value, str) and value:
+        return [v.strip() for v in value.split(",") if v.strip()]
+    return None

apps/backend/agents/planner.py

@@ -32,6 +32,7 @@
     print_status,
 )
 
+from .custom_agents import build_agents_catalog_prompt
 from .session import run_agent_session
 
 logger = logging.getLogger(__name__)
@@ -107,13 +108,18 @@ async def run_followup_planner(
     logger.info(
         f"[Planner] [Fast Mode] {'ENABLED' if fast_mode else 'disabled'} for follow-up planning"
     )
+
+    # Build catalog of available specialist agents
+    agents_catalog = build_agents_catalog_prompt()
+
     client = create_client(
         project_dir,
         spec_dir,
         planning_model,
         agent_type="planner",
         betas=planning_betas,
         fast_mode=fast_mode,
+        agents_catalog_prompt=agents_catalog,
         **thinking_kwargs,
     )
 

apps/backend/analysis/analyzers/base.py

@@ -68,6 +68,9 @@
     "core",
     "shared",
     "common",
+    "docs",
+    "documentation",
+    "microservices",
 }
 
 # Files that indicate a service root
@@ -83,8 +86,44 @@
     "build.gradle",
     "Makefile",
     "Dockerfile",
+    # Documentation tools
+    "mkdocs.yml",
+    "mkdocs.yaml",
+    "docusaurus.config.js",
+    "docusaurus.config.ts",
+    "conf.py",
+    "book.toml",
 }
 
+# Glob patterns that indicate a service root (for files with variable names)
+SERVICE_ROOT_GLOBS = [
+    "*.csproj",
+    "*.fsproj",
+    "*.sln",
+]
+
+# Deeper glob patterns for projects that nest manifests in subdirectories (e.g. .NET repos with src/)
+SERVICE_ROOT_DEEP_GLOBS = [
+    "src/**/*.csproj",
+    "src/**/*.fsproj",
+    "src/**/*.cs",  # Fallback: .cs source files without .csproj (incomplete repos)
+]
+
+
+def has_service_root(dir_path: Path) -> bool:
+    """Check if a directory has service root indicators (exact files or glob patterns)."""
+    if any((dir_path / f).exists() for f in SERVICE_ROOT_FILES):
+        return True
+    if any(
+        next(dir_path.glob(pattern), None) is not None for pattern in SERVICE_ROOT_GLOBS
+    ):
+        return True
+    # Check deeper patterns for .NET repos that keep .csproj in src/ subdirectories
+    return any(
+        next(dir_path.glob(pattern), None) is not None
+        for pattern in SERVICE_ROOT_DEEP_GLOBS
+    )
+
 
 class BaseAnalyzer:
     """Base class with common utilities for all analyzers."""