feat: knowledge base for long-term memory (#1099)

backend/app/agent/toolkit/knowledge_base_toolkit.py

@@ -0,0 +1,96 @@
+# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. =========
+
+"""
+Long-term memory via markdown files (issue #1099).
+
+Memory is architecture-level: .eigent/memory.md is the index; the agent
+reads/writes .eigent/*.md via file operations. No tools are exposed; prompt
+builders should inject MEMORY_ARCHITECTURE_PROMPT and get_index_for_prompt().
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Final
+
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+from app.agent.toolkit.abstract_toolkit import AbstractToolkit
+from app.component.environment import env
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_WORKING_DIR: Final[str] = "~/.eigent"
+
+
+def _resolve_working_directory(working_directory: str | None) -> str:
+    if working_directory is None or not str(working_directory).strip():
+        working_directory = env("file_save_path", os.path.expanduser(_DEFAULT_WORKING_DIR))
+    resolved = os.path.expanduser(str(working_directory).strip())
+    try:
+        os.makedirs(resolved, exist_ok=True)
+    except OSError as e:
+        logger.warning("Could not create working directory %s: %s", resolved, e)
+    return resolved
+
+
+class KnowledgeBaseToolkit(BaseToolkit, AbstractToolkit):
+    """
+    Project long-term memory (architecture-only). Adds no tools; the agent
+    uses file operations on .eigent/*.md. Kept so "knowledge_base_toolkit"
+    remains selectable and prompt builders can inject the memory index.
+    """
+
+    def __init__(
+        self,
+        api_task_id: str,
+        working_directory: str | None = None,
+        agent_name: str | None = None,
+        timeout: float | None = None,
+    ) -> None:
+        api_task_id = (api_task_id or "").strip()
+        if not api_task_id:
+            raise ValueError("api_task_id cannot be empty")
+
+        super().__init__(timeout=timeout)
+        self.api_task_id = api_task_id
+        self.working_directory = _resolve_working_directory(working_directory)
+        self.agent_name = (agent_name or "agent").strip() or "agent"
+
+        logger.debug(
+            "KnowledgeBaseToolkit initialized",
+            extra={
+                "api_task_id": self.api_task_id,
+                "working_directory": self.working_directory,
+                "agent_name": self.agent_name,
+            },
+        )
+
+    def get_tools(self) -> list[FunctionTool]:
+        return []
+
+
+def get_tools(
+    api_task_id: str,
+    working_directory: str | None = None,
+    agent_name: str | None = None,
+) -> list[FunctionTool]:
+    return KnowledgeBaseToolkit(
+        api_task_id=api_task_id,
+        working_directory=working_directory,
+        agent_name=agent_name,
+    ).get_tools()

backend/app/agent/tools.py

@@ -27,6 +27,7 @@
 from app.agent.toolkit.google_drive_mcp_toolkit import GoogleDriveMCPToolkit
 from app.agent.toolkit.google_gmail_mcp_toolkit import GoogleGmailMCPToolkit
 from app.agent.toolkit.image_analysis_toolkit import ImageAnalysisToolkit
+from app.agent.toolkit.knowledge_base_toolkit import KnowledgeBaseToolkit
 from app.agent.toolkit.lark_toolkit import LarkToolkit
 from app.agent.toolkit.linkedin_toolkit import LinkedInToolkit
 from app.agent.toolkit.mcp_search_toolkit import McpSearchToolkit
@@ -63,6 +64,7 @@ async def get_toolkits(tools: list[str], agent_name: str, api_task_id: str):
         "google_drive_mcp_toolkit": GoogleDriveMCPToolkit,
         "google_gmail_mcp_toolkit": GoogleGmailMCPToolkit,
         "image_analysis_toolkit": ImageAnalysisToolkit,
+        "knowledge_base_toolkit": KnowledgeBaseToolkit,
         "linkedin_toolkit": LinkedInToolkit,
         "lark_toolkit": LarkToolkit,
         "mcp_search_toolkit": McpSearchToolkit,

backend/app/router.py

@@ -13,8 +13,7 @@
 # ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. =========
 """
 Centralized router registration for the Eigent API.
-All routers are explicitly registered here
-for better visibility and maintainability.
+All routers are explicitly registered here for better visibility and maintainability.
 """
 
 import logging
@@ -79,9 +78,7 @@ def register_routers(app: FastAPI, prefix: str = "") -> None:
         )
         route_count = len(config["router"].routes)
         logger.info(
-            f"Registered {config['tags'][0]} router:"
-            f" {route_count} routes -"
-            f" {config['description']}"
+            f"Registered {config['tags'][0]} router: {route_count} routes - {config['description']}"
         )
 
     logger.info(f"Total routers registered: {len(routers_config)}")

backend/app/service/chat_service.py

@@ -235,7 +235,8 @@ def check_conversation_history_length(
 
 
 def build_conversation_context(
-    task_lock: TaskLock, header: str = "=== CONVERSATION HISTORY ==="
+    task_lock: TaskLock,
+    header: str = "=== CONVERSATION HISTORY ===",
 ) -> str:
     """Build conversation context from task_lock history
     with files listed only once at the end.
@@ -245,14 +246,13 @@ def build_conversation_context(
         header: Header text for the context section
 
     Returns:
-        Formatted context string with task history
-        and files listed once at the end
+        Formatted context string with task history and files listed once at the end
     """
     context = ""
     working_directories = set()  # Collect all unique working directories
 
     if task_lock.conversation_history:
-        context = f"{header}\n"
+        context += f"{header}\n"
 
         for entry in task_lock.conversation_history:
             if entry["role"] == "task_result":
@@ -553,7 +553,8 @@ async def step_solve(options: Chat, request: Request, task_lock: TaskLock):
                         "without workforce"
                     )
                     conv_ctx = build_conversation_context(
-                        task_lock, header="=== Previous Conversation ==="
+                        task_lock,
+                        header="=== Previous Conversation ===",
                     )
                     simple_answer_prompt = (
                         f"{conv_ctx}"
@@ -1967,7 +1968,8 @@ async def question_confirm(
     context_prompt = ""
     if task_lock:
         context_prompt = build_conversation_context(
-            task_lock, header="=== Previous Conversation ==="
+            task_lock,
+            header="=== Previous Conversation ===",
         )
 
     full_prompt = f"""{context_prompt}User Query: {prompt}

backend/app/utils/memory_file.py

@@ -0,0 +1,185 @@
+# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. =========
+
+"""
+Markdown-based long-term memory for agents (issue #1099).
+
+memory.md in the project's .eigent/ directory acts as an index: only a short
+prefix (e.g. first 200 lines) is passed into the system prompt. Topic-specific
+memories live in other .md files under .eigent/; the agent reads and writes
+them on demand via file operations (no dedicated remember/read tools).
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from datetime import datetime
+from pathlib import Path
+from typing import Final
+
+logger = logging.getLogger("memory_file")
+
+_LOCK: Final[threading.Lock] = threading.Lock()
+_MEMORY_FILENAME: Final[str] = "memory.md"
+_EIGENT_DIR: Final[str] = ".eigent"
+_DEFAULT_HEADER: Final[str] = "# Project Memory\n\nLong-term memory for this project.\n"
+_MAX_ENTRY_LENGTH: Final[int] = 10000
+_DEFAULT_INDEX_LINES: Final[int] = 200
+_MAX_INDEX_LINES: Final[int] = 2000
+
+_CONTINUATION_NOTE: Final[str] = "\n\n...(further memory in .eigent/; read files as needed)\n"
+_INDEX_HEADER: Final[str] = "=== Project memory index (.eigent/memory.md) ===\n"
+
+
+class MemoryFileError(Exception):
+    """Base exception for memory file operations."""
+
+
+class MemoryReadError(MemoryFileError):
+    """Raised when reading the memory file fails."""
+
+
+class MemoryWriteError(MemoryFileError):
+    """Raised when writing or appending to the memory file fails."""
+
+
+def _validate_working_directory(working_directory: str) -> Path:
+    if not working_directory or not working_directory.strip():
+        raise ValueError("working_directory cannot be empty")
+    path = Path(working_directory).expanduser().resolve()
+    if not path.exists():
+        raise ValueError(f"working_directory does not exist: {path}")
+    if not path.is_dir():
+        raise ValueError(f"working_directory is not a directory: {path}")
+    return path
+
+
+def _validate_content(content: str, max_length: int = _MAX_ENTRY_LENGTH) -> str:
+    if not content or not content.strip():
+        raise ValueError("content cannot be empty")
+    content = content.strip()
+    if len(content) > max_length:
+        raise ValueError(f"content exceeds maximum length of {max_length} characters")
+    return content
+
+
+def get_memory_file_path(working_directory: str) -> Path:
+    """Return the path to the project's memory file (.eigent/memory.md)."""
+    base_path = _validate_working_directory(working_directory)
+    eigent_dir = base_path / _EIGENT_DIR
+    eigent_dir.mkdir(parents=True, exist_ok=True)
+    return eigent_dir / _MEMORY_FILENAME
+
+
+def read_memory(working_directory: str) -> str | None:
+    """Read the full content of the memory file, or None if missing/invalid."""
+    try:
+        memory_path = get_memory_file_path(working_directory)
+    except ValueError as e:
+        logger.warning("Invalid working directory: %s", e)
+        return None
+
+    if not memory_path.exists():
+        return None
+
+    try:
+        content = memory_path.read_text(encoding="utf-8")
+        return content if content.strip() else None
+    except OSError as e:
+        logger.error("Failed to read memory file %s: %s", memory_path, e)
+        return None
+
+
+def write_memory(working_directory: str, content: str) -> bool:
+    """Overwrite the memory file with the given content. Returns True on success."""
+    try:
+        memory_path = get_memory_file_path(working_directory)
+        validated = _validate_content(content, max_length=_MAX_ENTRY_LENGTH * 10)
+    except ValueError as e:
+        logger.error("Validation failed: %s", e)
+        return False
+
+    with _LOCK:
+        try:
+            memory_path.write_text(validated, encoding="utf-8")
+            logger.info("Memory file updated", extra={"path": str(memory_path)})
+            return True
+        except OSError as e:
+            logger.error("Failed to write memory file: %s", e)
+            return False
+
+
+def append_memory(working_directory: str, entry: str) -> bool:
+    """Append a timestamped entry to the memory file. Returns True on success."""
+    try:
+        memory_path = get_memory_file_path(working_directory)
+        validated = _validate_content(entry)
+    except ValueError as e:
+        logger.error("Validation failed: %s", e)
+        return False
+
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+    formatted = f"\n## {timestamp}\n\n{validated}\n"
+
+    with _LOCK:
+        try:
+            if not memory_path.exists():
+                memory_path.write_text(_DEFAULT_HEADER, encoding="utf-8")
+            with memory_path.open("a", encoding="utf-8") as f:
+                f.write(formatted)
+            logger.info(
+                "Memory entry appended",
+                extra={"path": str(memory_path), "entry_length": len(validated)},
+            )
+            return True
+        except OSError as e:
+            logger.error("Failed to append to memory file: %s", e)
+            return False
+
+
+MEMORY_ARCHITECTURE_PROMPT: Final[str] = """
+Project long-term memory lives under .eigent/ in the project directory.
+- .eigent/memory.md is the index: it lists or summarizes memory topics (e.g. user_preferences.md, decisions.md).
+- You can read any .eigent/*.md file when you need topic-specific information.
+- To remember something: create or edit markdown files under .eigent/ (e.g. append to an existing topic file or create one). Use normal file operations (read/write/append) or shell commands; no dedicated memory tool is required.
+"""
+
+
+def get_index_for_prompt(
+    working_directory: str,
+    max_lines: int = _DEFAULT_INDEX_LINES,
+) -> str | None:
+    """
+    Return the first max_lines of memory.md formatted for system-prompt injection.
+    Callers should use this instead of dumping the full file; topic-specific
+    content is read by the agent via file operations.
+    """
+    if not working_directory or not working_directory.strip():
+        return None
+    if max_lines <= 0:
+        return None
+    effective_max = min(max_lines, _MAX_INDEX_LINES)
+
+    content = read_memory(working_directory)
+    if not content:
+        return None
+
+    lines = content.splitlines()
+    if len(lines) > effective_max:
+        index_content = "\n".join(lines[:effective_max]) + _CONTINUATION_NOTE
+    else:
+        index_content = content
+
+    return _INDEX_HEADER + index_content + "\n"

backend/pyproject.toml

@@ -33,6 +33,7 @@ dev = [
     "pre-commit>=4.0.0",
     "pytest>=8.4.1",
     "pytest-asyncio>=1.1.0",
+    "ruff>=0.15.0",
 ]
 
 [tool.ruff]