usr/ data dir fix

initialize.py

@@ -140,6 +140,14 @@ def initialize_preload():
     import preload
     return defer.DeferredTask().start_task(preload.preload)
 
+def initialize_migration():
+    from python.helpers import migration, dotenv
+    # run migration
+    migration.migrate_user_data()
+    # reload .env as it might have been moved
+    dotenv.load_dotenv()
+    # reload settings to ensure new paths are picked up
+    settings.reload_settings()
 
 def _args_override(config):
     # update config with runtime args

python/api/api_files_get.py

@@ -50,7 +50,7 @@ async def process(self, input: dict, request: Request) -> dict | Response:
                     if path.startswith("/a0/tmp/uploads/"):
                         # Internal path - convert to external
                         filename = path.replace("/a0/tmp/uploads/", "")
-                        external_path = files.get_abs_path("tmp/uploads", filename)
+                        external_path = files.get_abs_path("usr/uploads", filename)
                         filename = os.path.basename(external_path)
                     elif path.startswith("/a0/"):
                         # Other internal Agent Zero paths

python/api/api_message.py

@@ -40,8 +40,8 @@ async def process(self, input: dict, request: Request) -> dict | Response:
         # Handle attachments (base64 encoded)
         attachment_paths = []
         if attachments:
-            upload_folder_int = "/a0/tmp/uploads"
-            upload_folder_ext = files.get_abs_path("tmp/uploads")
+            upload_folder_int = "/a0/usr/uploads"
+            upload_folder_ext = files.get_abs_path("usr/uploads")
             os.makedirs(upload_folder_ext, exist_ok=True)
 
             for attachment in attachments:

python/api/message.py

@@ -29,8 +29,8 @@ async def communicate(self, input: dict, request: Request):
             attachments = request.files.getlist("attachments")
             attachment_paths = []
 
-            upload_folder_int = "/a0/tmp/uploads"
-            upload_folder_ext = files.get_abs_path("tmp/uploads") # for development environment
+            upload_folder_int = "/a0/usr/uploads"
+            upload_folder_ext = files.get_abs_path("usr/uploads") # for development environment
 
             if attachments:
                 os.makedirs(upload_folder_ext, exist_ok=True)

python/api/upload.py

@@ -14,7 +14,7 @@ async def process(self, input: dict, request: Request) -> dict | Response:
         for file in file_list:
             if file and self.allowed_file(file.filename):  # Check file type
                 filename = secure_filename(file.filename) # type: ignore
-                file.save(files.get_abs_path("tmp/upload", filename))
+                file.save(files.get_abs_path("usr/upload", filename))
                 saved_filenames.append(filename)
 
         return {"filenames": saved_filenames}  # Return saved filenames

python/helpers/backup.py

@@ -60,28 +60,12 @@ def _get_default_patterns(self) -> str:
         # Ensure paths don't have double slashes
         agent_root = self.agent_zero_root.rstrip('/')
 
-        return f"""# Agent Zero Knowledge (excluding defaults)
-{agent_root}/knowledge/**
-!{agent_root}/knowledge/default/**
-
-# Agent Zero Instruments (excluding defaults)
-{agent_root}/instruments/**
-!{agent_root}/instruments/default/**
-
-# Memory (excluding embeddings cache)
-{agent_root}/memory/**
-!{agent_root}/memory/**/embeddings/**
-
-# Configuration and Settings (CRITICAL)
-{agent_root}/.env
-{agent_root}/tmp/settings.json
-{agent_root}/tmp/secrets.env
-{agent_root}/tmp/chats/**
-{agent_root}/tmp/scheduler/**
-{agent_root}/tmp/uploads/**
-
-# User data
+        return f"""# User data
+# All persistent user data is now centralized in /usr for easier backup and restore
 {agent_root}/usr/**
+
+# Explicitly include .env
+{agent_root}/usr/.env
 """
 
     def _get_agent_zero_version(self) -> str:

python/helpers/dotenv.py

@@ -15,7 +15,7 @@ def load_dotenv():
 
 
 def get_dotenv_file_path():
-    return get_abs_path(".env")
+    return get_abs_path("usr/.env")
 
 def get_dotenv_value(key: str, default: Any = None):
     # load_dotenv()       

python/helpers/email_client.py

@@ -537,7 +537,7 @@ async def read_messages(
     port: int = 993,
     username: str = "",
     password: str = "",
-    download_folder: str = "tmp/email",
+    download_folder: str = "usr/email",
     options: Optional[Dict[str, Any]] = None,
     filter: Optional[Dict[str, Any]] = None,
 ) -> List[Message]:

python/helpers/files.py

@@ -410,6 +410,10 @@ def move_dir(old_path: str, new_path: str):
     abs_new = get_abs_path(new_path)
     if not os.path.isdir(abs_old):
         return  # nothing to rename
+
+    # ensure parent directory exists
+    os.makedirs(os.path.dirname(abs_new), exist_ok=True)
+
     try:
         os.rename(abs_old, abs_new)
     except Exception:

python/helpers/memory.py

@@ -139,7 +139,7 @@ def initialize(
             log_item.stream(progress="\nInitializing VectorDB")
 
         em_dir = files.get_abs_path(
-            "memory/embeddings"
+            "tmp/memory/embeddings"
         )  # just caching, no need to parameterize
         db_dir = abs_db_dir(memory_subdir)
 
@@ -333,6 +333,16 @@ def _preload_knowledge_folders(
             recursive=True,
         )
 
+        # load custom instruments descriptions
+        index = knowledge_import.load_knowledge(
+            log_item,
+            files.get_abs_path("usr/instruments"),
+            index,
+            {"area": Memory.Area.INSTRUMENTS.value},
+            filename_pattern="**/*.md",
+            recursive=True,
+        )
+
         return index
 
     def get_document_by_id(self, id: str) -> Document | None:
@@ -483,7 +493,9 @@ def get_timestamp():
 def get_custom_knowledge_subdir_abs(agent: Agent) -> str:
     for dir in agent.config.knowledge_subdirs:
         if dir != "default":
-            return files.get_abs_path("knowledge", dir)
+            if dir == "custom":
+                return files.get_abs_path("usr/knowledge")
+            return files.get_abs_path("usr/knowledge", dir)
     raise Exception("No custom knowledge subdir set")
 
 
@@ -499,7 +511,7 @@ def abs_db_dir(memory_subdir: str) -> str:
 
         return files.get_abs_path(get_project_meta_folder(memory_subdir[9:]), "memory")
     # standard subdirs
-    return files.get_abs_path("memory", memory_subdir)
+    return files.get_abs_path("usr/memory", memory_subdir)
 
 
 def abs_knowledge_dir(knowledge_subdir: str, *sub_dirs: str) -> str:
@@ -511,7 +523,11 @@ def abs_knowledge_dir(knowledge_subdir: str, *sub_dirs: str) -> str:
             get_project_meta_folder(knowledge_subdir[9:]), "knowledge", *sub_dirs
         )
     # standard subdirs
-    return files.get_abs_path("knowledge", knowledge_subdir, *sub_dirs)
+    if knowledge_subdir == "default":
+        return files.get_abs_path("knowledge", *sub_dirs)
+    if knowledge_subdir == "custom":
+        return files.get_abs_path("usr/knowledge", *sub_dirs)
+    return files.get_abs_path("usr/knowledge", knowledge_subdir, *sub_dirs)
 
 
 def get_memory_subdir_abs(agent: Agent) -> str:
@@ -546,7 +562,7 @@ def get_existing_memory_subdirs() -> list[str]:
         )
 
         # Get subdirectories from memory folder
-        subdirs = files.get_subdirectories("memory", exclude="embeddings")
+        subdirs = files.get_subdirectories("usr/memory")
 
         project_subdirs = files.get_subdirectories(get_projects_parent_folder())
         for project_subdir in project_subdirs:

python/helpers/migration.py

@@ -0,0 +1,112 @@
+import os
+from python.helpers import files
+from python.helpers.print_style import PrintStyle
+
+def migrate_user_data() -> None:
+    """
+    Migrate user data from /tmp and other locations to /usr.
+    """
+
+    PrintStyle().print("Checking for data migration...")
+
+    # --- Migrate Directories -------------------------------------------------------
+    # Move directories from tmp/ or other source locations to usr/
+
+    _move_dir("tmp/chats", "usr/chats")
+    _move_dir("tmp/scheduler", "usr/scheduler", overwrite=True)
+    _move_dir("tmp/uploads", "usr/uploads")
+    _move_dir("tmp/upload", "usr/upload")
+    _move_dir("tmp/downloads", "usr/downloads")
+    _move_dir("tmp/email", "usr/email")
+    _move_dir("knowledge/custom", "usr/knowledge", overwrite=True)
+    _move_dir("instruments/custom", "usr/instruments", overwrite=True)
+
+    # --- Migrate Files -------------------------------------------------------------
+    # Move specific configuration files to usr/
+
+    _move_file("tmp/settings.json", "usr/settings.json")
+    _move_file("tmp/secrets.env", "usr/secrets.env")
+    _move_file(".env", "usr/.env", overwrite=True)
+
+    # --- Special Migration Cases ---------------------------------------------------
+
+    # Migrate Memory
+    _migrate_memory()
+
+    # Flatten default directories (knowledge/default -> knowledge/, etc.)
+    # We use _merge_dir_contents because we want to move the *contents* of default/ 
+    # into the parent directory, not move the default directory itself.
+    _merge_dir_contents("knowledge/default", "knowledge")
+    _merge_dir_contents("instruments/default", "instruments")
+
+    # --- Cleanup -------------------------------------------------------------------
+
+    # Remove obsolete directories after migration
+    _cleanup_obsolete()
+
+    PrintStyle().print("Migration check complete.")
+
+# --- Helper Functions ----------------------------------------------------------
+
+def _move_dir(src: str, dst: str, overwrite: bool = False) -> None:
+    """
+    Move a directory from src to dst if src exists and dst does not.
+    """
+    if files.exists(src) and (not files.exists(dst) or overwrite):
+        PrintStyle().print(f"Migrating {src} to {dst}...")
+        if overwrite and files.exists(dst):
+            files.delete_dir(dst)
+        files.move_dir(src, dst)
+
+def _move_file(src: str, dst: str, overwrite: bool = False) -> None:
+    """
+    Move a file from src to dst if src exists and dst does not.
+    """
+    if files.exists(src) and (not files.exists(dst) or overwrite):
+        PrintStyle().print(f"Migrating {src} to {dst}...")
+        files.move_file(src, dst)
+
+def _migrate_memory(base_path: str = "memory") -> None:
+    """
+    Migrate memory subdirectories.
+    """
+    subdirs = files.get_subdirectories(base_path)
+    for subdir in subdirs:
+        if subdir == "embeddings":
+            # Special case: Embeddings
+            _move_dir("memory/embeddings", "tmp/memory/embeddings")
+        else:
+            # Move other memory items to usr/memory
+            dst = f"usr/memory/{subdir}"
+            _move_dir(f"memory/{subdir}", dst)
+
+def _merge_dir_contents(src_parent: str, dst_parent: str) -> None:
+    """
+    Moves all subdirectories from src_parent to dst_parent.
+    Useful for flattening structures like 'knowledge/default/*' -> 'knowledge/*'.
+    """
+    if not files.exists(src_parent):
+        return
+
+    # Iterate over subdirectories in the source parent
+    subdirs = files.get_subdirectories(src_parent)
+    for subdir in subdirs:
+        src = f"{src_parent}/{subdir}"
+        dst = f"{dst_parent}/{subdir}"
+
+        # Move the subdirectory if it doesn't exist in destination
+        _move_dir(src, dst)
+
+def _cleanup_obsolete() -> None:
+    """
+    Remove directories that are no longer needed.
+    """
+    to_remove = [
+        "knowledge/default",
+        "instruments/default",
+        "memory"
+    ]
+    for path in to_remove:
+        if files.exists(path):
+            PrintStyle().print(f"Removing {path}...")
+            files.delete_dir(path)

python/helpers/persist_chat.py

@@ -9,7 +9,7 @@
 
 from python.helpers.log import Log, LogItem
 
-CHATS_FOLDER = "tmp/chats"
+CHATS_FOLDER = "usr/chats"
 LOG_SIZE = 1000
 CHAT_FILE_NAME = "chat.json"
 

python/helpers/secrets.py

@@ -15,7 +15,7 @@
 
 # New alias-based placeholder format §§secret(KEY)
 ALIAS_PATTERN = r"§§secret\(([A-Za-z_][A-Za-z0-9_]*)\)"
-DEFAULT_SECRETS_FILE = "tmp/secrets.env"
+DEFAULT_SECRETS_FILE = "usr/secrets.env"
 
 
 def alias_for_key(key: str, placeholder: str = "§§secret({key})") -> str:

python/helpers/settings.py

@@ -161,7 +161,7 @@ class SettingsOutput(TypedDict):
 PASSWORD_PLACEHOLDER = "****PSWD****"
 API_KEY_PLACEHOLDER = "************"
 
-SETTINGS_FILE = files.get_abs_path("tmp/settings.json")
+SETTINGS_FILE = files.get_abs_path("usr/settings.json")
 _settings: Settings | None = None
 
 
@@ -1343,6 +1343,12 @@ def get_settings() -> Settings:
     return norm
 
 
+def reload_settings() -> Settings:
+    global _settings
+    _settings = None
+    return get_settings()
+
+
 def set_settings(settings: Settings, apply: bool = True):
     global _settings
     previous = _settings

python/helpers/task_scheduler.py

@@ -26,7 +26,7 @@
 import pytz
 from typing import Annotated
 
-SCHEDULER_FOLDER = "tmp/scheduler"
+SCHEDULER_FOLDER = "usr/scheduler"
 
 # ----------------------
 # Task Models

python/tools/browser_agent.py

@@ -56,7 +56,7 @@ async def _initialize(self):
                 disable_security=True,
                 chromium_sandbox=False,
                 accept_downloads=True,
-                downloads_path=files.get_abs_path("tmp/downloads"),
+                downloads_path=files.get_abs_path("usr/downloads"),
                 allowed_domains=["*", "http://*", "https://*"],
                 executable_path=pw_binary,
                 keep_alive=True,

run_ui.py

@@ -190,6 +190,9 @@ async def serve_index():
 def run():
     PrintStyle().print("Initializing framework...")
 
+    # migrate data before anything else
+    initialize.initialize_migration()
+
     # Suppress only request logs but keep the startup messages
     from werkzeug.serving import WSGIRequestHandler
     from werkzeug.serving import make_server

webui/components/chat/attachments/attachmentsStore.js

@@ -259,11 +259,11 @@ const model = {
 
   // Generate server-side API URL for file (for device sync)
   getServerImgUrl(filename) {
-    return `/image_get?path=/a0/tmp/uploads/${encodeURIComponent(filename)}`;
+    return `/image_get?path=/a0/usr/uploads/${encodeURIComponent(filename)}`;
   },
 
   getServerFileUrl(filename) {
-    return `/a0/tmp/uploads/${encodeURIComponent(filename)}`;
+    return `/a0/usr/uploads/${encodeURIComponent(filename)}`;
   },
 
   // Check if file is an image based on extension