feat: Multimodality

default_config/config.json

@@ -64,7 +64,8 @@
             "n_ctx": 24000,
             "max_tokens": 8192,
             "stop": ["<|eot_id|>"],
-            "temperature": 0.7
+            "temperature": 0.7,
+            "mmproj_path": "gemma-4-E4B-it-mmproj.gguf"
         }
     },
     "Qwen3.5-9B-Q4_K_M": {
@@ -74,7 +75,9 @@
             "n_batch": 8,
             "max_tokens": 8192,
             "stop": ["<|eot_id|>"],
-            "temperature": 0.7
+            "temperature": 0.7,
+            "mmproj_path": "Qwen3.5-9B-mmproj-F16.gguf",
+            "image_min_tokens": 1024
         }
     },
     "olmo-2-1124-7B-instruct-Q4_K_M": {

lib/analyze_images.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Processor for core:analyze-images — multimodal vision Q&A."""
+from typing import Any
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.runnables import Runnable
+
+from task_files import fetch_file_bytes
+from streaming import StreamContext, run_runnable_with_streaming
+
+
+class AnalyzeImagesProcessor:
+    """Ask a question about one or more images using a vision-capable chat model."""
+
+    runnable: Runnable
+    system_prompt: str = (
+        "You're an AI assistant that analyzes images. "
+        "Answer the user's question about the provided image(s) helpfully and accurately."
+    )
+
+    def __init__(self, runnable: Runnable):
+        self.runnable = runnable
+
+    async def __call__(
+            self,
+            inputs: dict[str, Any],
+            context: StreamContext | None = None,
+    ) -> dict[str, Any]:
+        if context is None or context.nc is None:
+            raise ValueError("StreamContext with Nextcloud client is required for analyze-images")
+
+        question = inputs.get("input") or ""
+        images = inputs.get("images") or []
+        if not images:
+            raise ValueError("core:analyze-images requires at least one image")
+        if len(images) > 10:
+            raise ValueError("Too many images")
+        images = [await fetch_file_bytes(context.nc, image) for image in images]
+
+        content: list[dict[str, Any]] = [{"type": "text", "text": question}]
+        for image in images:
+            if not image["mime"].startswith("image/"):
+                raise ValueError(f"Image MIME type {image['mime']} is not supported")
+            content.append({"type": "image_url", "image_url": {"url": image["data_url"]}})
+
+        reasoning_sink: dict[str, str] = {}
+        output = await run_runnable_with_streaming(
+            self.runnable,
+            [
+                SystemMessage(self.system_prompt),
+                HumanMessage(content=content),
+            ],
+            context,
+            reasoning_sink=reasoning_sink,
+        )
+        return {
+            "output": output,
+            "reasoning": reasoning_sink.get("reasoning", ""),
+        }

lib/llama_server.py

@@ -37,13 +37,17 @@
     "api_keys", "api_prefix",
     # Misc
     "model_alias", "verbosity",
+    # Multimodal (scalars only; mmproj path is set separately)
+    "image_min_tokens", "image_max_tokens",
 )
 
 
 def main() -> None:
     cfg = json.loads(sys.argv[1])
     p = xlc.CommonParams()
     p.model.path = cfg["model_path"]
+    if cfg.get("mmproj_path"):
+        p.mmproj.path = cfg["mmproj_path"]
     for k in SERVER_KEYS:
         if k in cfg:
             setattr(p, k, cfg[k])

lib/main.py

@@ -66,7 +66,9 @@ async def log(nc: AsyncNextcloudApp, level, content):
 
 models_to_fetch = {
     "https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf/resolve/main/gemma-4-E4B_q4_0-it.gguf": {"save_path": os.path.join(persistent_storage(), "gemma-4-E4B_q4_0-it.gguf")},
+    "https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf/resolve/main/gemma-4-E4B-it-mmproj.gguf": {"save_path": os.path.join(persistent_storage(), "gemma-4-E4B-it-mmproj.gguf")},
     "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/Qwen3.5-9B-Q4_K_M.gguf": {"save_path": os.path.join(persistent_storage(), "Qwen3.5-9B-Q4_K_M.gguf")},
+    "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/mmproj-F16.gguf": {"save_path": os.path.join(persistent_storage(), "Qwen3.5-9B-mmproj-F16.gguf")},
     "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/4f0c246f125fc7594238ebe7beb1435a8335f519/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf": {"save_path": os.path.join(persistent_storage(), "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf")},
     "https://huggingface.co/unsloth/Olmo-3-7B-Instruct-GGUF/resolve/86844f8ff856ba9cc8a22c7b9edd7cf95129a580/Olmo-3-7B-Instruct-Q4_K_M.gguf": {"save_path": os.path.join(persistent_storage(), "Olmo-3-7B-Instruct-Q4_K_M.gguf")},
 }
@@ -153,9 +155,17 @@ async def handle_task(task: dict, provider: dict, nc: AsyncNextcloudApp, task_pr
         processor = await loop.run_in_executor(None, task_processor_loader)
 
         stream_result = NextcloudTaskStreamResult(nc, task["id"], bool(task.get("preferStreaming")))
+        # Per-task client for user-scoped file access so concurrent set_user
+        # calls cannot race on the shared background-loop nc.
+        user_id = task.get("userId")
+        user_nc = None
+        if user_id:
+            user_nc = AsyncNextcloudApp()
+            await user_nc.set_user(user_id)
         stream_context = StreamContext(
             stream_result=stream_result.send if stream_result.enabled else None,
             progress_callback=stream_result.set_progress if stream_result.enabled else None,
+            nc=user_nc,
         )
 
         time_start = time.perf_counter()

lib/multimodal_chatwithtools.py

@@ -0,0 +1,171 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Processor for core:text2text:multimodal-chatwithtools."""
+from __future__ import annotations
+
+import json
+import logging
+import pprint
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.messages.ai import AIMessage
+from nc_py_api import AsyncNextcloudApp
+
+from chatwithtools import (
+    build_streaming_payload,
+    generate_tool_call,
+    try_parse_tool_calls,
+)
+from streaming import StreamContext, run_runnable_with_streaming
+from task_files import build_attachment_content_parts
+
+logger = logging.getLogger(__name__)
+
+
+async def resolve_message_content(nc: AsyncNextcloudApp, content: Any) -> str | list[dict[str, Any]]:
+    """Resolve history content: strings pass through; file parts are fetched."""
+
+    resolved: list[dict[str, Any]] = []
+    for item in content:
+        if not isinstance(item, dict) or "type" not in item:
+            raise ValueError("Invalid message history content")
+        if item["type"] != "file":
+            resolved.append(item)
+            continue
+        file_id = item.get("file_id")
+        if file_id is None:
+            raise ValueError("Invalid message history content")
+        try:
+            resolved.extend(await build_attachment_content_parts(nc, file_id))
+        except Exception as e:
+            logger.warning("Could not build file content from id: %s. Error: %s", file_id, e)
+    return resolved
+
+
+async def build_input_attachment_parts(
+    nc: AsyncNextcloudApp,
+    attachments: list[Any],
+) -> list[dict[str, Any]]:
+    """Fetch current-turn input_attachments and convert to content parts."""
+    parts: list[dict[str, Any]] = []
+    for attachment in attachments:
+        file_id = attachment
+        if isinstance(attachment, dict):
+            file_id = attachment.get("id", attachment.get("fileId", attachment.get("file_id")))
+        parts.extend(await build_attachment_content_parts(nc, file_id))
+    return parts
+
+
+class MultimodalChatWithToolsProcessor:
+    """Chat with tools plus image/text file attachments on the current turn and in history."""
+
+    model: BaseChatModel
+
+    def __init__(self, runner: BaseChatModel):
+        self.model = runner
+
+    async def _process_single_input(
+            self,
+            input_data: dict[str, Any],
+            context: StreamContext | None = None,
+    ) -> dict[str, Any]:
+        if context is None or context.nc is None:
+            raise ValueError("StreamContext with Nextcloud client is required for multimodal chat")
+
+        system_prompt = """
+{downstream_system_prompt}
+
+You have tools at your disposal that you can call on behalf of the user.
+You can call a tool by responding with a tool call.
+A tool call starts with an opening `tool_call` xml tag, then a JSON object with the name of the function and the arguments, and finally it ends with a closing `tool_call` xml tag.
+It looks like this, for example:
+{tool_call_example}
+
+Here is a second example:
+{tool_call_example2}
+
+When calling tools, do not output anything else, except the tool call. Do not add sample output of the tool call. Do not output the result of the tool call yourself.
+The following is a JSON specification of the tools you can call and their parameters.
+{tools}
+""".format(
+            tools=input_data['tools'],
+            downstream_system_prompt=input_data['system_prompt'],
+            tool_call_example='<tool_call>{"name": "the_function_to_call", "arguments": {"param1": "the first argument", "param2": "second argument"}}</tool_call>',
+            tool_call_example2='<tool_call>{"name": "search_the_web", "arguments": {"search_query": "Frank Sinatra"}}</tool_call>'
+        )
+
+        messages = []
+        messages.append(SystemMessage(content=system_prompt))
+
+        for raw_message in input_data['history']:
+            message = json.loads(raw_message)
+            content = await resolve_message_content(context.nc, message['content'])
+            if message['role'] == 'assistant':
+                if content == '' and message.get("tool_calls"):
+                    messages.append(AIMessage(content=generate_tool_call(message['tool_calls'][0])))
+                else:
+                    messages.append(AIMessage(content=content))
+            elif message['role'] == 'human':
+                messages.append(HumanMessage(content=content))
+
+        attachments = input_data['input_attachments']
+        if len(attachments) > 10:
+            raise ValueError("Too many attachments")
+
+        attachment_parts = await build_input_attachment_parts(context.nc, attachments)
+
+        if input_data['input'] != '':
+            content: list[dict[str, Any]] | str
+            if attachment_parts:
+                content = list(attachment_parts)
+                content.append({"type": "text", "text": input_data['input']})
+            else:
+                content = input_data['input']
+            messages.append(HumanMessage(content=content))
+        elif 'tool_message' in input_data and input_data['tool_message'] != '':
+            try:
+                tool_messages = json.loads(input_data['tool_message'])
+                for tool_message in tool_messages:
+                    message_content = """
+    The result of your tool call for the tool "{tool_name}" is the following:
+
+    ===
+    {tool_call_result}
+    ===
+
+    You can now formulate this in natural language for the user. Do not mention that you called a tool.
+    """.format(tool_call_result=tool_message['content'], tool_name=tool_message['name'])
+                    messages.append(HumanMessage(content=message_content))
+            except json.JSONDecodeError as e:
+                print('Failed to parse tool message')
+                print(e)
+        elif attachment_parts:
+            messages.append(HumanMessage(content=attachment_parts))
+        else:
+            messages.append(HumanMessage(content=''))
+
+        # Can't print as images will fill up too much
+        # pprint.pprint(messages)
+        reasoning_sink: dict[str, str] = {}
+        response_content = await run_runnable_with_streaming(
+            self.model,
+            messages,
+            context,
+            stream_payload_transform=build_streaming_payload,
+            suppress_empty_stream_updates=True,
+            reasoning_sink=reasoning_sink,
+        )
+
+        response = AIMessage(**try_parse_tool_calls(response_content))
+
+        return {
+            'output': response.content,
+            'tool_calls': json.dumps(response.tool_calls),
+            'output_attachments': [],
+            'reasoning': reasoning_sink.get('reasoning', ''),
+        }
+
+    async def __call__(self, inputs: dict[str, Any], context: StreamContext | None = None) -> dict[str, Any]:
+        return await self._process_single_input(inputs, context)

lib/streaming.py

@@ -7,6 +7,8 @@
 from time import monotonic
 from typing import Any, Awaitable, Callable
 
+from nc_py_api import AsyncNextcloudApp
+
 # langchain-openai's converters drop `reasoning_content` (emitted by llama.cpp's
 # deepseek reasoning format) from both streaming deltas and non-streaming
 # responses. Patch them to preserve it under additional_kwargs so downstream
@@ -82,6 +84,8 @@ def extract_reasoning_content(value: Any) -> str:
 
 @dataclass
 class StreamContext:
+    # Per-task client impersonating the task user for file downloads
+    nc: AsyncNextcloudApp | None = None
     stream_result: Callable[[dict[str, Any]], Awaitable[None] | None] | None = None
     progress_callback: Callable[[float], Awaitable[Any] | Any] | None = None
     stream_interval_seconds: float = 0.75