diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml
index 2e3b122..3226e6d 100644
--- a/.github/workflows/integration_test.yml
+++ b/.github/workflows/integration_test.yml
@@ -214,6 +214,51 @@ jobs:
[ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ]
[ $STREAMING_UPDATES -gt 0 ]
+ # core:image2text:ocr exists since Nextcloud 33
+ - name: Run multimodal OCR task
+ if: contains(fromJSON('["master", "stable34", "stable33"]'), matrix.server-versions)
+ env:
+ CREDS: "admin:password"
+ EXPECTED_TEXT: "Nextcloud Image"
+ run: |
+ set -x
+ IMAGE_PATH="${{ env.APP_NAME }}/tests/multimodal-ocr-text.png"
+ REMOTE_PATH="multimodal-ocr-text.png"
+
+ curl -u "$CREDS" -T "$IMAGE_PATH" "http://localhost:8080/remote.php/dav/files/admin/$REMOTE_PATH"
+
+ FILE_ID=$(curl -s -u "$CREDS" -X PROPFIND \
+ -H "Depth: 0" \
+ -H "Content-Type: application/xml" \
+ --data '' \
+ "http://localhost:8080/remote.php/dav/files/admin/$REMOTE_PATH" \
+ | tr '\n' ' ' \
+ | grep -oE '[0-9]+' \
+ | head -1 \
+ | grep -oE '[0-9]+')
+ echo "Uploaded image file id: $FILE_ID"
+ [ -n "$FILE_ID" ]
+
+ TASK=$(curl -X POST -u "$CREDS" -H "oCS-APIRequest: true" -H "Content-type: application/json" \
+ "http://localhost:8080/ocs/v2.php/taskprocessing/schedule?format=json" \
+ --data-raw "{\"input\": {\"input\": [$FILE_ID]}, \"type\": \"core:image2text:ocr\", \"appId\": \"test\", \"customId\": \"\"}")
+ echo $TASK
+ TASK_ID=$(echo $TASK | jq '.ocs.data.task.id')
+ NEXT_WAIT_TIME=0
+ TASK_STATUS='"STATUS_SCHEDULED"'
+ until [ $NEXT_WAIT_TIME -eq 35 ] || [ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ] || [ "$TASK_STATUS" == '"STATUS_FAILED"' ]; do
+ TASK=$(curl -u "$CREDS" -H "oCS-APIRequest: true" "http://localhost:8080/ocs/v2.php/taskprocessing/task/$TASK_ID?format=json")
+ echo $TASK
+ TASK_STATUS=$(echo $TASK | jq '.ocs.data.task.status')
+ echo $TASK_STATUS
+ sleep $(( NEXT_WAIT_TIME++ ))
+ done
+ curl -u "$CREDS" -H "oCS-APIRequest: true" "http://localhost:8080/ocs/v2.php/taskprocessing/task/$TASK_ID?format=json"
+ [ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ]
+ TASK_OUTPUT=$(echo $TASK | jq -r '.ocs.data.task.output.output | join("\n")')
+ echo "Model output: $TASK_OUTPUT"
+ echo "$TASK_OUTPUT" | grep -qi "$EXPECTED_TEXT"
+
- name: Show logs
if: always()
run: |
diff --git a/REUSE.toml b/REUSE.toml
index 9c3c63d..5993f0c 100644
--- a/REUSE.toml
+++ b/REUSE.toml
@@ -11,6 +11,11 @@ precedence = "aggregate"
SPDX-FileCopyrightText = "2024 Nextcloud GmbH and Nextcloud contributors"
SPDX-License-Identifier = "AGPL-3.0-or-later"
+[[annotations]]
+path = ["tests/multimodal-ocr-text.png"]
+SPDX-FileCopyrightText = "2026 Nextcloud GmbH and Nextcloud contributors"
+SPDX-License-Identifier = "AGPL-3.0-or-later"
+
[[annotations]]
path = ["img/*"]
SPDX-FileCopyrightText = "2018-2024 Google LLC"
diff --git a/default_config/config.json b/default_config/config.json
index 9fa7e8e..1cd1d29 100644
--- a/default_config/config.json
+++ b/default_config/config.json
@@ -64,8 +64,10 @@
"n_ctx": 24000,
"max_tokens": 8192,
"stop": ["<|eot_id|>"],
- "temperature": 0.7
- }
+ "temperature": 0.7,
+ "mmproj_path": "gemma-4-E4B-it-mmproj.gguf"
+ },
+ "modalities": ["audio", "vision"]
},
"Qwen3.5-9B-Q4_K_M": {
"prompt": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>\n{user_prompt}<|eot_id|>\n<|start_header_id|>assistant<|end_header_id|>\n",
@@ -74,8 +76,11 @@
"n_batch": 8,
"max_tokens": 8192,
"stop": ["<|eot_id|>"],
- "temperature": 0.7
- }
+ "temperature": 0.7,
+ "mmproj_path": "Qwen3.5-9B-mmproj-F16.gguf",
+ "image_min_tokens": 1024
+ },
+ "modalities": ["vision"]
},
"olmo-2-1124-7B-instruct-Q4_K_M": {
"prompt": "<|endoftext|><|system|>\n{system_prompt}\n<|user|>\n{system_prompt}\n{user_prompt}\n<|assistant|>\n",
diff --git a/lib/analyze_images.py b/lib/analyze_images.py
new file mode 100644
index 0000000..b7af545
--- /dev/null
+++ b/lib/analyze_images.py
@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Processor for core:analyze-images — multimodal vision Q&A."""
+from typing import Any
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.runnables import Runnable
+
+from task_files import fetch_file_bytes
+from streaming import StreamContext, run_runnable_with_streaming
+from multimodal_chatwithtools import MAX_ATTACHMENTS_COUNT
+
+
+class AnalyzeImagesProcessor:
+ """Ask a question about one or more images using a vision-capable chat model."""
+
+ runnable: Runnable
+ system_prompt: str = (
+ "You're an AI assistant that analyzes images. "
+ "Answer the user's question about the provided image(s) helpfully and accurately."
+ )
+
+ def __init__(self, runnable: Runnable):
+ self.runnable = runnable
+
+ async def __call__(
+ self,
+ inputs: dict[str, Any],
+ context: StreamContext | None = None,
+ ) -> dict[str, Any]:
+ if context is None or context.nc is None:
+ raise ValueError("StreamContext with Nextcloud client is required for analyze-images")
+
+ question = inputs.get("input") or ""
+ images = inputs.get("images") or []
+ if not images:
+ raise ValueError("core:analyze-images requires at least one image")
+ if len(images) > MAX_ATTACHMENTS_COUNT:
+ raise ValueError(f"Too many images (max {MAX_ATTACHMENTS_COUNT})")
+ images = [await fetch_file_bytes(context.nc, image) for image in images]
+
+ content: list[dict[str, Any]] = [{"type": "text", "text": question}]
+ for image in images:
+ if not image["mime"].startswith("image/"):
+ raise ValueError(f"Image MIME type {image['mime']} is not supported")
+ content.append({"type": "image_url", "image_url": {"url": image["data_url"]}})
+
+ reasoning_sink: dict[str, str] = {}
+ output = await run_runnable_with_streaming(
+ self.runnable,
+ [
+ SystemMessage(self.system_prompt),
+ HumanMessage(content=content),
+ ],
+ context,
+ reasoning_sink=reasoning_sink,
+ )
+ return {
+ "output": output,
+ "reasoning": reasoning_sink.get("reasoning", ""),
+ }
diff --git a/lib/llama_server.py b/lib/llama_server.py
index 36e765c..7513703 100644
--- a/lib/llama_server.py
+++ b/lib/llama_server.py
@@ -37,6 +37,8 @@
"api_keys", "api_prefix",
# Misc
"model_alias", "verbosity",
+ # Multimodal (scalars only; mmproj path is set separately)
+ "image_min_tokens", "image_max_tokens",
)
@@ -44,6 +46,8 @@ def main() -> None:
cfg = json.loads(sys.argv[1])
p = xlc.CommonParams()
p.model.path = cfg["model_path"]
+ if cfg.get("mmproj_path"):
+ p.mmproj.path = cfg["mmproj_path"]
for k in SERVER_KEYS:
if k in cfg:
setattr(p, k, cfg[k])
diff --git a/lib/main.py b/lib/main.py
index 0c010e3..0d9ed19 100644
--- a/lib/main.py
+++ b/lib/main.py
@@ -66,7 +66,9 @@ async def log(nc: AsyncNextcloudApp, level, content):
models_to_fetch = {
"https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf/resolve/main/gemma-4-E4B_q4_0-it.gguf": {"save_path": os.path.join(persistent_storage(), "gemma-4-E4B_q4_0-it.gguf")},
+ "https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf/resolve/main/gemma-4-E4B-it-mmproj.gguf": {"save_path": os.path.join(persistent_storage(), "gemma-4-E4B-it-mmproj.gguf")},
"https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/Qwen3.5-9B-Q4_K_M.gguf": {"save_path": os.path.join(persistent_storage(), "Qwen3.5-9B-Q4_K_M.gguf")},
+ "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/mmproj-F16.gguf": {"save_path": os.path.join(persistent_storage(), "Qwen3.5-9B-mmproj-F16.gguf")},
"https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/4f0c246f125fc7594238ebe7beb1435a8335f519/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf": {"save_path": os.path.join(persistent_storage(), "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf")},
"https://huggingface.co/unsloth/Olmo-3-7B-Instruct-GGUF/resolve/86844f8ff856ba9cc8a22c7b9edd7cf95129a580/Olmo-3-7B-Instruct-Q4_K_M.gguf": {"save_path": os.path.join(persistent_storage(), "Olmo-3-7B-Instruct-Q4_K_M.gguf")},
}
@@ -153,9 +155,17 @@ async def handle_task(task: dict, provider: dict, nc: AsyncNextcloudApp, task_pr
processor = await loop.run_in_executor(None, task_processor_loader)
stream_result = NextcloudTaskStreamResult(nc, task["id"], bool(task.get("preferStreaming")))
+ # Per-task client for user-scoped file access so concurrent set_user
+ # calls cannot race on the shared background-loop nc.
+ user_id = task.get("userId")
+ user_nc = None
+ if user_id:
+ user_nc = AsyncNextcloudApp()
+ await user_nc.set_user(user_id)
stream_context = StreamContext(
stream_result=stream_result.send if stream_result.enabled else None,
progress_callback=stream_result.set_progress if stream_result.enabled else None,
+ nc=user_nc,
)
time_start = time.perf_counter()
@@ -363,7 +373,7 @@ async def enabled_handler(enabled: bool, nc: AsyncNextcloudApp) -> str:
optional_input_defaults=get_optional_input_defaults(task),
optional_output_shape=[
ShapeDescriptor(name="reasoning", description="Reasoning trace produced by the model, if any", shape_type=ShapeType.TEXT)
- ] if task != "core:text2text:summary" else [],
+ ] if task != "core:text2text:summary" and task != "core:image2text:ocr" else [],
)
await nc.providers.task_processing.register(provider)
await log(nc, LogLvl.INFO, f"Registered {task_processor_name}")
diff --git a/lib/multimodal_chatwithtools.py b/lib/multimodal_chatwithtools.py
new file mode 100644
index 0000000..88a3b61
--- /dev/null
+++ b/lib/multimodal_chatwithtools.py
@@ -0,0 +1,172 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Processor for core:text2text:multimodal-chatwithtools."""
+from __future__ import annotations
+
+import json
+import logging
+import pprint
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.messages.ai import AIMessage
+from nc_py_api import AsyncNextcloudApp
+
+from chatwithtools import (
+ build_streaming_payload,
+ generate_tool_call,
+ try_parse_tool_calls,
+)
+from streaming import StreamContext, run_runnable_with_streaming
+from task_files import build_attachment_content_parts
+
+logger = logging.getLogger(__name__)
+
+MAX_ATTACHMENTS_COUNT = 10
+
+async def resolve_message_content(nc: AsyncNextcloudApp, content: Any, modalities: list[str]) -> str | list[dict[str, Any]]:
+ """Resolve history content: strings pass through; file parts are fetched."""
+ if not isinstance(content, list):
+ raise ValueError("Invalid message history content")
+ resolved: list[dict[str, Any]] = []
+ for item in content:
+ if not isinstance(item, dict) or "type" not in item:
+ raise ValueError("Invalid message history content")
+ if item["type"] != "file":
+ resolved.append(item)
+ continue
+ file_id = item.get("file_id")
+ if file_id is None:
+ raise ValueError("Invalid message history content")
+ try:
+ resolved.extend(await build_attachment_content_parts(nc, file_id, modalities))
+ except Exception as e:
+ logger.warning("Could not build file content from id: %s. Error: %s", file_id, e)
+ return resolved
+
+
+async def build_input_attachment_parts(
+ nc: AsyncNextcloudApp,
+ attachments: list[int],
+ modalities: list[str],
+) -> list[dict[str, Any]]:
+ """Fetch current-turn input_attachments and convert to content parts."""
+ parts: list[dict[str, Any]] = []
+ for attachment in attachments:
+ parts.extend(await build_attachment_content_parts(nc, attachment, modalities))
+ return parts
+
+
+class MultimodalChatWithToolsProcessor:
+ """Chat with tools plus image/text file attachments on the current turn and in history."""
+
+ model: BaseChatModel
+
+ def __init__(self, runner: BaseChatModel, modalities: list[str]):
+ self.model = runner
+ self.modalities = modalities
+
+ async def _process_single_input(
+ self,
+ input_data: dict[str, Any],
+ context: StreamContext | None = None,
+ ) -> dict[str, Any]:
+ if context is None or context.nc is None:
+ raise ValueError("StreamContext with Nextcloud client is required for multimodal chat")
+
+ system_prompt = """
+{downstream_system_prompt}
+
+You have tools at your disposal that you can call on behalf of the user.
+You can call a tool by responding with a tool call.
+A tool call starts with an opening `tool_call` xml tag, then a JSON object with the name of the function and the arguments, and finally it ends with a closing `tool_call` xml tag.
+It looks like this, for example:
+{tool_call_example}
+
+Here is a second example:
+{tool_call_example2}
+
+When calling tools, do not output anything else, except the tool call. Do not add sample output of the tool call. Do not output the result of the tool call yourself.
+The following is a JSON specification of the tools you can call and their parameters.
+{tools}
+""".format(
+ tools=input_data['tools'],
+ downstream_system_prompt=input_data['system_prompt'],
+ tool_call_example='{"name": "the_function_to_call", "arguments": {"param1": "the first argument", "param2": "second argument"}}',
+ tool_call_example2='{"name": "search_the_web", "arguments": {"search_query": "Frank Sinatra"}}'
+ )
+
+ messages = []
+ messages.append(SystemMessage(content=system_prompt))
+
+ for raw_message in input_data['history']:
+ message = json.loads(raw_message)
+ content = await resolve_message_content(context.nc, message['content'], self.modalities)
+ if message['role'] == 'assistant':
+ if content == '' and message.get("tool_calls"):
+ messages.append(AIMessage(content=generate_tool_call(message['tool_calls'][0])))
+ else:
+ messages.append(AIMessage(content=content))
+ elif message['role'] == 'human':
+ messages.append(HumanMessage(content=content))
+
+ attachments = input_data['input_attachments']
+ if len(attachments) > MAX_ATTACHMENTS_COUNT:
+ raise ValueError(f"Too many attachments (max {MAX_ATTACHMENTS_COUNT})")
+
+ attachment_parts = await build_input_attachment_parts(context.nc, attachments, self.modalities)
+
+ if input_data['input'] != '':
+ content: list[dict[str, Any]] | str
+ if attachment_parts:
+ content = list(attachment_parts)
+ content.append({"type": "text", "text": input_data['input']})
+ else:
+ content = input_data['input']
+ messages.append(HumanMessage(content=content))
+ elif 'tool_message' in input_data and input_data['tool_message'] != '':
+ try:
+ tool_messages = json.loads(input_data['tool_message'])
+ for tool_message in tool_messages:
+ message_content = """
+ The result of your tool call for the tool "{tool_name}" is the following:
+
+ ===
+ {tool_call_result}
+ ===
+
+ You can now formulate this in natural language for the user. Do not mention that you called a tool.
+ """.format(tool_call_result=tool_message['content'], tool_name=tool_message['name'])
+ messages.append(HumanMessage(content=message_content))
+ except json.JSONDecodeError as e:
+ logger.error('Failed to parse tool message')
+ logger.error(e)
+ elif attachment_parts:
+ messages.append(HumanMessage(content=attachment_parts))
+ else:
+ messages.append(HumanMessage(content=''))
+
+ # Can't print as images will fill up too much
+ # pprint.pprint(messages)
+ reasoning_sink: dict[str, str] = {}
+ response_content = await run_runnable_with_streaming(
+ self.model,
+ messages,
+ context,
+ stream_payload_transform=build_streaming_payload,
+ suppress_empty_stream_updates=True,
+ reasoning_sink=reasoning_sink,
+ )
+
+ response = AIMessage(**try_parse_tool_calls(response_content))
+
+ return {
+ 'output': response.content,
+ 'tool_calls': json.dumps(response.tool_calls),
+ 'output_attachments': [],
+ 'reasoning': reasoning_sink.get('reasoning', ''),
+ }
+
+ async def __call__(self, inputs: dict[str, Any], context: StreamContext | None = None) -> dict[str, Any]:
+ return await self._process_single_input(inputs, context)
diff --git a/lib/ocr.py b/lib/ocr.py
new file mode 100644
index 0000000..9a1e894
--- /dev/null
+++ b/lib/ocr.py
@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Processor for core:image2text:ocr — vision-based OCR."""
+from typing import Any
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.runnables import Runnable
+
+from task_files import fetch_file_bytes
+from streaming import StreamContext, run_runnable_with_streaming
+from multimodal_chatwithtools import MAX_ATTACHMENTS_COUNT
+
+
+class OcrProcessor:
+ """Extract text from one or more images using a vision-capable chat model."""
+
+ runnable: Runnable
+ system_prompt: str = (
+ "You're an OCR assistant. "
+ "Extract all text visible in the provided image. "
+ "Output only the extracted text, nothing else."
+ )
+ user_prompt: str = "Extract all text from this image. Reply with only the extracted text."
+
+ def __init__(self, runnable: Runnable):
+ self.runnable = runnable
+
+ async def __call__(
+ self,
+ inputs: dict[str, Any],
+ context: StreamContext | None = None,
+ ) -> dict[str, Any]:
+ if context is None or context.nc is None:
+ raise ValueError("StreamContext with Nextcloud client is required for OCR")
+
+ files = inputs.get("input") or []
+ if not files:
+ raise ValueError("core:image2text:ocr requires at least one file")
+ if len(files) > MAX_ATTACHMENTS_COUNT:
+ raise ValueError(f"Too many files (max {MAX_ATTACHMENTS_COUNT})")
+
+ texts: list[str] = []
+ for file_id in files:
+ fetched = await fetch_file_bytes(context.nc, file_id)
+ if not fetched["mime"].startswith("image/"):
+ raise ValueError(f"File MIME type {fetched['mime']} is not supported for OCR")
+
+ output = await run_runnable_with_streaming(
+ self.runnable,
+ [
+ SystemMessage(self.system_prompt),
+ HumanMessage(content=[
+ {"type": "text", "text": self.user_prompt},
+ {"type": "image_url", "image_url": {"url": fetched["data_url"]}},
+ ]),
+ ],
+ context,
+ stream_payload_transform=lambda partial: {"output": [*texts, partial]},
+ )
+ texts.append(output)
+
+ return {
+ "output": texts,
+ }
diff --git a/lib/streaming.py b/lib/streaming.py
index 9f35194..9861abd 100644
--- a/lib/streaming.py
+++ b/lib/streaming.py
@@ -7,6 +7,8 @@
from time import monotonic
from typing import Any, Awaitable, Callable
+from nc_py_api import AsyncNextcloudApp
+
# langchain-openai's converters drop `reasoning_content` (emitted by llama.cpp's
# deepseek reasoning format) from both streaming deltas and non-streaming
# responses. Patch them to preserve it under additional_kwargs so downstream
@@ -82,6 +84,8 @@ def extract_reasoning_content(value: Any) -> str:
@dataclass
class StreamContext:
+ # Per-task client impersonating the task user for file downloads
+ nc: AsyncNextcloudApp | None = None
stream_result: Callable[[dict[str, Any]], Awaitable[None] | None] | None = None
progress_callback: Callable[[float], Awaitable[Any] | Any] | None = None
stream_interval_seconds: float = 0.75
diff --git a/lib/task_files.py b/lib/task_files.py
new file mode 100644
index 0000000..69ff743
--- /dev/null
+++ b/lib/task_files.py
@@ -0,0 +1,115 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Helpers for fetching Task Processing input files (images, etc.)."""
+from __future__ import annotations
+
+import base64
+from typing import Any
+
+from nc_py_api import AsyncNextcloudApp
+
+VALID_TEXT_MIME_TYPES = frozenset({
+ "application/javascript",
+ "application/typescript",
+ "message/rfc822",
+ "application/x-sql",
+ "application/x-scala",
+ "application/x-rust",
+ "application/x-powershell",
+ "application/x-patch",
+ "application/x-php",
+ "application/x-httpd-php",
+ "application/x-httpd-php-source",
+ "application/json",
+ "application/x-bash",
+ "application/x-protobuf",
+ "application/x-terraform",
+ "application/x-toml",
+ "application/graphql",
+ "application/x-graphql",
+ "application/x-ndjson",
+ "application/json5",
+ "application/x-json5",
+ "application/toml",
+ "application/x-yaml",
+ "application/yaml",
+ "application/x-awk",
+ "application/x-subrip",
+ "application/csv",
+})
+
+SUPPORTED_INPUT_AUDIO_FORMATS = {
+ "audio/mp3": "mp3",
+ "audio/mpeg": "mp3",
+ "audio/wav": "wav",
+ "audio/x-wav": "wav",
+}
+
+
+
+def bytes_to_data_url(data: bytes, mime: str) -> str:
+ content_type = mime.split(";")[0].strip() or "application/octet-stream"
+ encoded = base64.b64encode(data).decode("ascii")
+ return f"data:{content_type};base64,{encoded}"
+
+
+def is_text_mime(mime: str) -> bool:
+ return mime.startswith("text/") or mime in VALID_TEXT_MIME_TYPES
+
+
+async def fetch_file_bytes(nc: AsyncNextcloudApp, file_id: int | str) -> dict[str, Any]:
+ """Download a Nextcloud file by ID using the public Files API.
+
+ MIME type comes from FsNode metadata (same source Nextcloud uses for Content-Type).
+ """
+ node = await nc.files.by_id(int(file_id))
+ if node is None:
+ raise RuntimeError(f"File not found: {file_id}")
+ data = await nc.files.download(node)
+ if not data:
+ raise RuntimeError(f"Empty file content for file {file_id}")
+ mime = (node.info.mimetype or "").split(";")[0].strip()
+ if not mime:
+ raise RuntimeError(f"Missing mimetype metadata for file {file_id}")
+ return {
+ "data": data,
+ "mime": mime,
+ "data_url": bytes_to_data_url(data, mime),
+ "name": node.name,
+ "file_id": int(file_id),
+ }
+
+
+async def build_attachment_content_parts(
+ nc: AsyncNextcloudApp,
+ file_id: int | str,
+ modalities: list[str],
+) -> list[dict[str, Any]]:
+ """Fetch a file and turn it into OpenAI-style message content parts (image + text only)."""
+ fetched = await fetch_file_bytes(nc, file_id)
+ mime = fetched["mime"]
+ if mime.startswith("image/") and "vision" in modalities:
+ return [{
+ "type": "image_url",
+ "image_url": {"url": fetched["data_url"]},
+ }]
+ elif mime.startswith("audio/") and "audio" in modalities:
+ format = SUPPORTED_INPUT_AUDIO_FORMATS.get(mime)
+ if format is None:
+ raise ValueError(f"Unsupported audio format: {mime}")
+ return [{
+ "type": "input_audio",
+ "input_audio": {"data": base64.b64encode(fetched["data"]).decode("ascii"), "format": format},
+ }]
+ elif is_text_mime(mime):
+ try:
+ text_body = fetched["data"].decode("utf-8")
+ except UnicodeDecodeError as e:
+ raise ValueError(
+ f"Invalid input file type: {mime} (not valid UTF-8 text)"
+ ) from e
+ return [{
+ "type": "text",
+ "text": f"Filename:{fetched['name']}\nContent:\n{text_body}",
+ }]
+ raise ValueError(f"Invalid input file type: {mime}")
diff --git a/lib/task_processors.py b/lib/task_processors.py
index 4e3e009..2ea17ad 100644
--- a/lib/task_processors.py
+++ b/lib/task_processors.py
@@ -31,6 +31,9 @@
from topics import TopicsProcessor
from summarize import SummarizeProcessor
from reformat_paragraphs import ReformatParagraphsProcessor
+from analyze_images import AnalyzeImagesProcessor
+from ocr import OcrProcessor
+from multimodal_chatwithtools import MultimodalChatWithToolsProcessor
dir_path = os.path.dirname(os.path.realpath(__file__))
models_folder_path = os.path.join(dir_path , "../models/")
@@ -60,6 +63,20 @@ def tail(self) -> str:
return "\n".join(self._tail)
+def resolve_model_file(name: str) -> str:
+ """Resolve a GGUF / mmproj filename under models/ or persistent_storage()."""
+ for root in (models_folder_path, persistent_storage()):
+ candidate = os.path.join(root, name)
+ if os.path.exists(candidate):
+ return candidate
+ raise FileNotFoundError(f"Model file not found: {name}")
+
+
+def _is_language_model_gguf(file_name: str) -> bool:
+ """Skip multimodal projector GGUFs when discovering models."""
+ return file_name.endswith(".gguf") and "mmproj" not in file_name.lower()
+
+
def get_model_config(file_name):
file_name = file_name.split('.gguf')[0]
if os.path.exists(os.path.join(models_folder_path, file_name + ".json")):
@@ -123,9 +140,7 @@ def generate_chat_model(file_name: str) -> ChatOpenAI:
model_config = get_model_config(file_name)
loader_config = model_config["loader_config"]
- path = os.path.join(models_folder_path, file_name)
- if not os.path.exists(path):
- path = os.path.join(persistent_storage(), file_name)
+ path = resolve_model_file(file_name)
compute_device = os.getenv("COMPUTE_DEVICE", "CUDA")
n_gpu_layers = -1 if compute_device != "CPU" else 0
@@ -133,7 +148,7 @@ def generate_chat_model(file_name: str) -> ChatOpenAI:
port = _find_free_port()
model_alias = file_name.split(".gguf")[0]
- server_config = json.dumps({
+ server_cfg: dict = {
"model_path": path,
"hostname": "127.0.0.1",
"port": port,
@@ -142,7 +157,12 @@ def generate_chat_model(file_name: str) -> ChatOpenAI:
"n_parallel": loader_config.get("n_parallel", 1),
"cont_batching": True,
**loader_config,
- })
+ }
+
+ if server_cfg.get("mmproj_path"):
+ server_cfg["mmproj_path"] = resolve_model_file(server_cfg["mmproj_path"])
+
+ server_config = json.dumps(server_cfg)
logger.info(f"Starting llama-server for {file_name} on port {port}")
try:
@@ -206,23 +226,27 @@ def stop_all_servers() -> None:
def generate_task_processors(task_processors = {}):
for file in os.scandir(models_folder_path):
- if file.name.endswith(".gguf"):
- if file.name.split('.gguf')[0] in task_processors:
- continue
- generate_task_processors_for_model(file.name, task_processors)
+ if not _is_language_model_gguf(file.name):
+ continue
+ if file.name.split('.gguf')[0] in task_processors:
+ continue
+ generate_task_processors_for_model(file.name, task_processors)
for file in os.scandir(persistent_storage()):
- if file.name.endswith('.gguf'):
- if file.name.split('.gguf')[0] in task_processors:
- continue
- generate_task_processors_for_model(file.name, task_processors)
+ if not _is_language_model_gguf(file.name):
+ continue
+ if file.name.split('.gguf')[0] in task_processors:
+ continue
+ generate_task_processors_for_model(file.name, task_processors)
return task_processors
def generate_task_processors_for_model(file_name, task_processors):
model_name = file_name.split('.gguf')[0]
- n_ctx = get_model_config(file_name)["loader_config"]["n_ctx"]
+ config = get_model_config(file_name)
+ n_ctx = config["loader_config"]["n_ctx"]
+ modalities = config.get("modalities", [])
task_processors[model_name + ":core:text2text:summary"] = lambda: SummarizeProcessor(generate_chat_model(file_name), n_ctx)
task_processors[model_name + ":core:text2text:headline"] = lambda: HeadlineProcessor(generate_chat_model(file_name))
@@ -236,4 +260,8 @@ def generate_task_processors_for_model(file_name, task_processors):
task_processors[model_name + ":core:text2text:proofread"] = lambda: ProofreadProcessor(generate_chat_model(file_name))
task_processors[model_name + ":core:text2text:changetone"] = lambda: ChangeToneProcessor(generate_chat_model(file_name))
task_processors[model_name + ":core:text2text:chatwithtools"] = lambda: ChatWithToolsProcessor(generate_chat_model(file_name))
+ task_processors[model_name + ":core:text2text:multimodal-chatwithtools"] = lambda: MultimodalChatWithToolsProcessor(generate_chat_model(file_name), modalities)
task_processors[model_name + ":core:text2text:reformatparagraphs"] = lambda: ReformatParagraphsProcessor(generate_chat_model(file_name))
+ if "vision" in modalities:
+ task_processors[model_name + ":core:analyze-images"] = lambda: AnalyzeImagesProcessor(generate_chat_model(file_name))
+ task_processors[model_name + ":core:image2text:ocr"] = lambda: OcrProcessor(generate_chat_model(file_name))
diff --git a/tests/multimodal-ocr-text.png b/tests/multimodal-ocr-text.png
new file mode 100644
index 0000000..78cfca5
Binary files /dev/null and b/tests/multimodal-ocr-text.png differ