Add ocr task type and add integration test for multimodality

Signed-off-by: Lukas Schaefer <lukas@lschaefer.xyz>

Author: lukasdotcom

.github/workflows/integration_test.yml

@@ -214,6 +214,51 @@ jobs:
           [ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ]
           [ $STREAMING_UPDATES -gt 0 ]
 
+      # core:image2text:ocr exists since Nextcloud 33
+      - name: Run multimodal OCR task
+        if: contains(fromJSON('["master", "stable34", "stable33"]'), matrix.server-versions)
+        env:
+          CREDS: "admin:password"
+          EXPECTED_TEXT: "Nextcloud Image"
+        run: |
+          set -x
+          IMAGE_PATH="${{ env.APP_NAME }}/tests/multimodal-ocr-text.png"
+          REMOTE_PATH="multimodal-ocr-text.png"
+
+          curl -u "$CREDS" -T "$IMAGE_PATH" "http://localhost:8080/remote.php/dav/files/admin/$REMOTE_PATH"
+
+          FILE_ID=$(curl -s -u "$CREDS" -X PROPFIND \
+            -H "Depth: 0" \
+            -H "Content-Type: application/xml" \
+            --data '<?xml version="1.0"?><d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop><oc:fileid/></d:prop></d:propfind>' \
+            "http://localhost:8080/remote.php/dav/files/admin/$REMOTE_PATH" \
+            | tr '\n' ' ' \
+            | grep -oE '<oc:fileid>[0-9]+</oc:fileid>' \
+            | head -1 \
+            | grep -oE '[0-9]+')
+          echo "Uploaded image file id: $FILE_ID"
+          [ -n "$FILE_ID" ]
+
+          TASK=$(curl -X POST -u "$CREDS" -H "oCS-APIRequest: true" -H "Content-type: application/json" \
+            "http://localhost:8080/ocs/v2.php/taskprocessing/schedule?format=json" \
+            --data-raw "{\"input\": {\"input\": [$FILE_ID]}, \"type\": \"core:image2text:ocr\", \"appId\": \"test\", \"customId\": \"\"}")
+          echo $TASK
+          TASK_ID=$(echo $TASK | jq '.ocs.data.task.id')
+          NEXT_WAIT_TIME=0
+          TASK_STATUS='"STATUS_SCHEDULED"'
+          until [ $NEXT_WAIT_TIME -eq 35 ] || [ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ] || [ "$TASK_STATUS" == '"STATUS_FAILED"' ]; do
+            TASK=$(curl -u "$CREDS" -H "oCS-APIRequest: true" "http://localhost:8080/ocs/v2.php/taskprocessing/task/$TASK_ID?format=json")
+            echo $TASK
+            TASK_STATUS=$(echo $TASK | jq '.ocs.data.task.status')
+            echo $TASK_STATUS
+            sleep $(( NEXT_WAIT_TIME++ ))
+          done
+          curl -u "$CREDS" -H "oCS-APIRequest: true" "http://localhost:8080/ocs/v2.php/taskprocessing/task/$TASK_ID?format=json"
+          [ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ]
+          TASK_OUTPUT=$(echo $TASK | jq -r '.ocs.data.task.output.output | join("\n")')
+          echo "Model output: $TASK_OUTPUT"
+          echo "$TASK_OUTPUT" | grep -qi "$EXPECTED_TEXT"
+
       - name: Show logs
         if: always()
         run: |

lib/main.py

@@ -373,7 +373,7 @@ async def enabled_handler(enabled: bool, nc: AsyncNextcloudApp) -> str:
                     optional_input_defaults=get_optional_input_defaults(task),
                     optional_output_shape=[
                         ShapeDescriptor(name="reasoning", description="Reasoning trace produced by the model, if any", shape_type=ShapeType.TEXT)
-                    ] if task != "core:text2text:summary" else [],
+                    ] if task != "core:text2text:summary" and task != "core:image2text:ocr" else [],
                 )
                 await nc.providers.task_processing.register(provider)
                 await log(nc, LogLvl.INFO, f"Registered {task_processor_name}")

lib/ocr.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Processor for core:image2text:ocr — vision-based OCR."""
+from typing import Any
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.runnables import Runnable
+
+from task_files import fetch_file_bytes
+from streaming import StreamContext, run_runnable_with_streaming
+from multimodal_chatwithtools import MAX_ATTACHMENTS_COUNT
+
+
+class OcrProcessor:
+    """Extract text from one or more images using a vision-capable chat model."""
+
+    runnable: Runnable
+    system_prompt: str = (
+        "You're an OCR assistant. "
+        "Extract all text visible in the provided image. "
+        "Output only the extracted text, nothing else."
+    )
+    user_prompt: str = "Extract all text from this image. Reply with only the extracted text."
+
+    def __init__(self, runnable: Runnable):
+        self.runnable = runnable
+
+    async def __call__(
+            self,
+            inputs: dict[str, Any],
+            context: StreamContext | None = None,
+    ) -> dict[str, Any]:
+        if context is None or context.nc is None:
+            raise ValueError("StreamContext with Nextcloud client is required for OCR")
+
+        files = inputs.get("input") or []
+        if not files:
+            raise ValueError("core:image2text:ocr requires at least one file")
+        if len(files) > MAX_ATTACHMENTS_COUNT:
+            raise ValueError(f"Too many files (max {MAX_ATTACHMENTS_COUNT})")
+
+        texts: list[str] = []
+        for file_id in files:
+            fetched = await fetch_file_bytes(context.nc, file_id)
+            if not fetched["mime"].startswith("image/"):
+                raise ValueError(f"File MIME type {fetched['mime']} is not supported for OCR")
+
+            output = await run_runnable_with_streaming(
+                self.runnable,
+                [
+                    SystemMessage(self.system_prompt),
+                    HumanMessage(content=[
+                        {"type": "text", "text": self.user_prompt},
+                        {"type": "image_url", "image_url": {"url": fetched["data_url"]}},
+                    ]),
+                ],
+                context,
+                stream_payload_transform=lambda partial: {"output": [*texts, partial]},
+            )
+            texts.append(output)
+
+        return {
+            "output": texts,
+        }

lib/task_processors.py

@@ -32,6 +32,7 @@
 from summarize import SummarizeProcessor
 from reformat_paragraphs import ReformatParagraphsProcessor
 from analyze_images import AnalyzeImagesProcessor
+from ocr import OcrProcessor
 from multimodal_chatwithtools import MultimodalChatWithToolsProcessor
 
 dir_path = os.path.dirname(os.path.realpath(__file__))
@@ -263,3 +264,4 @@ def generate_task_processors_for_model(file_name, task_processors):
     task_processors[model_name + ":core:text2text:reformatparagraphs"] = lambda: ReformatParagraphsProcessor(generate_chat_model(file_name))
     if "vision" in modalities:
         task_processors[model_name + ":core:analyze-images"] = lambda: AnalyzeImagesProcessor(generate_chat_model(file_name))
+        task_processors[model_name + ":core:image2text:ocr"] = lambda: OcrProcessor(generate_chat_model(file_name))