feat: add context chat provider

.github/workflows/psalm-matrix.yml

@@ -42,7 +42,7 @@ jobs:
       # do not stop on another job's failure
       fail-fast: false
       matrix:
-        ocp-version: [ 'dev-stable31', 'dev-stable32', 'dev-master' ]
+        ocp-version: [ 'dev-stable32', 'dev-master' ]
 
     name: static-psalm-analysis ${{ matrix.ocp-version }}
     steps:

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         php-versions: ['8.2', '8.3']
-        nextcloud-versions: ['master', 'stable31']
+        nextcloud-versions: ['master']
         include:
           - php-versions: '8.4'
             nextcloud-versions: 'master'

appinfo/info.xml

@@ -57,6 +57,7 @@ Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud
 	</dependencies>
 	<background-jobs>
 		<job>OCA\Mail\BackgroundJob\CleanupJob</job>
+		<job>OCA\Mail\BackgroundJob\ContextChat\SubmitContentJob</job>
 		<job>OCA\Mail\BackgroundJob\OutboxWorkerJob</job>
 		<job>OCA\Mail\BackgroundJob\IMipMessageJob</job>
 		<job>OCA\Mail\BackgroundJob\DraftsJob</job>

lib/AppInfo/Application.php

@@ -11,6 +11,7 @@
 namespace OCA\Mail\AppInfo;
 
 use Horde_Translation;
+use OCA\Mail\ContextChat\ContextChatProvider;
 use OCA\Mail\Contracts\IAttachmentService;
 use OCA\Mail\Contracts\IAvatarService;
 use OCA\Mail\Contracts\IDkimService;
@@ -73,6 +74,7 @@
 use OCP\AppFramework\Bootstrap\IBootContext;
 use OCP\AppFramework\Bootstrap\IBootstrap;
 use OCP\AppFramework\Bootstrap\IRegistrationContext;
+use OCP\ContextChat\Events\ContentProviderRegisterEvent;
 use OCP\DB\Events\AddMissingIndicesEvent;
 use OCP\IServerContainer;
 use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
@@ -168,6 +170,13 @@ public function register(IRegistrationContext $context): void {
 		Horde_Translation::setHandler('Horde_Imap_Client', new HordeTranslationHandler());
 		Horde_Translation::setHandler('Horde_Mime', new HordeTranslationHandler());
 		Horde_Translation::setHandler('Horde_Smtp', new HordeTranslationHandler());
+
+		// Added in version 5.6.0
+		if (class_exists(ContentProviderRegisterEvent::class)) {
+			$context->registerEventListener(ContentProviderRegisterEvent::class, ContextChatProvider::class);
+			$context->registerEventListener(NewMessagesSynchronized::class, ContextChatProvider::class);
+			$context->registerEventListener(MessageDeletedEvent::class, ContextChatProvider::class);
+		}
 	}
 
 	#[\Override]

lib/BackgroundJob/ContextChat/ScheduleJob.php

@@ -0,0 +1,81 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\BackgroundJob\ContextChat;
+
+use OCA\Mail\Exception\ServiceException;
+use OCA\Mail\Service\AccountService;
+use OCA\Mail\Service\ContextChat\ContextChatSettingsService;
+use OCA\Mail\Service\ContextChat\TaskService;
+use OCA\Mail\Service\MailManager;
+use OCP\AppFramework\Db\DoesNotExistException;
+use OCP\AppFramework\Db\MultipleObjectsReturnedException;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJob;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\TimedJob;
+use OCP\ContextChat\IContentManager;
+use OCP\DB\Exception;
+use Psr\Log\LoggerInterface;
+
+class ScheduleJob extends TimedJob {
+	public function __construct(
+		ITimeFactory $time,
+		private TaskService $taskService,
+		private AccountService $accountService,
+		private MailManager $mailManager,
+		private LoggerInterface $logger,
+		private IJobList $jobList,
+		private ContextChatSettingsService $contextChatSettingsService,
+		private IContentManager $contentManager,
+	) {
+		parent::__construct($time);
+
+		$this->setInterval(60 * 60 * 24);
+		$this->setTimeSensitivity(IJob::TIME_INSENSITIVE);
+	}
+
+	#[\Override]
+	protected function run($argument): void {
+		$accountId = $argument['accountId'];
+
+		if (!$this->contentManager->isContextChatAvailable()) {
+			return;
+		}
+
+		try {
+			$account = $this->accountService->findById($accountId);
+		} catch (DoesNotExistException $e) {
+			$this->logger->debug('Could not find account <' . $accountId . '> removing from jobs');
+			$this->jobList->remove(self::class, $argument);
+			return;
+		}
+
+		if (!$this->contextChatSettingsService->isIndexingEnabled($account->getUserId())) {
+			$this->logger->debug("indexing is turned off for account $accountId");
+			return;
+		}
+
+		try {
+			$mailboxes = $this->mailManager->getMailboxes($account);
+		} catch (ServiceException $e) {
+			$this->logger->debug('Could not find mailboxes for account <' . $accountId . '>');
+			return;
+		}
+
+		foreach ($mailboxes as $mailbox) {
+			try {
+				$this->taskService->findByMailboxId($mailbox->getId());
+				$this->taskService->updateOrCreate($mailbox->getId(), 0);
+			} catch (DoesNotExistException|MultipleObjectsReturnedException|Exception $e) {
+				$this->logger->warning('Could not schedule context chat indexing tasks for mailbox <' . $mailbox->getId() . '>');
+			}
+		}
+	}
+}

lib/BackgroundJob/ContextChat/SubmitContentJob.php

@@ -0,0 +1,171 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\BackgroundJob\ContextChat;
+
+use OCA\Mail\ContextChat\ContextChatProvider;
+use OCA\Mail\Db\MailboxMapper;
+use OCA\Mail\Db\Message;
+use OCA\Mail\Db\MessageMapper;
+use OCA\Mail\Exception\ServiceException;
+use OCA\Mail\Exception\SmimeDecryptException;
+use OCA\Mail\IMAP\IMAPClientFactory;
+use OCA\Mail\Service\AccountService;
+use OCA\Mail\Service\ContextChat\TaskService;
+use OCA\Mail\Service\MailManager;
+use OCP\AppFramework\Db\DoesNotExistException;
+use OCP\AppFramework\Db\MultipleObjectsReturnedException;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJob;
+use OCP\BackgroundJob\TimedJob;
+use OCP\ContextChat\ContentItem;
+use OCP\ContextChat\IContentManager;
+use OCP\DB\Exception;
+use Psr\Log\LoggerInterface;
+
+class SubmitContentJob extends TimedJob {
+	public function __construct(
+		ITimeFactory $time,
+		private TaskService $taskService,
+		private AccountService $accountService,
+		private MailManager $mailManager,
+		private MessageMapper $messageMapper,
+		private IMAPClientFactory $clientFactory,
+		private ContextChatProvider $contextChatProvider,
+		private IContentManager $contentManager,
+		private LoggerInterface $logger,
+		private MailboxMapper $mailboxMapper,
+	) {
+		parent::__construct($time);
+
+		$this->setAllowParallelRuns(false);
+		$this->setInterval(ContextChatProvider::CONTEXT_CHAT_JOB_INTERVAL);
+		$this->setTimeSensitivity(IJob::TIME_INSENSITIVE);
+	}
+
+	#[\Override]
+	protected function run($argument): void {
+		if (!$this->contentManager->isContextChatAvailable()) {
+			return;
+		}
+
+		try {
+			$task = $this->taskService->findNext();
+		} catch (Exception $e) {
+			$this->logger->warning('Exception occurred when trying to fetch next task', ['exception' => $e]);
+			return;
+		} catch (DoesNotExistException $e) {
+			// nothing to be done, let's defer to the next iteration of this job
+			return;
+		} catch (MultipleObjectsReturnedException $e) {
+			$this->logger->warning('Multiple tasks found for context chat. This is unexpected.', ['exception' => $e]);
+			return;
+		}
+
+		try {
+			$mailbox = $this->mailboxMapper->findById($task->getMailboxId());
+		} catch (ServiceException $e) {
+			$this->logger->warning('Multiple mailboxes found for context chat task, but only one expected. ERROR!', ['exception' => $e]);
+			return;
+		} catch (DoesNotExistException $e) {
+			// mailbox does not exist, lets wait for this task to be removed
+			return;
+		}
+
+		$processMailsAfter = $this->time->getTime() - ContextChatProvider::CONTEXT_CHAT_MESSAGE_MAX_AGE;
+		$messageIds = $this->messageMapper->findIdsAfter($mailbox, $task->getLastMessageId(), $processMailsAfter, ContextChatProvider::CONTEXT_CHAT_IMPORT_MAX_ITEMS);
+
+		if (empty($messageIds)) {
+			try {
+				$this->taskService->delete($task->getId());
+			} catch (MultipleObjectsReturnedException|Exception $e) {
+				$this->logger->warning('Exception occurred when trying to delete task', ['exception' => $e]);
+			}
+			return;
+		}
+
+		try {
+			$account = $this->accountService->findById($mailbox->getAccountId());
+		} catch (DoesNotExistException $e) {
+			// well, what do you know. Then let's just skip this and wait for the next iteration of this job. tasks should be cascade deleted anyway
+			return;
+		}
+
+		$messages = $this->messageMapper->findByIds($account->getUserId(), $messageIds, 'asc', 'id');
+
+		if (empty($messages)) {
+			try {
+				$this->taskService->delete($task->getId());
+			} catch (MultipleObjectsReturnedException|Exception $e) {
+				$this->logger->warning('Exception occurred when trying to delete task', ['exception' => $e]);
+			}
+			return;
+		}
+
+
+		$client = $this->clientFactory->getClient($account);
+		$items = [];
+
+		try {
+			$startTime = $this->time->getTime();
+			foreach ($messages as $message) {
+				if ($this->time->getTime() - $startTime > ContextChatProvider::CONTEXT_CHAT_JOB_INTERVAL) {
+					break;
+				}
+				try {
+					$imapMessage = $this->mailManager->getImapMessage($client, $account, $mailbox, $message->getUid(), true);
+				} catch (ServiceException $e) {
+					// couldn't load message, let's skip it. Retrying would be too costly
+					continue;
+				} catch (SmimeDecryptException $e) {
+					// encryption problem, skip this message
+					continue;
+				}
+
+
+				// Skip encrypted messages
+				if ($imapMessage->isEncrypted()) {
+					continue;
+				}
+
+
+				$fullMessage = $imapMessage->getFullMessage($imapMessage->getUid(), true);
+
+
+				$items[] = new ContentItem(
+					$mailbox->getId() . ':' . $message->getId(),
+					$this->contextChatProvider->getId(),
+					$imapMessage->getSubject(),
+					$fullMessage['body'] ?? '',
+					'E-Mail',
+					$imapMessage->getSentDate(),
+					[$account->getUserId()],
+				);
+			}
+		} catch (\Throwable $e) {
+			$this->logger->warning('Exception occurred when trying to fetch messages for context chat', ['exception' => $e]);
+		} finally {
+			try {
+				$client->close();
+			} catch (\Horde_Imap_Client_Exception $e) {
+				$this->logger->debug('Failed to close IMAP client', ['exception' => $e]);
+			}
+		}
+
+		if (count($items) > 0) {
+			$this->contentManager->submitContent($this->contextChatProvider->getAppId(), $items);
+		}
+
+		try {
+			$this->taskService->setLastMessage($task->getMailboxId(), $message?->getId() ?? $messageIds[0]);
+		} catch (MultipleObjectsReturnedException|Exception $e) {
+			$this->logger->warning('Exception occurred when trying to update task', ['exception' => $e]);
+		}
+	}
+}