Skip to content

Commit 73c1f74

Browse files
lukasdotcomlukasdotcom
authored
Merge pull request #602 from nextcloud/multimodal-chat
feat: support multimodal chat
2 parents 8824437 + dd7156b commit 73c1f74

9 files changed

Lines changed: 478 additions & 51 deletions

File tree

‎lib/Listener/BeforeTemplateRenderedListener.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ public function handle(Event $event): void {
7676
$this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat);
7777
$agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes());
7878
$this->initialStateService->provideInitialState('agency_available', $agencyAvailable);
79+
80+
$multimodalChatAvailable = $agencyAvailable
81+
? (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes()))
82+
: (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes()));
83+
$this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable);
7984
}
8085
if (class_exists(\OCA\Viewer\Event\LoadViewer::class)) {
8186
$this->eventDispatcher->dispatchTyped(new \OCA\Viewer\Event\LoadViewer());

‎lib/Listener/ChattyLLMTaskListener.php‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ public function handle(Event $event): void {
8282
&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID;
8383
$isAgencyAudioChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction')
8484
&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID;
85+
$isMultimodalChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')
86+
&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID;
87+
$isMultimodalAgencyChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')
88+
&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID;
8589

8690
$taskOutput = $task->getOutput();
8791

@@ -128,6 +132,17 @@ public function handle(Event $event): void {
128132
// the task is not an audio one, but we might still need to Tts the answer
129133
// if it is a response to a ContextAgentInteraction confirmation that was asked about an audio message
130134
$this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId());
135+
if ($isMultimodalChat || $isMultimodalAgencyChat) {
136+
$attachments = $taskOutput['output_attachments'] ?? [];
137+
$attachments = array_map(function ($attachment) use ($task) {
138+
return [
139+
'type' => 'File',
140+
'file_id' => $attachment,
141+
'ocp_task_id' => $task->getId(),
142+
];
143+
}, $attachments);
144+
$message->setAttachments(json_encode($attachments));
145+
}
131146
}
132147
try {
133148
$this->messageMapper->insert($message);
@@ -138,7 +153,7 @@ public function handle(Event $event): void {
138153
$session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId);
139154

140155
// store the conversation token and the actions if we are using the agency feature
141-
if ($isAgency || $isAgencyAudioChat) {
156+
if ($isAgency || $isAgencyAudioChat || $isMultimodalAgencyChat) {
142157
$conversationToken = ($taskOutput['conversation_token'] ?? null) ?: null;
143158
$pendingActions = ($taskOutput['actions'] ?? null) ?: null;
144159
$session->setAgencyConversationToken($conversationToken);

‎lib/Service/AssistantService.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,7 @@ private function extractFileIdsFromTask(Task $task): array {
773773
/** @var int|list<int> $inputSlot */
774774
$inputSlot = $task->getInput()[$key];
775775
if (is_array($inputSlot)) {
776-
$ids += $inputSlot;
776+
$ids = array_merge($ids, $inputSlot);
777777
} else {
778778
$ids[] = $inputSlot;
779779
}
@@ -785,14 +785,14 @@ private function extractFileIdsFromTask(Task $task): array {
785785
/** @var int|list<int> $outputSlot */
786786
$outputSlot = $task->getOutput()[$key];
787787
if (is_array($outputSlot)) {
788-
$ids += $outputSlot;
788+
$ids = array_merge($ids, $outputSlot);
789789
} else {
790790
$ids[] = $outputSlot;
791791
}
792792
}
793793
}
794794
}
795-
return array_values($ids);
795+
return $ids;
796796
}
797797

798798
/**

‎lib/Service/ChatService.php‎

Lines changed: 147 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,13 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
399399
// audio agency
400400
$fileId = $audioAttachment['file_id'];
401401
$taskId = $this->scheduleAgencyAudioTask($userId, $fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId());
402+
} elseif ($this->isMultimodalContextAgentAvailable()) {
403+
// multimodal agency
404+
$prompt = $lastUserMessage->getContent();
405+
$inputAttachments = array_map(static function (array $attachment) {
406+
return $attachment['file_id'];
407+
}, $lastAttachments);
408+
$taskId = $this->scheduleAgencyMultimodalTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, $inputAttachments);
402409
} else {
403410
// classic agency
404411
$prompt = $lastUserMessage->getContent();
@@ -446,14 +453,40 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
446453
$fileId = $audioAttachment['file_id'];
447454
$taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId());
448455
} else {
449-
// for a text chat task, let's only use text in the history
450-
$history = array_map(static function (Message $message) {
451-
return json_encode([
452-
'role' => $message->getRole(),
453-
'content' => $message->getContent(),
454-
]);
455-
}, $history);
456-
$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $history, $sessionId);
456+
if ($this->isMultimodalChatAvailable()) {
457+
// for a multimodal chat also attachments need to be added to the history
458+
$historyMessages = array_map(static function (Message $message) {
459+
$attachments = $message->jsonSerialize()['attachments'];
460+
$content = array_map(static function (array $attachment) {
461+
$newAttachment = ['type' => 'file', 'file_id' => $attachment['file_id']];
462+
if (isset($attachment['ocp_task_id'])) {
463+
$newAttachment['ocp_task_id'] = $attachment['ocp_task_id'];
464+
}
465+
return $newAttachment;
466+
}, $attachments);
467+
$content[] = [
468+
'type' => 'text',
469+
'text' => $message->getContent(),
470+
];
471+
return json_encode([
472+
'role' => $message->getRole(),
473+
'content' => $content,
474+
]);
475+
}, $history);
476+
$lastAttachments = array_map(static function (array $attachment) {
477+
return $attachment['file_id'];
478+
}, $lastAttachments);
479+
$taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $lastAttachments);
480+
} else {
481+
// for a text chat task only use text in the history
482+
$historyMessages = array_map(static function (Message $message) {
483+
return json_encode([
484+
'role' => $message->getRole(),
485+
'content' => $message->getContent(),
486+
]);
487+
}, $history);
488+
$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId);
489+
}
457490
}
458491
}
459492
return $taskId;
@@ -567,6 +600,20 @@ public function isContextAgentAudioAvailable(): bool {
567600
return in_array(\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
568601
}
569602

603+
public function isMultimodalChatAvailable(): bool {
604+
if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')) {
605+
return false;
606+
}
607+
return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
608+
}
609+
610+
public function isMultimodalContextAgentAvailable(): bool {
611+
if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')) {
612+
return false;
613+
}
614+
return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
615+
}
616+
570617
private function getAudioHistory(array $history): array {
571618
// history is a list of JSON strings
572619
// The content is the remote audio ID (or the transcription as fallback)
@@ -682,6 +729,49 @@ private function scheduleLLMChatTask(
682729
return $task->getId() ?? 0;
683730
}
684731

732+
/**
733+
* Schedule a Multimodal Chat task
734+
*
735+
* @throws BadRequestException
736+
* @throws InternalException
737+
*/
738+
private function scheduleMultimodalChatTask(
739+
?string $userId,
740+
string $content,
741+
string $systemPrompt,
742+
array $history,
743+
int $sessionId,
744+
array $attachmentsHistory,
745+
): int {
746+
$customId = 'chatty-llm:' . $sessionId;
747+
$this->checkIfSessionIsThinking($userId, $customId);
748+
$input = [
749+
'input' => $content,
750+
'system_prompt' => $systemPrompt,
751+
'history' => $history,
752+
'input_attachments' => $attachmentsHistory,
753+
'tools' => '[]', // Empty tools as there is not a non tools version
754+
'tool_message' => '',
755+
];
756+
/** @psalm-suppress UndefinedClass */
757+
$task = new Task(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId);
758+
/** @psalm-suppress UndefinedMethod */
759+
$task->setPreferStreaming(true);
760+
try {
761+
$this->taskProcessingManager->scheduleTask($task);
762+
} catch (PreConditionNotMetException $e) {
763+
throw new BadRequestException('pre_condition_not_met', previous: $e);
764+
} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
765+
throw new BadRequestException('unauthorized', previous: $e);
766+
} catch (ValidationException $e) {
767+
throw new BadRequestException('validation_failed', previous: $e);
768+
} catch (\OCP\TaskProcessing\Exception\Exception $e) {
769+
$this->logger->error($e->getMessage(), ['exception' => $e]);
770+
throw new InternalException(previous: $e);
771+
}
772+
return $task->getId() ?? 0;
773+
}
774+
685775
/**
686776
* Schedule an agency chat task
687777
*
@@ -731,6 +821,55 @@ private function scheduleAgencyTask(
731821
return $task->getId() ?? 0;
732822
}
733823

824+
/**
825+
* Schedule a multimodal agency chat task
826+
*
827+
* @param list<int> $inputAttachments
828+
* @throws BadRequestException
829+
* @throws InternalException
830+
*/
831+
private function scheduleAgencyMultimodalTask(
832+
?string $userId,
833+
string $content,
834+
int $confirmation,
835+
string $conversationToken,
836+
int $sessionId,
837+
array $inputAttachments,
838+
): int {
839+
$customId = 'chatty-llm:' . $sessionId;
840+
$this->checkIfSessionIsThinking($userId, $customId);
841+
$taskInput = [
842+
'input' => $content,
843+
'input_attachments' => $inputAttachments,
844+
'confirmation' => $confirmation,
845+
'conversation_token' => $conversationToken,
846+
];
847+
$taskInput['memories'] = $this->sessionSummaryService->getMemories($userId);
848+
/** @psalm-suppress UndefinedClass */
849+
$task = new Task(
850+
\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID,
851+
$taskInput,
852+
Application::APP_ID . ':chatty-llm',
853+
$userId,
854+
$customId
855+
);
856+
/** @psalm-suppress UndefinedMethod */
857+
$task->setPreferStreaming(true);
858+
try {
859+
$this->taskProcessingManager->scheduleTask($task);
860+
} catch (PreConditionNotMetException $e) {
861+
throw new BadRequestException('pre_condition_not_met', previous: $e);
862+
} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
863+
throw new BadRequestException('unauthorized', previous: $e);
864+
} catch (ValidationException $e) {
865+
throw new BadRequestException('validation_failed', previous: $e);
866+
} catch (\OCP\TaskProcessing\Exception\Exception $e) {
867+
$this->logger->error($e->getMessage(), ['exception' => $e]);
868+
throw new InternalException(previous: $e);
869+
}
870+
return $task->getId() ?? 0;
871+
}
872+
734873
/**
735874
* Schedule an audio chat task
736875
* @throws BadRequestException

‎src/components/ChattyLLM/ChattyLLMInputForm.vue‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -665,16 +665,20 @@ export default {
665665
return session.timestamp ? (' ' + moment(session.timestamp * 1000).format('LLL')) : t('assistant', 'Untitled conversation')
666666
},
667667
668-
async handleSubmit(event) {
668+
async handleSubmit(attachedFileIds) {
669669
if (this.chatContent.trim() === '') {
670670
console.debug('empty message')
671671
return
672672
}
673+
const attachments = attachedFileIds.map(fileId => ({ type: SHAPE_TYPE_NAMES.File, file_id: fileId }))
673674
674675
const role = Roles.HUMAN
675676
const content = this.chatContent.trim()
676677
const timestamp = +new Date() / 1000 | 0
677678
console.debug('[Assistant] submit text', content)
679+
if (attachedFileIds.length > 0) {
680+
console.debug('[Assistant] submit attachments', attachments)
681+
}
678682
679683
if (this.active === null) {
680684
await this.newSession()
@@ -686,10 +690,10 @@ export default {
686690
this.active.agencyAnswered = true
687691
}
688692
689-
this.messages.push({ role, content, timestamp, session_id: this.active.id })
693+
this.messages.push({ role, content, timestamp, session_id: this.active.id, attachments })
690694
this.chatContent = ''
691695
this.scrollToLastMessage()
692-
await this.newMessage(role, content, timestamp, this.active.id)
696+
await this.newMessage(role, content, timestamp, this.active.id, attachments)
693697
},
694698
695699
async handleSubmitAudio(fileId) {

‎src/components/ChattyLLM/DocumentPreviews.vue‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<!--
2+
- SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
3+
- SPDX-License-Identifier: AGPL-3.0-or-later
4+
-->
5+
<template>
6+
<div class="document-previews">
7+
<div class="document-previews__list">
8+
<div v-for="fileId in fileIds"
9+
:key="fileId"
10+
class="document-previews__item">
11+
<FileDisplay :file-id="fileId"
12+
:task-id="null" />
13+
<NcButton class="document-previews__delete"
14+
variant="tertiary"
15+
:aria-label="t('assistant', 'Remove this media')"
16+
@click="$emit('delete', fileId)">
17+
<template #icon>
18+
<TrashCanOutlineIcon :size="20" />
19+
</template>
20+
</NcButton>
21+
</div>
22+
</div>
23+
</div>
24+
</template>
25+
26+
<script>
27+
import TrashCanOutlineIcon from 'vue-material-design-icons/TrashCanOutline.vue'
28+
29+
import NcButton from '@nextcloud/vue/components/NcButton'
30+
31+
import FileDisplay from '../fields/FileDisplay.vue'
32+
33+
export default {
34+
name: 'DocumentPreviews',
35+
36+
components: {
37+
FileDisplay,
38+
NcButton,
39+
TrashCanOutlineIcon,
40+
},
41+
42+
props: {
43+
fileIds: {
44+
type: Array,
45+
required: true,
46+
},
47+
},
48+
49+
emits: [
50+
'delete',
51+
],
52+
}
53+
</script>
54+
55+
<style lang="scss" scoped>
56+
.document-previews {
57+
overflow-x: auto;
58+
59+
&__list {
60+
display: flex;
61+
flex-direction: row;
62+
gap: 8px;
63+
}
64+
65+
&__item {
66+
position: relative;
67+
padding: 8px;
68+
border-radius: var(--border-radius-large);
69+
background-color: var(--color-main-background);
70+
71+
&:hover {
72+
background-color: var(--color-primary-element-light-hover);
73+
}
74+
}
75+
76+
&__delete {
77+
position: absolute;
78+
right: 0;
79+
bottom: 0;
80+
}
81+
}
82+
</style>

0 commit comments

Comments
 (0)