From ae795776be027bbe346b65cac55c0af60f3c3345 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 10 Dec 2025 14:35:12 +0100 Subject: [PATCH 1/2] feat: Implement optional input slots for chat memories Signed-off-by: Marcel Klehr --- lib/TaskProcessing/AudioToAudioChatProvider.php | 10 ++++++++++ lib/TaskProcessing/TextToTextChatProvider.php | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/lib/TaskProcessing/AudioToAudioChatProvider.php b/lib/TaskProcessing/AudioToAudioChatProvider.php index 9e0502c0..bdb9fb5c 100644 --- a/lib/TaskProcessing/AudioToAudioChatProvider.php +++ b/lib/TaskProcessing/AudioToAudioChatProvider.php @@ -81,6 +81,11 @@ public function getOptionalInputShape(): array { $this->l->t('The voice used to generate speech'), EShapeType::Enum ), + 'memories' => new ShapeDescriptor( + $this->l->t('Memories'), + $this->l->t('The memories to be injected into the chat session.'), + EShapeType::ListOfTexts + ), ]; if (!$isUsingOpenAi) { $ois['tts_model'] = new ShapeDescriptor( @@ -165,6 +170,11 @@ public function process(?string $userId, array $input, callable $reportProgress) } $systemPrompt = $input['system_prompt']; + if (isset($input['memories']) && is_array($input['memories'])) { + /** @psalm-suppress InvalidArgument */ + $systemPrompt .= "\n\nIf they are relevant, take into account the following memories:\n" . implode("\n\n", $input['memories']) . "\n\nDo not mention these memories explicitly. You may use them as context, but do not repeat them. At most, you can mention that you remember something."; + } + if (!isset($input['history']) || !is_array($input['history'])) { throw new RuntimeException('Invalid chat history, array expected'); } diff --git a/lib/TaskProcessing/TextToTextChatProvider.php b/lib/TaskProcessing/TextToTextChatProvider.php index 52084d7b..a6622314 100644 --- a/lib/TaskProcessing/TextToTextChatProvider.php +++ b/lib/TaskProcessing/TextToTextChatProvider.php @@ -60,6 +60,11 @@ public function getOptionalInputShape(): array { $this->l->t('The maximum number of words/tokens that can be generated in the completion.'), EShapeType::Number ), + 'memories' => new ShapeDescriptor( + $this->l->t('Memories'), + $this->l->t('The memories to be injected into the chat session.'), + EShapeType::ListOfTexts + ), ]; } @@ -97,6 +102,11 @@ public function process(?string $userId, array $input, callable $reportProgress) } $systemPrompt = $input['system_prompt']; + if (isset($input['memories']) && is_array($input['memories'])) { + /** @psalm-suppress InvalidArgument */ + $systemPrompt .= "\n\nIf they are relevant, take into account the following memories:\n" . implode("\n\n", $input['memories']) . "\n\nDo not mention these memories explicitly. You may use them as context, but do not repeat them. At most, you can mention that you remember something."; + } + if (!isset($input['history']) || !is_array($input['history'])) { throw new RuntimeException('Invalid history'); } From f1adaabe4ae2b8f598da7fc51f4a7a2d01585c3d Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 10 Dec 2025 14:55:10 +0100 Subject: [PATCH 2/2] fix: Improve memories prompt to inform the llm that it can remember things Signed-off-by: Marcel Klehr --- lib/TaskProcessing/AudioToAudioChatProvider.php | 4 ++-- lib/TaskProcessing/TextToTextChatProvider.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/TaskProcessing/AudioToAudioChatProvider.php b/lib/TaskProcessing/AudioToAudioChatProvider.php index bdb9fb5c..b1362489 100644 --- a/lib/TaskProcessing/AudioToAudioChatProvider.php +++ b/lib/TaskProcessing/AudioToAudioChatProvider.php @@ -170,9 +170,9 @@ public function process(?string $userId, array $input, callable $reportProgress) } $systemPrompt = $input['system_prompt']; - if (isset($input['memories']) && is_array($input['memories'])) { + if (isset($input['memories']) && is_array($input['memories']) && count($input['memories'])) { /** @psalm-suppress InvalidArgument */ - $systemPrompt .= "\n\nIf they are relevant, take into account the following memories:\n" . implode("\n\n", $input['memories']) . "\n\nDo not mention these memories explicitly. You may use them as context, but do not repeat them. At most, you can mention that you remember something."; + $systemPrompt .= "\n\nYou can remember things from other conversation with the user. If they are relevant, take into account the following memories:\n" . implode("\n\n", $input['memories']) . "\n\nDo not mention these memories explicitly. You may use them as context, but do not repeat them. At most, you can mention that you remember something."; } if (!isset($input['history']) || !is_array($input['history'])) { diff --git a/lib/TaskProcessing/TextToTextChatProvider.php b/lib/TaskProcessing/TextToTextChatProvider.php index a6622314..c50d7cc3 100644 --- a/lib/TaskProcessing/TextToTextChatProvider.php +++ b/lib/TaskProcessing/TextToTextChatProvider.php @@ -102,9 +102,9 @@ public function process(?string $userId, array $input, callable $reportProgress) } $systemPrompt = $input['system_prompt']; - if (isset($input['memories']) && is_array($input['memories'])) { + if (isset($input['memories']) && is_array($input['memories']) && count($input['memories'])) { /** @psalm-suppress InvalidArgument */ - $systemPrompt .= "\n\nIf they are relevant, take into account the following memories:\n" . implode("\n\n", $input['memories']) . "\n\nDo not mention these memories explicitly. You may use them as context, but do not repeat them. At most, you can mention that you remember something."; + $systemPrompt .= "\n\nYou can remember things from other conversations with the user. If they are relevant, take into account the following memories:\n" . implode("\n\n", $input['memories']) . "\n\nDo not mention these memories explicitly. You may use them as context, but do not repeat them. At most, you can mention that you remember something."; } if (!isset($input['history']) || !is_array($input['history'])) {