Add support for live translations in calls

docs/capabilities.md

@@ -203,3 +203,5 @@
 * `federated-shared-items` - Whether shared items endpoints can be called in a federated conversation
 * `config => chat => style` (local) - User selected chat style (split or unified for now)
 * `scheduled-messages` (local) - Whether a user can schedule messages
+* `config => call => live-translation` - Whether live translation is supported in calls
+* `config => call => live-transcription-target-language-id` (local) - User defined string value with the id of the target language to use for live translations

lib/Capabilities.php

@@ -171,6 +171,7 @@ class Capabilities implements IPublicCapability {
 			'can-upload-background',
 			'start-without-media',
 			'blur-virtual-background',
+			'live-transcription-target-language-id',
 		],
 		'chat' => [
 			'read-privacy',
@@ -256,8 +257,8 @@ public function getCapabilities(): array {
 					'max-duration' => $this->appConfig->getAppValueInt('max_call_duration'),
 					'blur-virtual-background' => $this->talkConfig->getBlurVirtualBackground($user?->getUID()),
 					'end-to-end-encryption' => $this->talkConfig->isCallEndToEndEncryptionEnabled(),
-					'live-transcription' => $this->talkConfig->getSignalingMode() === Config::SIGNALING_EXTERNAL
-						&& $this->liveTranscriptionService->isLiveTranscriptionAppEnabled(),
+					'live-transcription' => $this->isLiveTranscriptionSupported(),
+					'live-translation' => $this->isLiveTranslationSupported(),
 				],
 				'chat' => [
 					'max-length' => ChatManager::MAX_CHAT_LENGTH,
@@ -370,11 +371,59 @@ public function getCapabilities(): array {
 			$capabilities['features'][] = 'call-end-to-end-encryption';
 		}
 
+		if ($user instanceof IUser) {
+			$capabilities['config']['call']['live-transcription-target-language-id'] = $this->talkConfig->getLiveTranscriptionTargetLanguageId($user->getUID());
+		} else {
+			$capabilities['config']['call']['live-transcription-target-language-id'] = $this->talkConfig->getLiveTranscriptionTargetLanguageId();
+		}
+
 		return [
 			'spreed' => $capabilities,
 		];
 	}
 
+	protected function isLiveTranscriptionSupported(): bool {
+		return $this->talkConfig->getSignalingMode() === Config::SIGNALING_EXTERNAL
+			&& $this->liveTranscriptionService->isLiveTranscriptionAppEnabled();
+	}
+
+	protected function isLiveTranslationSupported(): bool {
+		if (!$this->isLiveTranscriptionSupported()) {
+			return false;
+		}
+
+		// FIXME Getting the capabilities from the live_transcription app causes
+		// the Nextcloud capabilities to be requested, so it enters in a loop.
+		// For now checking whether text2text tasks are supported or not is
+		// directly done here instead (but that does not guarantee that
+		// translations are supported, as an old live_transcription app might be
+		// being used).
+		// $this->getLiveTranslationSupportedFromExAppCapabilities();
+
+		$supportedTaskTypeIds = $this->taskProcessingManager->getAvailableTaskTypeIds();
+
+		return in_array(TextToTextTranslate::ID, $supportedTaskTypeIds, true);
+	}
+
+	protected function getLiveTranslationSupportedFromExAppCapabilities(): bool {
+		$cacheKey = 'is_live_translation_supported';
+
+		$isLiveTranslationSupported = $this->talkCache->get($cacheKey);
+		if (is_bool($isLiveTranslationSupported)) {
+			return $isLiveTranslationSupported;
+		}
+
+		try {
+			$isLiveTranslationSupported = $this->liveTranscriptionService->isLiveTranslationSupported();
+		} catch (\Exception $e) {
+			$isLiveTranslationSupported = false;
+		}
+
+		$this->talkCache->set($cacheKey, $isLiveTranslationSupported, 300);
+
+		return $isLiveTranslationSupported;
+	}
+
 	/**
 	 * @return list<string>
 	 */

lib/Config.php

@@ -242,6 +242,15 @@ public function getRecordingFolder(string $userId): string {
 		);
 	}
 
+	public function getLiveTranscriptionTargetLanguageId(?string $userId = null): string {
+		return $this->config->getUserValue(
+			$userId,
+			'spreed',
+			UserPreference::LIVE_TRANSCRIPTION_TARGET_LANGUAGE_ID,
+			''
+		);
+	}
+
 	public function isDisabledForUser(IUser $user): bool {
 		$allowedGroups = $this->getAllowedTalkGroupIds();
 		if (empty($allowedGroups)) {

lib/Controller/LiveTranscriptionController.php

@@ -9,6 +9,7 @@
 namespace OCA\Talk\Controller;
 
 use OCA\Talk\Exceptions\LiveTranscriptionAppNotEnabledException;
+use OCA\Talk\Exceptions\LiveTranslationNotSupportedException;
 use OCA\Talk\Middleware\Attribute\RequireCallEnabled;
 use OCA\Talk\Middleware\Attribute\RequireModeratorOrNoLobby;
 use OCA\Talk\Middleware\Attribute\RequireModeratorParticipant;
@@ -125,6 +126,38 @@ public function getAvailableLanguages(): DataResponse {
 		return new DataResponse($languages);
 	}
 
+	/**
+	 * Get available languages for live translations
+	 *
+	 * The returned array provides a list of origin languages
+	 * ("originLanguages") and a list of target languages ("targetLanguages").
+	 * Any origin language can be translated to any target language.
+	 *
+	 * The origin language list can contain "detect_language" as a special value
+	 * indicating auto-detection support.
+	 *
+	 * @return DataResponse<Http::STATUS_OK, array{originLanguages: array<string, TalkLiveTranscriptionLanguage>, targetLanguages: array<string, TalkLiveTranscriptionLanguage>, defaultTargetLanguageId: string}, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'app'|'translations'}, array{}>
+	 *
+	 * 200: Available languages got successfully
+	 * 400: The external app "live_transcription" is not available or
+	 * translations are not supported.
+	 */
+	#[PublicPage]
+	#[ApiRoute(verb: 'GET', url: '/api/{apiVersion}/live-transcription/translation-languages', requirements: [
+		'apiVersion' => '(v1)',
+	])]
+	public function getAvailableTranslationLanguages(): DataResponse {
+		try {
+			$languages = $this->liveTranscriptionService->getAvailableTranslationLanguages();
+		} catch (LiveTranscriptionAppNotEnabledException $e) {
+			return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST);
+		} catch (LiveTranslationNotSupportedException $e) {
+			return new DataResponse(['error' => 'translations'], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse($languages);
+	}
+
 	/**
 	 * Set language for live transcriptions
 	 *
@@ -150,4 +183,50 @@ public function setLanguage(string $languageId): DataResponse {
 
 		return new DataResponse(null);
 	}
+
+	/**
+	 * Set target language for live translations
+	 *
+	 * Each participant can set the language in which they want to receive the
+	 * translations.
+	 *
+	 * Setting the target language is possible only during a call and
+	 * immediately enables the translations. Translations can be disabled by
+	 * sending a null value as the language id.
+	 *
+	 * @param string $targetLanguageId the ID of the language to set
+	 * @return DataResponse<Http::STATUS_OK, null, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'app'|'translations'|'in-call'}, array{}>
+	 *
+	 * 200: Target language set successfully
+	 * 400: The external app "live_transcription" is not available or
+	 * translations are not supported.
+	 * 400: The participant is not in the call.
+	 */
+	#[PublicPage]
+	#[RequireCallEnabled]
+	#[RequireModeratorOrNoLobby]
+	#[RequireParticipant]
+	#[ApiRoute(verb: 'POST', url: '/api/{apiVersion}/live-transcription/{token}/target-language', requirements: [
+		'apiVersion' => '(v1)',
+		'token' => '[a-z0-9]{4,30}',
+	])]
+	public function setTargetLanguage(?string $targetLanguageId): DataResponse {
+		if ($this->room->getCallFlag() === Participant::FLAG_DISCONNECTED) {
+			return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST);
+		}
+
+		if (!$this->participant->getSession() || $this->participant->getSession()->getInCall() === Participant::FLAG_DISCONNECTED) {
+			return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST);
+		}
+
+		try {
+			$this->liveTranscriptionService->setTargetLanguage($this->room, $this->participant, $targetLanguageId);
+		} catch (LiveTranscriptionAppNotEnabledException $e) {
+			return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST);
+		} catch (LiveTranslationNotSupportedException $e) {
+			return new DataResponse(['error' => 'translations'], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse(null);
+	}
 }

lib/Exceptions/LiveTranslationNotSupportedException.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Exceptions;
+
+class LiveTranslationNotSupportedException extends \Exception {
+}

lib/ResponseDefinitions.php

@@ -518,6 +518,8 @@
  *             blur-virtual-background: bool,
  *             end-to-end-encryption: bool,
  *             live-transcription: bool,
+ *             live-translation: bool,
+ *             live-transcription-target-language-id: string,
  *         },
  *         chat: array{
  *             max-length: int,