diff --git a/docs/capabilities.md b/docs/capabilities.md index fcb35a7efd0..dbceffe7226 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -203,3 +203,5 @@ * `federated-shared-items` - Whether shared items endpoints can be called in a federated conversation * `config => chat => style` (local) - User selected chat style (split or unified for now) * `scheduled-messages` (local) - Whether a user can schedule messages +* `config => call => live-translation` - Whether live translation is supported in calls +* `config => call => live-transcription-target-language-id` (local) - User defined string value with the id of the target language to use for live translations diff --git a/lib/Capabilities.php b/lib/Capabilities.php index b31e0a38032..50a7118141b 100644 --- a/lib/Capabilities.php +++ b/lib/Capabilities.php @@ -171,6 +171,7 @@ class Capabilities implements IPublicCapability { 'can-upload-background', 'start-without-media', 'blur-virtual-background', + 'live-transcription-target-language-id', ], 'chat' => [ 'read-privacy', @@ -256,8 +257,8 @@ public function getCapabilities(): array { 'max-duration' => $this->appConfig->getAppValueInt('max_call_duration'), 'blur-virtual-background' => $this->talkConfig->getBlurVirtualBackground($user?->getUID()), 'end-to-end-encryption' => $this->talkConfig->isCallEndToEndEncryptionEnabled(), - 'live-transcription' => $this->talkConfig->getSignalingMode() === Config::SIGNALING_EXTERNAL - && $this->liveTranscriptionService->isLiveTranscriptionAppEnabled(), + 'live-transcription' => $this->isLiveTranscriptionSupported(), + 'live-translation' => $this->isLiveTranslationSupported(), ], 'chat' => [ 'max-length' => ChatManager::MAX_CHAT_LENGTH, @@ -370,11 +371,59 @@ public function getCapabilities(): array { $capabilities['features'][] = 'call-end-to-end-encryption'; } + if ($user instanceof IUser) { + $capabilities['config']['call']['live-transcription-target-language-id'] = $this->talkConfig->getLiveTranscriptionTargetLanguageId($user->getUID()); + } else { + $capabilities['config']['call']['live-transcription-target-language-id'] = $this->talkConfig->getLiveTranscriptionTargetLanguageId(); + } + return [ 'spreed' => $capabilities, ]; } + protected function isLiveTranscriptionSupported(): bool { + return $this->talkConfig->getSignalingMode() === Config::SIGNALING_EXTERNAL + && $this->liveTranscriptionService->isLiveTranscriptionAppEnabled(); + } + + protected function isLiveTranslationSupported(): bool { + if (!$this->isLiveTranscriptionSupported()) { + return false; + } + + // FIXME Getting the capabilities from the live_transcription app causes + // the Nextcloud capabilities to be requested, so it enters in a loop. + // For now checking whether text2text tasks are supported or not is + // directly done here instead (but that does not guarantee that + // translations are supported, as an old live_transcription app might be + // being used). + // $this->getLiveTranslationSupportedFromExAppCapabilities(); + + $supportedTaskTypeIds = $this->taskProcessingManager->getAvailableTaskTypeIds(); + + return in_array(TextToTextTranslate::ID, $supportedTaskTypeIds, true); + } + + protected function getLiveTranslationSupportedFromExAppCapabilities(): bool { + $cacheKey = 'is_live_translation_supported'; + + $isLiveTranslationSupported = $this->talkCache->get($cacheKey); + if (is_bool($isLiveTranslationSupported)) { + return $isLiveTranslationSupported; + } + + try { + $isLiveTranslationSupported = $this->liveTranscriptionService->isLiveTranslationSupported(); + } catch (\Exception $e) { + $isLiveTranslationSupported = false; + } + + $this->talkCache->set($cacheKey, $isLiveTranslationSupported, 300); + + return $isLiveTranslationSupported; + } + /** * @return list */ diff --git a/lib/Config.php b/lib/Config.php index 892135141c3..35b0e870393 100644 --- a/lib/Config.php +++ b/lib/Config.php @@ -242,6 +242,15 @@ public function getRecordingFolder(string $userId): string { ); } + public function getLiveTranscriptionTargetLanguageId(?string $userId = null): string { + return $this->config->getUserValue( + $userId, + 'spreed', + UserPreference::LIVE_TRANSCRIPTION_TARGET_LANGUAGE_ID, + '' + ); + } + public function isDisabledForUser(IUser $user): bool { $allowedGroups = $this->getAllowedTalkGroupIds(); if (empty($allowedGroups)) { diff --git a/lib/Controller/LiveTranscriptionController.php b/lib/Controller/LiveTranscriptionController.php index f8e549886cc..bb5b0ac4102 100644 --- a/lib/Controller/LiveTranscriptionController.php +++ b/lib/Controller/LiveTranscriptionController.php @@ -9,6 +9,7 @@ namespace OCA\Talk\Controller; use OCA\Talk\Exceptions\LiveTranscriptionAppNotEnabledException; +use OCA\Talk\Exceptions\LiveTranslationNotSupportedException; use OCA\Talk\Middleware\Attribute\RequireCallEnabled; use OCA\Talk\Middleware\Attribute\RequireModeratorOrNoLobby; use OCA\Talk\Middleware\Attribute\RequireModeratorParticipant; @@ -125,6 +126,38 @@ public function getAvailableLanguages(): DataResponse { return new DataResponse($languages); } + /** + * Get available languages for live translations + * + * The returned array provides a list of origin languages + * ("originLanguages") and a list of target languages ("targetLanguages"). + * Any origin language can be translated to any target language. + * + * The origin language list can contain "detect_language" as a special value + * indicating auto-detection support. + * + * @return DataResponse, targetLanguages: array, defaultTargetLanguageId: string}, array{}>|DataResponse + * + * 200: Available languages got successfully + * 400: The external app "live_transcription" is not available or + * translations are not supported. + */ + #[PublicPage] + #[ApiRoute(verb: 'GET', url: '/api/{apiVersion}/live-transcription/translation-languages', requirements: [ + 'apiVersion' => '(v1)', + ])] + public function getAvailableTranslationLanguages(): DataResponse { + try { + $languages = $this->liveTranscriptionService->getAvailableTranslationLanguages(); + } catch (LiveTranscriptionAppNotEnabledException $e) { + return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST); + } catch (LiveTranslationNotSupportedException $e) { + return new DataResponse(['error' => 'translations'], Http::STATUS_BAD_REQUEST); + } + + return new DataResponse($languages); + } + /** * Set language for live transcriptions * @@ -150,4 +183,50 @@ public function setLanguage(string $languageId): DataResponse { return new DataResponse(null); } + + /** + * Set target language for live translations + * + * Each participant can set the language in which they want to receive the + * translations. + * + * Setting the target language is possible only during a call and + * immediately enables the translations. Translations can be disabled by + * sending a null value as the language id. + * + * @param string $targetLanguageId the ID of the language to set + * @return DataResponse|DataResponse + * + * 200: Target language set successfully + * 400: The external app "live_transcription" is not available or + * translations are not supported. + * 400: The participant is not in the call. + */ + #[PublicPage] + #[RequireCallEnabled] + #[RequireModeratorOrNoLobby] + #[RequireParticipant] + #[ApiRoute(verb: 'POST', url: '/api/{apiVersion}/live-transcription/{token}/target-language', requirements: [ + 'apiVersion' => '(v1)', + 'token' => '[a-z0-9]{4,30}', + ])] + public function setTargetLanguage(?string $targetLanguageId): DataResponse { + if ($this->room->getCallFlag() === Participant::FLAG_DISCONNECTED) { + return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST); + } + + if (!$this->participant->getSession() || $this->participant->getSession()->getInCall() === Participant::FLAG_DISCONNECTED) { + return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST); + } + + try { + $this->liveTranscriptionService->setTargetLanguage($this->room, $this->participant, $targetLanguageId); + } catch (LiveTranscriptionAppNotEnabledException $e) { + return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST); + } catch (LiveTranslationNotSupportedException $e) { + return new DataResponse(['error' => 'translations'], Http::STATUS_BAD_REQUEST); + } + + return new DataResponse(null); + } } diff --git a/lib/Exceptions/LiveTranslationNotSupportedException.php b/lib/Exceptions/LiveTranslationNotSupportedException.php new file mode 100644 index 00000000000..9e2b124cb8c --- /dev/null +++ b/lib/Exceptions/LiveTranslationNotSupportedException.php @@ -0,0 +1,12 @@ +getCapabilities(); + if (!isset($capabilities['features'])) { + return false; + } + + return in_array('live_translation', $capabilities['features'], true); + } + /** * @throws LiveTranscriptionAppNotEnabledException if the external app * "live_transcription" is @@ -167,6 +194,173 @@ public function setLanguage(Room $room, string $languageId): void { $this->roomService->setLiveTranscriptionLanguageId($room, $languageId); } + /** + * Returns the supported translation languages. + * + * The returned array provides a list of origin languages + * ("originLanguages") and a list of target languages ("targetLanguages"). + * Any origin language can be translated to any target language. + * + * The origin language list can contain "detect_language" as a special value + * indicating auto-detection support. + * + * @throws LiveTranscriptionAppNotEnabledException if the external app + * "live_transcription" is + * not enabled. + * @throws LiveTranscriptionAppAPIException if the request could not be sent + * to the app or the response could + * not be processed. + * @throws LiveTranscriptionAppResponseException if the request itself + * succeeded but the app + * responded with an error. + * @throws LiveTranslationNotSupportedException if live translations are not + * supported. + */ + public function getAvailableTranslationLanguages(): array { + // Target languages can be got from capabilities or directly for a + // specific room, but the list should be the same in both cases. + $capabilities = $this->getCapabilities(); + + if (!isset($capabilities['live_translation']) + || !isset($capabilities['live_translation']['supported_translation_languages'])) { + throw new LiveTranslationNotSupportedException(); + } + + $translationLanguages = $capabilities['live_translation']['supported_translation_languages']; + + if (!is_array($translationLanguages['origin_languages'])) { + $this->logger->error('Request to live_transcription (ExApp) failed: list of translation origin languages not found'); + + throw new LiveTranscriptionAppAPIException('response-no-origin-language-list'); + } + + if (!is_array($translationLanguages['target_languages'])) { + $this->logger->error('Request to live_transcription (ExApp) failed: list of translation target languages not found'); + + throw new LiveTranscriptionAppAPIException('response-no-target-language-list'); + } + + if (count($translationLanguages['target_languages']) === 0) { + $this->logger->error('Request to live_transcription (ExApp) failed: empty list of translation target languages'); + + throw new LiveTranscriptionAppAPIException('response-empty-language-list'); + } + + $translationLanguages['originLanguages'] = $translationLanguages['origin_languages']; + $translationLanguages['targetLanguages'] = $translationLanguages['target_languages']; + unset($translationLanguages['origin_languages']); + unset($translationLanguages['target_languages']); + + $translationLanguages['defaultTargetLanguageId'] = $this->getDefaultTargetLanguageId($translationLanguages['targetLanguages']); + + return $translationLanguages; + } + + private function getDefaultTargetLanguageId(array $targetLanguages): string { + if (count($targetLanguages) === 0) { + return ''; + } + + $defaultTargetLanguageId = $this->l10nFactory->findLanguage(Application::APP_ID); + + if (array_key_exists($defaultTargetLanguageId, $targetLanguages)) { + return $defaultTargetLanguageId; + } + + if (strpos($defaultTargetLanguageId, '_') !== false) { + $defaultTargetLanguageId = substr($defaultTargetLanguageId, 0, strpos($defaultTargetLanguageId, '_')); + + if (array_key_exists($defaultTargetLanguageId, $targetLanguages)) { + return $defaultTargetLanguageId; + } + } + + $defaultTargetLanguageId = 'en'; + if (array_key_exists($defaultTargetLanguageId, $targetLanguages)) { + return $defaultTargetLanguageId; + } + + return array_key_first($targetLanguages); + } + + /** + * @throws LiveTranscriptionAppNotEnabledException if the external app + * "live_transcription" is + * not enabled. + * @throws LiveTranscriptionAppAPIException if the request could not be sent + * to the app or the response could + * not be processed. + * @throws LiveTranscriptionAppResponseException if the request itself + * succeeded but the app + * responded with an error. + */ + public function setTargetLanguage(Room $room, Participant $participant, ?string $targetLanguageId): void { + if ($targetLanguageId === '') { + throw new \InvalidArgumentException('Empty target language id'); + } + + $parameters = [ + 'roomToken' => $room->getToken(), + 'ncSessionId' => $participant->getSession()->getSessionId(), + 'langId' => $targetLanguageId, + ]; + + try { + $this->requestToExAppLiveTranscription('POST', '/api/v1/translation/set-target-language', $parameters); + } catch (LiveTranscriptionAppResponseException $e) { + if ($e->getResponse()->getStatusCode() === 550) { + throw new LiveTranslationNotSupportedException(); + } + + throw $e; + } + } + + /** + * Returns the capabilities for the live_transcription app. + * + * If the installed live_transcription app version does not support yet + * capabilities an empty array will be returned. On the other hand, if the + * app is expected to provide capabilities but they are not returned + * LiveTranscriptionAppApiException is thrown instead. + * + * @return array an array with the capabilities. + * @throws LiveTranscriptionAppNotEnabledException if the external app + * "live_transcription" is + * not enabled. + * @throws LiveTranscriptionAppAPIException if the request could not be sent + * to the app or the response could + * not be processed. + * @throws LiveTranscriptionAppResponseException if the request itself + * succeeded but the app + * responded with an error. + */ + private function getCapabilities(): array { + try { + $capabilities = $this->requestToExAppLiveTranscription('GET', '/capabilities'); + } catch (LiveTranscriptionAppResponseException $e) { + if ($e->getResponse()->getStatusCode() !== 404) { + throw $e; + } + + return []; + } + + if (!is_array($capabilities)) { + $this->logger->error('Request to live_transcription (ExApp) failed: capabilities is not an array'); + + throw new LiveTranscriptionAppAPIException('response-capabilities-not-array'); + } + + if (!isset($capabilities['live_transcription'])) { + $this->logger->error('Request to live_transcription (ExApp) failed: wrong capabilities structure'); + + throw new LiveTranscriptionAppAPIException('response-capabilities-wrong-structure'); + } + + return $capabilities['live_transcription']; + } + /** * @throws LiveTranscriptionAppNotEnabledException if the external app * "live_transcription" is diff --git a/lib/Settings/BeforePreferenceSetEventListener.php b/lib/Settings/BeforePreferenceSetEventListener.php index b25536be9b4..bacd965650e 100644 --- a/lib/Settings/BeforePreferenceSetEventListener.php +++ b/lib/Settings/BeforePreferenceSetEventListener.php @@ -85,6 +85,13 @@ public function validatePreference(string $userId, string $key, string|int|null return $value === UserPreference::CHAT_STYLE_SPLIT || $value === UserPreference::CHAT_STYLE_UNIFIED; } + if ($key === UserPreference::LIVE_TRANSCRIPTION_TARGET_LANGUAGE_ID) { + // Accept any value, as it will be used for both local and federated + // instances and therefore the valid values might change depending + // on the instance. + return true; + } + return false; } diff --git a/lib/Settings/UserPreference.php b/lib/Settings/UserPreference.php index ab88c466919..8c7df0ab972 100644 --- a/lib/Settings/UserPreference.php +++ b/lib/Settings/UserPreference.php @@ -24,4 +24,6 @@ class UserPreference { public const CHAT_STYLE_SPLIT = 'split'; public const CHAT_STYLE_UNIFIED = 'unified'; + + public const LIVE_TRANSCRIPTION_TARGET_LANGUAGE_ID = 'live_transcription_target_language_id'; } diff --git a/openapi-administration.json b/openapi-administration.json index 8741cabd548..feba87aabd6 100644 --- a/openapi-administration.json +++ b/openapi-administration.json @@ -156,7 +156,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -219,6 +221,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, diff --git a/openapi-backend-recording.json b/openapi-backend-recording.json index 516d85f7d4a..f6231061de2 100644 --- a/openapi-backend-recording.json +++ b/openapi-backend-recording.json @@ -89,7 +89,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -152,6 +154,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, diff --git a/openapi-backend-signaling.json b/openapi-backend-signaling.json index 94aa669b4f6..8028c7ff716 100644 --- a/openapi-backend-signaling.json +++ b/openapi-backend-signaling.json @@ -89,7 +89,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -152,6 +154,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, diff --git a/openapi-backend-sipbridge.json b/openapi-backend-sipbridge.json index 297ef688060..061d0ddd79e 100644 --- a/openapi-backend-sipbridge.json +++ b/openapi-backend-sipbridge.json @@ -132,7 +132,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -195,6 +197,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, diff --git a/openapi-bots.json b/openapi-bots.json index 74ff534c867..f6a2c1d1560 100644 --- a/openapi-bots.json +++ b/openapi-bots.json @@ -89,7 +89,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -152,6 +154,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, diff --git a/openapi-federation.json b/openapi-federation.json index c9a349a122f..de679150911 100644 --- a/openapi-federation.json +++ b/openapi-federation.json @@ -132,7 +132,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -195,6 +197,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, diff --git a/openapi-full.json b/openapi-full.json index bfc64b6d231..73139c722f9 100644 --- a/openapi-full.json +++ b/openapi-full.json @@ -290,7 +290,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -353,6 +355,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, @@ -12399,6 +12407,145 @@ } } }, + "/ocs/v2.php/apps/spreed/api/{apiVersion}/live-transcription/translation-languages": { + "get": { + "operationId": "live_transcription-get-available-translation-languages", + "summary": "Get available languages for live translations", + "description": "The returned array provides a list of origin languages (\"originLanguages\") and a list of target languages (\"targetLanguages\"). Any origin language can be translated to any target language.\nThe origin language list can contain \"detect_language\" as a special value indicating auto-detection support.\ntranslations are not supported.", + "tags": [ + "live_transcription" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Available languages got successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "originLanguages", + "targetLanguages", + "defaultTargetLanguageId" + ], + "properties": { + "originLanguages": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/LiveTranscriptionLanguage" + } + }, + "targetLanguages": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/LiveTranscriptionLanguage" + } + }, + "defaultTargetLanguageId": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "400": { + "description": "The external app \"live_transcription\" is not available or", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string", + "enum": [ + "app", + "translations" + ] + } + } + } + } + } + } + } + } + } + } + } + } + }, "/ocs/v2.php/apps/spreed/api/{apiVersion}/live-transcription/{token}/language": { "post": { "operationId": "live_transcription-set-language", @@ -12583,6 +12730,152 @@ } } }, + "/ocs/v2.php/apps/spreed/api/{apiVersion}/live-transcription/{token}/target-language": { + "post": { + "operationId": "live_transcription-set-target-language", + "summary": "Set target language for live translations", + "description": "Each participant can set the language in which they want to receive the translations.\nSetting the target language is possible only during a call and immediately enables the translations. Translations can be disabled by sending a null value as the language id.\ntranslations are not supported.", + "tags": [ + "live_transcription" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "targetLanguageId" + ], + "properties": { + "targetLanguageId": { + "type": "string", + "description": "the ID of the language to set" + } + } + } + } + } + }, + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "token", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^[a-z0-9]{4,30}$" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Target language set successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "nullable": true + } + } + } + } + } + } + } + }, + "400": { + "description": "The participant is not in the call.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string", + "enum": [ + "app", + "translations", + "in-call" + ] + } + } + } + } + } + } + } + } + } + } + } + } + }, "/ocs/v2.php/apps/spreed/api/{apiVersion}/bridge/{token}": { "get": { "operationId": "matterbridge-get-bridge-of-room", diff --git a/openapi.json b/openapi.json index 28ddb4538bb..f031f2e5efd 100644 --- a/openapi.json +++ b/openapi.json @@ -249,7 +249,9 @@ "max-duration", "blur-virtual-background", "end-to-end-encryption", - "live-transcription" + "live-transcription", + "live-translation", + "live-transcription-target-language-id" ], "properties": { "enabled": { @@ -312,6 +314,12 @@ }, "live-transcription": { "type": "boolean" + }, + "live-translation": { + "type": "boolean" + }, + "live-transcription-target-language-id": { + "type": "string" } } }, @@ -12304,6 +12312,145 @@ } } }, + "/ocs/v2.php/apps/spreed/api/{apiVersion}/live-transcription/translation-languages": { + "get": { + "operationId": "live_transcription-get-available-translation-languages", + "summary": "Get available languages for live translations", + "description": "The returned array provides a list of origin languages (\"originLanguages\") and a list of target languages (\"targetLanguages\"). Any origin language can be translated to any target language.\nThe origin language list can contain \"detect_language\" as a special value indicating auto-detection support.\ntranslations are not supported.", + "tags": [ + "live_transcription" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Available languages got successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "originLanguages", + "targetLanguages", + "defaultTargetLanguageId" + ], + "properties": { + "originLanguages": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/LiveTranscriptionLanguage" + } + }, + "targetLanguages": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/LiveTranscriptionLanguage" + } + }, + "defaultTargetLanguageId": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "400": { + "description": "The external app \"live_transcription\" is not available or", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string", + "enum": [ + "app", + "translations" + ] + } + } + } + } + } + } + } + } + } + } + } + } + }, "/ocs/v2.php/apps/spreed/api/{apiVersion}/live-transcription/{token}/language": { "post": { "operationId": "live_transcription-set-language", @@ -12488,6 +12635,152 @@ } } }, + "/ocs/v2.php/apps/spreed/api/{apiVersion}/live-transcription/{token}/target-language": { + "post": { + "operationId": "live_transcription-set-target-language", + "summary": "Set target language for live translations", + "description": "Each participant can set the language in which they want to receive the translations.\nSetting the target language is possible only during a call and immediately enables the translations. Translations can be disabled by sending a null value as the language id.\ntranslations are not supported.", + "tags": [ + "live_transcription" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "targetLanguageId" + ], + "properties": { + "targetLanguageId": { + "type": "string", + "description": "the ID of the language to set" + } + } + } + } + } + }, + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "token", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^[a-z0-9]{4,30}$" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Target language set successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "nullable": true + } + } + } + } + } + } + } + }, + "400": { + "description": "The participant is not in the call.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string", + "enum": [ + "app", + "translations", + "in-call" + ] + } + } + } + } + } + } + } + } + } + } + } + } + }, "/ocs/v2.php/apps/spreed/api/{apiVersion}/bridge/{token}": { "get": { "operationId": "matterbridge-get-bridge-of-room", diff --git a/src/__mocks__/capabilities.ts b/src/__mocks__/capabilities.ts index 326ea4cc6cf..b23b37e5172 100644 --- a/src/__mocks__/capabilities.ts +++ b/src/__mocks__/capabilities.ts @@ -151,6 +151,8 @@ export const mockedCapabilities: Capabilities = { 'blur-virtual-background': false, 'end-to-end-encryption': false, 'live-transcription': false, + 'live-translation': false, + 'live-transcription-target-language-id': '', }, chat: { 'max-length': 32000, diff --git a/src/components/CallView/BottomBar.vue b/src/components/CallView/BottomBar.vue index ae2be40ddf2..b43a93146f0 100644 --- a/src/components/CallView/BottomBar.vue +++ b/src/components/CallView/BottomBar.vue @@ -16,8 +16,10 @@ import { computed, onMounted, onUnmounted, ref, toValue, useTemplateRef, watch } import { useStore } from 'vuex' import NcActionButton from '@nextcloud/vue/components/NcActionButton' import NcActions from '@nextcloud/vue/components/NcActions' +import NcActionSeparator from '@nextcloud/vue/components/NcActionSeparator' import NcButton from '@nextcloud/vue/components/NcButton' import NcLoadingIcon from '@nextcloud/vue/components/NcLoadingIcon' +import IconChevronUp from 'vue-material-design-icons/ChevronUp.vue' import IconFullscreen from 'vue-material-design-icons/Fullscreen.vue' import IconFullscreenExit from 'vue-material-design-icons/FullscreenExit.vue' import IconHandBackLeft from 'vue-material-design-icons/HandBackLeft.vue' // Filled for better indication @@ -40,6 +42,7 @@ import { useActorStore } from '../../stores/actor.ts' import { useBreakoutRoomsStore } from '../../stores/breakoutRooms.ts' import { useCallViewStore } from '../../stores/callView.ts' import { useLiveTranscriptionStore } from '../../stores/liveTranscription.ts' +import { useSettingsStore } from '../../stores/settings.ts' import { localCallParticipantModel, localMediaModel } from '../../utils/webrtc/index.js' const { isSidebar = false } = defineProps<{ @@ -55,6 +58,7 @@ const breakoutRoomsStore = useBreakoutRoomsStore() const isFullscreen = !isSidebar && useDocumentFullscreen() const callViewStore = useCallViewStore() const liveTranscriptionStore = useLiveTranscriptionStore() +const settingsStore = useSettingsStore() const isLiveTranscriptionLoading = ref(false) const bottomBar = useTemplateRef('bottomBar') @@ -73,15 +77,72 @@ const canModerate = computed(() => [PARTICIPANT.TYPE.OWNER, PARTICIPANT.TYPE.MOD .includes(conversation.value.participantType)) const isLiveTranscriptionSupported = computed(() => getTalkConfig(token.value, 'call', 'live-transcription') || false) +const isLiveTranslationSupported = computed(() => getTalkConfig(token.value, 'call', 'live-translation') || false) const liveTranscriptionButtonLabel = computed(() => { - if (!callViewStore.isLiveTranscriptionEnabled) { - return t('spreed', 'Enable live transcription') + if (callViewStore.isLiveTranscriptionEnabled && languageType.value === LanguageType.Original) { + return t('spreed', 'Disable live transcription') + } + + return t('spreed', 'Enable live transcription') +}) + +const liveTranslationButtonLabel = computed(() => { + if (callViewStore.isLiveTranscriptionEnabled && languageType.value === LanguageType.Target) { + return t('spreed', 'Disable live translation') } - return t('spreed', 'Disable live transcription') + return t('spreed', 'Enable live translation') +}) + +const originalLanguageButtonLabel = computed(() => { + const languageId = conversation.value.liveTranscriptionLanguageId || 'en' + + const languageName = liveTranscriptionStore.getLiveTranscriptionLanguages()?.[languageId]?.name ?? languageId + + return t('spreed', 'Original language: {languageName}', { + languageName, + }) }) +const targetLanguageButtonLabel = computed(() => { + const languageId = targetLanguageId.value + if (!languageId) { + return t('spreed', 'Translated language') + } + + const languageName = liveTranscriptionStore.getLiveTranscriptionTargetLanguages()?.[languageId]?.name ?? languageId + + return t('spreed', 'Translated language: {languageName}', { + languageName, + }) +}) + +const targetLanguageId = computed(() => { + const languageId = settingsStore.liveTranscriptionTargetLanguageId + + if (languageId) { + return languageId + } + + return liveTranscriptionStore.getLiveTranscriptionDefaultTargetLanguageId() +}) + +const targetLanguageAvailable = computed(() => { + const liveTranscriptionTargetLanguages = liveTranscriptionStore.getLiveTranscriptionTargetLanguages() + + return targetLanguageId.value + && targetLanguageId.value !== conversation.value.liveTranscriptionLanguageId + && liveTranscriptionTargetLanguages && liveTranscriptionTargetLanguages[targetLanguageId.value] +}) + +const LanguageType = { + Original: 'original', + Target: 'target', +} as const + +const languageType = ref(LanguageType.Original) + const isHandRaised = computed(() => localMediaModel.attributes.raisedHand.state === true) const raiseHandButtonLabel = computed(() => { @@ -164,10 +225,21 @@ onUnmounted(() => { debounceAdjustLayout.clear?.() }) +/** + * Load live transcription and translation languages. + */ +function handleLiveTranscriptionLanguageSelectorOpen() { + liveTranscriptionStore.loadLiveTranscriptionLanguages() + liveTranscriptionStore.loadLiveTranscriptionTranslationLanguages() +} + /** * Toggle live transcriptions. + * + * Live translations are enabled again if needed when live transcriptions are + * toggled on. */ -async function toggleLiveTranscription() { +async function toggleLiveTranscriptionAndTranslation() { if (isLiveTranscriptionLoading.value) { return } @@ -176,10 +248,98 @@ async function toggleLiveTranscription() { if (!callViewStore.isLiveTranscriptionEnabled) { await enableLiveTranscription() + + if (languageType.value === LanguageType.Target) { + await enableLiveTranslation() + } + } else { + await disableLiveTranscription() + } + + isLiveTranscriptionLoading.value = false +} + +/** + * Toggle live transcriptions. + */ +async function toggleLiveTranscription() { + if (isLiveTranscriptionLoading.value) { + return + } + + if (callViewStore.isLiveTranscriptionEnabled && languageType.value === LanguageType.Original) { + isLiveTranscriptionLoading.value = true + + await disableLiveTranscription() + + isLiveTranscriptionLoading.value = false } else { + await switchToOriginalLanguage() + } +} + +/** + * Toggle live translations. + * + * Disabling live translations disables live transcriptions as well. + */ +async function toggleLiveTranslation() { + if (isLiveTranscriptionLoading.value) { + return + } + + if (callViewStore.isLiveTranscriptionEnabled && languageType.value === LanguageType.Target) { + isLiveTranscriptionLoading.value = true + await disableLiveTranscription() + + isLiveTranscriptionLoading.value = false + } else { + await switchToTargetLanguage() + } +} + +/** + * Enable live transcriptions, disabling live translations if they were already + * enabled. + */ +async function switchToOriginalLanguage() { + if (isLiveTranscriptionLoading.value) { + return + } + + isLiveTranscriptionLoading.value = true + + if (!callViewStore.isLiveTranscriptionEnabled && !(await enableLiveTranscription())) { + isLiveTranscriptionLoading.value = false + + return + } + + await disableLiveTranslation() + + isLiveTranscriptionLoading.value = false +} + +/** + * Enable live translations, enabling live transcriptions first if they were not + * enabled yet. + */ +async function switchToTargetLanguage() { + if (isLiveTranscriptionLoading.value) { + return + } + + isLiveTranscriptionLoading.value = true + + if (!callViewStore.isLiveTranscriptionEnabled && !(await enableLiveTranscription())) { + isLiveTranscriptionLoading.value = false + + return } + await enableLiveTranslation() + isLiveTranscriptionLoading.value = false } @@ -196,14 +356,18 @@ async function enableLiveTranscription() { } catch (exception) { showError(t('spreed', 'Error when trying to load the available live transcription languages')) - return + return false } try { await callViewStore.enableLiveTranscription(token.value) } catch (error) { showError(t('spreed', 'Failed to enable live transcription')) + + return false } + + return true } /** @@ -219,6 +383,50 @@ async function disableLiveTranscription() { } } +/** + * Enable live translations. + * + * Live transcriptions need to have been enabled first before enabling live + * translations. + */ +async function enableLiveTranslation() { + if (languageType.value === LanguageType.Target) { + return + } + + try { + await callViewStore.setLiveTranscriptionTargetLanguage(token.value, targetLanguageId.value as string) + } catch (error) { + showError(t('spreed', 'Failed to enable live translations')) + + return + } + + languageType.value = LanguageType.Target +} + +/** + * Disable live translations. + * + * This does not disable live transcriptions, so the transcription will continue + * in the original language. + */ +async function disableLiveTranslation() { + if (languageType.value === LanguageType.Original) { + return + } + + try { + await callViewStore.setLiveTranscriptionTargetLanguage(token.value, null) + } catch (error) { + showError(t('spreed', 'Failed to disable live translations')) + + return + } + + languageType.value = LanguageType.Original +} + let lowerHandDelay = AUTO_LOWER_HAND_THRESHOLD let speakingTimestamp: number | null = null let lowerHandTimeout: ReturnType | null = null @@ -338,25 +546,63 @@ useHotKey('r', toggleHandRaised) :supported-reactions="supportedReactions" :local-call-participant-model="localCallParticipantModel" /> - - - + class="live-transcription-button-wrapper"> + + + + + + + + {{ originalLanguageButtonLabel }} + + + + {{ targetLanguageButtonLabel }} + + +