Add support for live transcriptions in calls

appinfo/info.xml

@@ -18,7 +18,7 @@
 * 🌉 **Sync with other chat solutions** With [Matterbridge](https://github.com/42wim/matterbridge/) being integrated in Talk, you can easily sync a lot of other chat solutions to Nextcloud Talk and vice-versa.
 ]]></description>
 
-	<version>22.0.0-beta.1</version>
+	<version>22.0.0-beta.1.1</version>
 	<licence>agpl</licence>
 
 	<author>Anna Larch</author>

docs/capabilities.md

@@ -196,3 +196,4 @@
 
 ## 22
 * `threads` - Whether the chat supports threads
+* `config => call => live-transcription` - Whether live transcription is supported in calls

lib/Capabilities.php

@@ -9,6 +9,7 @@
 namespace OCA\Talk;
 
 use OCA\Talk\Chat\ChatManager;
+use OCA\Talk\Service\LiveTranscriptionService;
 use OCP\App\IAppManager;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\Capabilities\IPublicCapability;
@@ -208,6 +209,7 @@ public function __construct(
 		protected IAppManager $appManager,
 		protected ITranslationManager $translationManager,
 		protected ITaskProcessingManager $taskProcessingManager,
+		protected LiveTranscriptionService $liveTranscriptionService,
 		ICacheFactory $cacheFactory,
 	) {
 		$this->talkCache = $cacheFactory->createLocal('talk::');
@@ -249,6 +251,8 @@ public function getCapabilities(): array {
 					'max-duration' => $this->appConfig->getAppValueInt('max_call_duration'),
 					'blur-virtual-background' => $this->talkConfig->getBlurVirtualBackground($user?->getUID()),
 					'end-to-end-encryption' => $this->talkConfig->isCallEndToEndEncryptionEnabled(),
+					'live-transcription' => $this->talkConfig->getSignalingMode() === Config::SIGNALING_EXTERNAL
+						&& $this->liveTranscriptionService->isLiveTranscriptionAppEnabled(),
 				],
 				'chat' => [
 					'max-length' => ChatManager::MAX_CHAT_LENGTH,

lib/Controller/LiveTranscriptionController.php

@@ -0,0 +1,153 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Controller;
+
+use OCA\Talk\Exceptions\LiveTranscriptionAppNotEnabledException;
+use OCA\Talk\Middleware\Attribute\RequireCallEnabled;
+use OCA\Talk\Middleware\Attribute\RequireModeratorOrNoLobby;
+use OCA\Talk\Middleware\Attribute\RequireModeratorParticipant;
+use OCA\Talk\Middleware\Attribute\RequireParticipant;
+use OCA\Talk\Participant;
+use OCA\Talk\ResponseDefinitions;
+use OCA\Talk\Service\LiveTranscriptionService;
+use OCP\AppFramework\Http;
+use OCP\AppFramework\Http\Attribute\ApiRoute;
+use OCP\AppFramework\Http\Attribute\PublicPage;
+use OCP\AppFramework\Http\DataResponse;
+use OCP\IRequest;
+
+/**
+ * @psalm-import-type TalkLiveTranscriptionLanguage from ResponseDefinitions
+ */
+class LiveTranscriptionController extends AEnvironmentAwareOCSController {
+	public function __construct(
+		string $appName,
+		IRequest $request,
+		private LiveTranscriptionService $liveTranscriptionService,
+	) {
+		parent::__construct($appName, $request);
+	}
+
+	/**
+	 * Enable the live transcription
+	 *
+	 * @return DataResponse<Http::STATUS_OK, null, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'app'|'in-call'}, array{}>
+	 *
+	 * 200: Live transcription enabled successfully
+	 * 400: The external app "live_transcription" is not available
+	 * 400: The participant is not in the call
+	 */
+	#[PublicPage]
+	#[RequireCallEnabled]
+	#[RequireModeratorOrNoLobby]
+	#[RequireParticipant]
+	#[ApiRoute(verb: 'POST', url: '/api/{apiVersion}/live-transcription/{token}', requirements: [
+		'apiVersion' => '(v1)',
+		'token' => '[a-z0-9]{4,30}',
+	])]
+	public function enable(): DataResponse {
+		if ($this->room->getCallFlag() === Participant::FLAG_DISCONNECTED) {
+			return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST);
+		}
+
+		if ($this->participant->getSession() && $this->participant->getSession()->getInCall() === Participant::FLAG_DISCONNECTED) {
+			return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST);
+		}
+
+		try {
+			$this->liveTranscriptionService->enable($this->room, $this->participant);
+		} catch (LiveTranscriptionAppNotEnabledException $e) {
+			return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse(null);
+	}
+
+	/**
+	 * Disable the live transcription
+	 *
+	 * @return DataResponse<Http::STATUS_OK, null, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'app'|'in-call'}, array{}>
+	 *
+	 * 200: Live transcription stopped successfully
+	 * 400: The external app "live_transcription" is not available
+	 * 400: The participant is not in the call
+	 */
+	#[PublicPage]
+	#[RequireModeratorOrNoLobby]
+	#[RequireParticipant]
+	#[ApiRoute(verb: 'DELETE', url: '/api/{apiVersion}/live-transcription/{token}', requirements: [
+		'apiVersion' => '(v1)',
+		'token' => '[a-z0-9]{4,30}',
+	])]
+	public function disable(): DataResponse {
+		if ($this->room->getCallFlag() === Participant::FLAG_DISCONNECTED) {
+			return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST);
+		}
+
+		if ($this->participant->getSession() && $this->participant->getSession()->getInCall() === Participant::FLAG_DISCONNECTED) {
+			return new DataResponse(['error' => 'in-call'], Http::STATUS_BAD_REQUEST);
+		}
+
+		try {
+			$this->liveTranscriptionService->disable($this->room, $this->participant);
+		} catch (LiveTranscriptionAppNotEnabledException $e) {
+			return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse(null);
+	}
+
+	/**
+	 * Get available languages for live transcriptions
+	 *
+	 * @return DataResponse<Http::STATUS_OK, array<string, TalkLiveTranscriptionLanguage>, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'app'}, array{}>
+	 *
+	 * 200: Available languages got successfully
+	 * 400: The external app "live_transcription" is not available
+	 */
+	#[PublicPage]
+	#[ApiRoute(verb: 'GET', url: '/api/{apiVersion}/live-transcription/languages', requirements: [
+		'apiVersion' => '(v1)',
+	])]
+	public function getAvailableLanguages(): DataResponse {
+		try {
+			$languages = $this->liveTranscriptionService->getAvailableLanguages();
+		} catch (LiveTranscriptionAppNotEnabledException $e) {
+			return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse($languages);
+	}
+
+	/**
+	 * Set language for live transcriptions
+	 *
+	 * @param string $languageId the ID of the language to set
+	 * @return DataResponse<Http::STATUS_OK, null, array{}>|DataResponse<Http::STATUS_BAD_REQUEST|Http::STATUS_FORBIDDEN, array{error: 'app'}, array{}>
+	 *
+	 * 200: Language set successfully
+	 * 400: The external app "live_transcription" is not available
+	 * 403: Participant is not a moderator
+	 */
+	#[PublicPage]
+	#[RequireModeratorParticipant]
+	#[ApiRoute(verb: 'POST', url: '/api/{apiVersion}/live-transcription/{token}/language', requirements: [
+		'apiVersion' => '(v1)',
+		'token' => '[a-z0-9]{4,30}',
+	])]
+	public function setLanguage(string $languageId): DataResponse {
+		try {
+			$this->liveTranscriptionService->setLanguage($this->room, $languageId);
+		} catch (LiveTranscriptionAppNotEnabledException $e) {
+			return new DataResponse(['error' => 'app'], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse(null);
+	}
+}

lib/Events/ARoomModifiedEvent.php

@@ -24,6 +24,7 @@ abstract class ARoomModifiedEvent extends ARoomEvent {
 	public const PROPERTY_IN_CALL = 'inCall';
 	public const PROPERTY_LISTABLE = 'listable';
 	public const PROPERTY_LOBBY = 'lobby';
+	public const PROPERTY_LIVE_TRANSCRIPTION_LANGUAGE_ID = 'liveTranscriptionLanguageId';
 	public const PROPERTY_MESSAGE_EXPIRATION = 'messageExpiration';
 	public const PROPERTY_MENTION_PERMISSIONS = 'mentionPermissions';
 	public const PROPERTY_NAME = 'name';

lib/Exceptions/LiveTranscriptionAppAPIException.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Exceptions;
+
+class LiveTranscriptionAppAPIException extends \Exception {
+}

lib/Exceptions/LiveTranscriptionAppNotEnabledException.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Exceptions;
+
+class LiveTranscriptionAppNotEnabledException extends \Exception {
+}

lib/Exceptions/LiveTranscriptionAppResponseException.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Exceptions;
+
+use OCP\Http\Client\IResponse;
+
+class LiveTranscriptionAppResponseException extends \Exception {
+
+	public function __construct(
+		string $message = '',
+		int $code = 0,
+		?\Throwable $previous = null,
+		protected IResponse $response,
+	) {
+		parent::__construct($message, $code, $previous);
+	}
+
+	public function getResponse(): IResponse {
+		return $this->response;
+	}
+}

lib/Manager.php

@@ -122,6 +122,7 @@ public function createRoomObjectFromData(array $data): Room {
 			'recording_consent' => 0,
 			'has_federation' => 0,
 			'mention_permissions' => 0,
+			'transcription_language' => '',
 		], $data));
 	}
 
@@ -191,6 +192,7 @@ public function createRoomObject(array $row): Room {
 			(int)$row['recording_consent'],
 			(int)$row['has_federation'],
 			(int)$row['mention_permissions'],
+			(string)$row['transcription_language'],
 		);
 	}
 

lib/Migration/Version22000Date20250813122342.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version22000Date20250813122342 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	#[\Override]
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		$table = $schema->getTable('talk_rooms');
+		if (!$table->hasColumn('transcription_language')) {
+			$table->addColumn('transcription_language', Types::STRING, [
+				'notnull' => false,
+				'default' => '',
+				'length' => 16,
+			]);
+		}
+
+		return $schema;
+	}
+}

lib/Model/SelectHelper.php

@@ -45,6 +45,7 @@ public function selectRoomsTable(IQueryBuilder $query, string $alias = 'r'): voi
 			->addSelect($alias . 'recording_consent')
 			->addSelect($alias . 'has_federation')
 			->addSelect($alias . 'mention_permissions')
+			->addSelect($alias . 'transcription_language')
 			->selectAlias($alias . 'id', 'r_id');
 	}
 

lib/ResponseDefinitions.php

@@ -308,6 +308,8 @@
  *     lastReadMessage: int,
  *     // Listable scope for the room (only available with `listable-rooms` capability)
  *     listable: int,
+ *     // ID of the language to use for live transcriptions in the room,
+ *     liveTranscriptionLanguageId: string,
  *     // Webinar lobby restriction (0-1), if the participant is a moderator they can always join the conversation (only available with `webinary-lobby` capability) (See [Webinar lobby states](https://nextcloud-talk.readthedocs.io/en/latest/constants#webinar-lobby-states))
  *     lobbyState: int,
  *     // Timestamp when the lobby will be automatically disabled (only available with `webinary-lobby` capability)
@@ -493,6 +495,7 @@
  *             max-duration: int,
  *             blur-virtual-background: bool,
  *             end-to-end-encryption: bool,
+ *             live-transcription: bool,
  *         },
  *         chat: array{
  *             max-length: int,
@@ -531,6 +534,14 @@
  *     config-local: array<string, non-empty-list<string>>,
  *     version: string,
  * }
+ *
+ * @psalm-type TalkLiveTranscriptionLanguage = array{
+ *     name: string,
+ *     metadata: array{
+ *         separator: string,
+ *         rtl: bool,
+ *     },
+ * }
  */
 class ResponseDefinitions {
 }

lib/Room.php

@@ -142,6 +142,7 @@ public function __construct(
 		private int $recordingConsent,
 		private int $hasFederation,
 		private int $mentionPermissions,
+		private string $liveTranscriptionLanguageId,
 	) {
 	}
 
@@ -608,4 +609,12 @@ public function setObjectId(string $objectId): void {
 	public function setObjectType(string $objectType): void {
 		$this->objectType = $objectType;
 	}
+
+	public function getLiveTranscriptionLanguageId(): string {
+		return $this->liveTranscriptionLanguageId;
+	}
+
+	public function setLiveTranscriptionLanguageId(string $liveTranscriptionLanguageId): void {
+		$this->liveTranscriptionLanguageId = $liveTranscriptionLanguageId;
+	}
 }