diff --git a/REUSE.toml b/REUSE.toml
index ddcc5768..8974eeb0 100644
--- a/REUSE.toml
+++ b/REUSE.toml
@@ -24,7 +24,7 @@ SPDX-FileCopyrightText = "2024 Nextcloud GmbH and Nextcloud contributors"
SPDX-License-Identifier = "AGPL-3.0-or-later"
[[annotations]]
-path = ["img/app.svg", "img/app-dark.svg"]
+path = ["img/app.svg", "img/app-dark.svg", "img/client_integration/speech_to_text.svg", "img/client_integration/summarize.svg", "img/client_integration/text_to_speech.svg"]
precedence = "aggregate"
SPDX-FileCopyrightText = "2018-2024 Google LLC"
SPDX-License-Identifier = "Apache-2.0"
diff --git a/img/client_integration/speech_to_text.svg b/img/client_integration/speech_to_text.svg
new file mode 100644
index 00000000..1969dfab
--- /dev/null
+++ b/img/client_integration/speech_to_text.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/img/client_integration/summarize.svg b/img/client_integration/summarize.svg
new file mode 100644
index 00000000..5fe91d61
--- /dev/null
+++ b/img/client_integration/summarize.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/img/client_integration/text_to_speech.svg b/img/client_integration/text_to_speech.svg
new file mode 100644
index 00000000..d997e5ad
--- /dev/null
+++ b/img/client_integration/text_to_speech.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/lib/Capabilities.php b/lib/Capabilities.php
index 0273617a..3666c67d 100644
--- a/lib/Capabilities.php
+++ b/lib/Capabilities.php
@@ -14,6 +14,11 @@
use OCP\Capabilities\IPublicCapability;
use OCP\IAppConfig;
use OCP\IConfig;
+use OCP\IL10N;
+use OCP\IURLGenerator;
+use OCP\TaskProcessing\IManager;
+use OCP\TaskProcessing\TaskTypes\AudioToText;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
class Capabilities implements IPublicCapability {
@@ -21,6 +26,9 @@ public function __construct(
private IAppManager $appManager,
private IConfig $config,
private IAppConfig $appConfig,
+ private IManager $taskProcessingManager,
+ private IL10N $l,
+ private IUrlGenerator $urlGenerator,
private ?string $userId,
) {
}
@@ -30,22 +38,117 @@ public function __construct(
* assistant: array{
* version: string,
* enabled?: bool
- * }
+ * },
+ * client_integration?: array
+ * }>,
* }
*/
public function getCapabilities(): array {
+ // App version
$appVersion = $this->appManager->getAppVersion(Application::APP_ID);
- $capability = [
+ $capabilities = [
Application::APP_ID => [
'version' => $appVersion,
],
];
- if ($this->userId !== null) {
- $adminAssistantEnabled = $this->appConfig->getValueString(Application::APP_ID, 'assistant_enabled', '1') === '1';
- $userAssistantEnabled = $this->config->getUserValue($this->userId, Application::APP_ID, 'assistant_enabled', '1') === '1';
- $assistantEnabled = $adminAssistantEnabled && $userAssistantEnabled;
- $capability[Application::APP_ID]['enabled'] = $assistantEnabled;
+ if ($this->userId === null) {
+ return $capabilities;
}
- return $capability;
+
+ $adminAssistantEnabled = $this->appConfig->getValueString(Application::APP_ID, 'assistant_enabled', '1') === '1';
+ $userAssistantEnabled = $this->config->getUserValue($this->userId, Application::APP_ID, 'assistant_enabled', '1') === '1';
+ $assistantEnabled = $adminAssistantEnabled && $userAssistantEnabled;
+ $capabilities[Application::APP_ID]['enabled'] = $assistantEnabled;
+
+ // client integration UI
+ $availableTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
+ $summarizeAvailable = array_key_exists(TextToTextSummary::ID, $availableTaskTypes);
+ $sttAvailable = array_key_exists(AudioToText::ID, $availableTaskTypes);
+ $ttsAvailable = false;
+ if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech')) {
+ if (array_key_exists(\OCP\TaskProcessing\TaskTypes\TextToSpeech::ID, $availableTaskTypes)) {
+ $ttsAvailable = true;
+ }
+ }
+
+ if ($summarizeAvailable || $sttAvailable || $ttsAvailable) {
+ $capabilities['client_integration'] = [
+ Application::APP_ID => [
+ 'version' => 0.1,
+ 'context-menu' => [],
+ ],
+ ];
+
+ $textMimeTypes = [
+ 'text/',
+ 'application/msword',
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+ 'application/vnd.oasis.opendocument.text',
+ 'application/pdf',
+ ];
+ if ($summarizeAvailable) {
+ $url = $this->urlGenerator->linkToOCSRouteAbsolute(Application::APP_ID . '.assistantApi.runFileAction', [
+ 'apiVersion' => 'v1',
+ 'fileId' => '123456789',
+ 'taskTypeId' => TextToTextSummary::ID,
+ ]);
+ $url = str_replace($this->urlGenerator->getBaseUrl(), '', $url);
+ $url = str_replace('123456789', '{fileId}', $url);
+ $endpoint = [
+ 'name' => $this->l->t('Summarize'),
+ 'url' => $url,
+ 'method' => 'POST',
+ 'mimetype_filters' => implode(', ', $textMimeTypes),
+ 'icon' => $this->urlGenerator->imagePath(Application::APP_ID, 'client_integration/summarize.svg'),
+ ];
+ $capabilities['client_integration'][Application::APP_ID]['context-menu'][] = $endpoint;
+ }
+
+ if ($sttAvailable) {
+ $url = $this->urlGenerator->linkToOCSRouteAbsolute(Application::APP_ID . '.assistantApi.runFileAction', [
+ 'apiVersion' => 'v1',
+ 'fileId' => '123456789',
+ 'taskTypeId' => AudioToText::ID,
+ ]);
+ $url = str_replace($this->urlGenerator->getBaseUrl(), '', $url);
+ $url = str_replace('123456789', '{fileId}', $url);
+ $endpoint = [
+ 'name' => $this->l->t('Transcribe audio'),
+ 'url' => $url,
+ 'method' => 'POST',
+ 'mimetype_filters' => 'audio/',
+ 'icon' => $this->urlGenerator->imagePath(Application::APP_ID, 'client_integration/speech_to_text.svg'),
+ ];
+ $capabilities['client_integration'][Application::APP_ID]['context-menu'][] = $endpoint;
+ }
+
+ if ($ttsAvailable) {
+ $url = $this->urlGenerator->linkToOCSRouteAbsolute(Application::APP_ID . '.assistantApi.runFileAction', [
+ 'apiVersion' => 'v1',
+ 'fileId' => '123456789',
+ 'taskTypeId' => \OCP\TaskProcessing\TaskTypes\TextToSpeech::ID,
+ ]);
+ $url = str_replace($this->urlGenerator->getBaseUrl(), '', $url);
+ $url = str_replace('123456789', '{fileId}', $url);
+ $endpoint = [
+ 'name' => $this->l->t('Text to speech'),
+ 'url' => $url,
+ 'method' => 'POST',
+ 'mimetype_filters' => implode(', ', $textMimeTypes),
+ 'icon' => $this->urlGenerator->imagePath(Application::APP_ID, 'client_integration/text_to_speech.svg'),
+ ];
+ $capabilities['client_integration'][Application::APP_ID]['context-menu'][] = $endpoint;
+ }
+ }
+
+ return $capabilities;
}
}
diff --git a/lib/Controller/AssistantApiController.php b/lib/Controller/AssistantApiController.php
index 185797a6..d19645f4 100644
--- a/lib/Controller/AssistantApiController.php
+++ b/lib/Controller/AssistantApiController.php
@@ -28,6 +28,8 @@
use OCP\IRequest;
use OCP\Lock\LockedException;
use OCP\TaskProcessing\Task;
+use OCP\TaskProcessing\TaskTypes\AudioToText;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
use Psr\Log\LoggerInterface;
use Throwable;
@@ -418,7 +420,7 @@ public function getOutputFile(int $ocpTaskId, int $fileId): DataDownloadResponse
*
* @param int $fileId The input file ID
* @param string $taskTypeId The task type of the operation to perform
- * @return DataResponse|DataResponse
+ * @return DataResponse|DataResponse
*
* 200: The task has been scheduled successfully
* 400: There was an issue while scheduling the task
@@ -426,8 +428,18 @@ public function getOutputFile(int $ocpTaskId, int $fileId): DataDownloadResponse
#[NoAdminRequired]
public function runFileAction(int $fileId, string $taskTypeId): DataResponse {
try {
- $taskId = $this->taskProcessingService->runFileAction($this->userId, $fileId, $taskTypeId);
- return new DataResponse(['taskId' => $taskId]);
+ $this->taskProcessingService->runFileAction($this->userId, $fileId, $taskTypeId);
+ $message = $this->l10n->t('Assistant task submitted successfully');
+ if ($taskTypeId === AudioToText::ID) {
+ $message = $this->l10n->t('Transcription task submitted successfully');
+ } elseif ($taskTypeId === TextToTextSummary::ID) {
+ $message = $this->l10n->t('Summarization task submitted successfully');
+ } elseif (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech')) {
+ if ($taskTypeId === \OCP\TaskProcessing\TaskTypes\TextToSpeech::ID) {
+ $message = $this->l10n->t('Text-to-speech task submitted successfully');
+ }
+ }
+ return new DataResponse($message);
} catch (Exception|Throwable $e) {
return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
}
diff --git a/openapi.json b/openapi.json
index 2756ac7b..6289a608 100644
--- a/openapi.json
+++ b/openapi.json
@@ -212,6 +212,52 @@
"type": "boolean"
}
}
+ },
+ "client_integration": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "required": [
+ "version",
+ "context-menu"
+ ],
+ "properties": {
+ "version": {
+ "type": "number",
+ "format": "double"
+ },
+ "context-menu": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "name",
+ "url",
+ "method",
+ "mimetype_filters",
+ "icon"
+ ],
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ },
+ "method": {
+ "type": "string"
+ },
+ "mimetype_filters": {
+ "type": "string"
+ },
+ "icon": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
}
}
},
@@ -2513,16 +2559,7 @@
"$ref": "#/components/schemas/OCSMeta"
},
"data": {
- "type": "object",
- "required": [
- "taskId"
- ],
- "properties": {
- "taskId": {
- "type": "integer",
- "format": "int64"
- }
- }
+ "type": "string"
}
}
}
diff --git a/src/files/fileActions.js b/src/files/fileActions.js
index 19e4d860..209edcb8 100644
--- a/src/files/fileActions.js
+++ b/src/files/fileActions.js
@@ -61,8 +61,7 @@ function registerSummarizeAction() {
const { showError, showSuccess } = await import('@nextcloud/dialogs')
const url = generateOcsUrl('/apps/assistant/api/v1/file-action/{fileId}/core:text2text:summary', { fileId: node.fileid })
try {
- const response = await axios.post(url)
- console.debug('taskId', response.data.ocs.data.taskId)
+ await axios.post(url)
showSuccess(
t('assistant', 'Summarization task submitted successfully.') + '\n'
+ t('assistant', 'You will be notified when it is ready.') + '\n'
@@ -100,8 +99,7 @@ function registerTtsAction() {
const { showError, showSuccess } = await import('@nextcloud/dialogs')
const url = generateOcsUrl('/apps/assistant/api/v1/file-action/{fileId}/core:text2speech', { fileId: node.fileid })
try {
- const response = await axios.post(url)
- console.debug('taskId', response.data.ocs.data.taskId)
+ await axios.post(url)
showSuccess(
t('assistant', 'Text-to-speech task submitted successfully.') + '\n'
+ t('assistant', 'You will be notified when it is ready.') + '\n'
@@ -139,8 +137,7 @@ function registerSttAction() {
const { showError, showSuccess } = await import('@nextcloud/dialogs')
const url = generateOcsUrl('/apps/assistant/api/v1/file-action/{fileId}/core:audio2text', { fileId: node.fileid })
try {
- const response = await axios.post(url)
- console.debug('taskId', response.data.ocs.data.taskId)
+ await axios.post(url)
showSuccess(
t('assistant', 'Transcription task submitted successfully.') + '\n'
+ t('assistant', 'You will be notified when it is ready.') + '\n'