Skip to content

Commit 83024f3

Browse files
edward-lyedward-ly
committed
feat: add core:audio2text:subtitles task type, register file action
Signed-off-by: Edward Ly <contact@edward.ly>
1 parent 910a8e9 commit 83024f3

7 files changed

Lines changed: 96 additions & 9 deletions

File tree

‎lib/Capabilities.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,24 @@ public function getCapabilities(): array {
128128
'icon' => $this->urlGenerator->imagePath(Application::APP_ID, 'client_integration/speech_to_text.svg'),
129129
];
130130
$capabilities['client_integration'][Application::APP_ID]['context-menu'][] = $endpoint;
131+
132+
if (class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles')) {
133+
$url = $this->urlGenerator->linkToOCSRouteAbsolute(Application::APP_ID . '.assistantApi.runFileAction', [
134+
'apiVersion' => 'v1',
135+
'fileId' => '123456789',
136+
'taskTypeId' => \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID,
137+
]);
138+
$url = str_replace($this->urlGenerator->getBaseUrl(), '', $url);
139+
$url = str_replace('123456789', '{fileId}', $url);
140+
$endpoint = [
141+
'name' => $this->l->t('Generate subtitles using AI'),
142+
'url' => $url,
143+
'method' => 'POST',
144+
'mimetype_filters' => 'audio/, video/',
145+
'icon' => $this->urlGenerator->imagePath(Application::APP_ID, 'client_integration/speech_to_text.svg'),
146+
];
147+
$capabilities['client_integration'][Application::APP_ID]['context-menu'][] = $endpoint;
148+
}
131149
}
132150

133151
if ($ttsAvailable) {

‎lib/Controller/AssistantApiController.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,10 @@ public function runFileAction(int $fileId, string $taskTypeId): DataResponse {
432432
$message = $this->l10n->t('Assistant task submitted successfully');
433433
if ($taskTypeId === AudioToText::ID) {
434434
$message = $this->l10n->t('Transcription task submitted successfully');
435+
} elseif (class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles')) {
436+
if ($taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID) {
437+
$message = $this->l10n->t('Subtitles task submitted successfully');
438+
}
435439
} elseif ($taskTypeId === TextToTextSummary::ID) {
436440
$message = $this->l10n->t('Summarization task submitted successfully');
437441
} elseif (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech')) {

‎lib/Listener/FileActionTaskSuccessfulListener.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,18 @@ class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech')
7575
}
7676
$targetFileName = $sourceFile->getName() . ' - text to speech.' . $extension;
7777
$targetFile = $sourceFileParent->newFile($targetFileName, $speechFile->fopen('rb'));
78+
} elseif (
79+
class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles')
80+
&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID
81+
) {
82+
$subtitlesFileId = (int)$task->getOutput()['output'];
83+
$subtitlesFile = $this->taskProcessingService->getOutputFile($subtitlesFileId);
84+
$mimeType = mime_content_type($subtitlesFile->fopen('rb'));
85+
$mimeType = $mimeType ?: 'text/plain';
86+
$mimes = new \Mimey\MimeTypes;
87+
$extension = $mimes->getExtension($mimeType);
88+
$targetFileName = $sourceFile->getName() . ' - subtitles.' . $extension;
89+
$targetFile = $sourceFileParent->newFile($targetFileName, $subtitlesFile->fopen('rb'));
7890
} else {
7991
$textResult = $task->getOutput()['output'];
8092
$suffix = $taskTypeId === TextToTextSummary::ID ? 'summarized' : 'transcribed';

‎lib/Notification/Notifier.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ public function prepare(INotification $notification, string $languageCode): INot
9595
$taskTypeName = $l->t('AI image generation');
9696
} elseif ($params['taskTypeId'] === AudioToText::ID) {
9797
$taskTypeName = $l->t('AI audio transcription');
98+
} elseif (class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles')
99+
&& $params['taskTypeId'] === \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID) {
100+
$taskTypeName = $l->t('AI subtitles generation');
98101
} elseif ($params['taskTypeId'] === 'copywriter') {
99102
// TODO adjust that when we have copywriter back on its feet
100103
// Catch the custom copywriter task type built on top of the FreePrompt task type.
@@ -202,6 +205,9 @@ public function prepare(INotification $notification, string $languageCode): INot
202205
case AudioToText::ID:
203206
$message = $l->t('{sourceFile} has been transcribed in {targetFile}');
204207
break;
208+
case class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles') ? \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID : 'nope':
209+
$message = $l->t('{sourceFile} has been subtitled in {targetFile}');
210+
break;
205211
case class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech') ? \OCP\TaskProcessing\TaskTypes\TextToSpeech::ID : 'nope':
206212
$message = $l->t('{sourceFile} has been converted to audio in {targetFile}');
207213
break;
@@ -253,6 +259,9 @@ public function prepare(INotification $notification, string $languageCode): INot
253259
case AudioToText::ID:
254260
$message = $l->t('Transcription of {sourceFile} has failed');
255261
break;
262+
case class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles') ? \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID : 'nope':
263+
$message = $l->t('Subtitling of {sourceFile} has failed');
264+
break;
256265
case class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech') ? \OCP\TaskProcessing\TaskTypes\TextToSpeech::ID : 'nope':
257266
$message = $l->t('The text-to-speech process for {sourceFile} has failed');
258267
break;

‎lib/Service/AssistantService.php‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,14 @@ class AssistantService {
6969
'context_chat:context_chat' => 3,
7070
'legacy:TextProcessing:OCA\ContextChat\TextProcessing\ContextChatTaskType' => 3,
7171
'context_chat:context_chat_search' => 4,
72-
AudioToText::ID => 5,
73-
TextToTextTranslate::ID => 6,
74-
ContextWrite::ID => 7,
75-
TextToImage::ID => 8,
76-
TextToTextSummary::ID => 9,
77-
TextToTextHeadline::ID => 10,
78-
TextToTextTopics::ID => 11,
72+
AudioToText::ID => 10,
73+
'core:audio2text:subtitles' => 11,
74+
TextToTextTranslate::ID => 20,
75+
ContextWrite::ID => 21,
76+
TextToImage::ID => 22,
77+
TextToTextSummary::ID => 23,
78+
TextToTextHeadline::ID => 24,
79+
TextToTextTopics::ID => 25,
7980
];
8081

8182
public array $informationSources;

‎lib/Service/TaskProcessingService.php‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ public function isFileActionTaskTypeSupported(string $taskTypeId): bool {
9393
if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToSpeech')) {
9494
$authorizedTaskTypes[] = \OCP\TaskProcessing\TaskTypes\TextToSpeech::ID;
9595
}
96+
if (class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles')) {
97+
$authorizedTaskTypes[] = \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID;
98+
}
9699
return in_array($taskTypeId, $authorizedTaskTypes, true);
97100
}
98101

@@ -111,7 +114,7 @@ public function runFileAction(string $userId, int $fileId, string $taskTypeId):
111114
throw new Exception('Invalid task type for file action');
112115
}
113116
try {
114-
$input = $taskTypeId === AudioToText::ID
117+
$input = ($taskTypeId === AudioToText::ID) || (class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToTextSubtitles') && $taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles::ID)
115118
? ['input' => $fileId]
116119
: ['input' => $this->assistantService->parseTextFromFile($userId, fileId: $fileId)];
117120
} catch (NotPermittedException|GenericFileException|LockedException|\OCP\Files\NotFoundException|Exception $e) {

‎src/files/fileActions.js‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import CreationSvgIcon from '@mdi/svg/svg/creation.svg?raw'
1010
import SummarizeSymbol from '@material-symbols/svg-700/outlined/summarize.svg?raw'
1111
import TTSSymbol from '@material-symbols/svg-700/outlined/text_to_speech.svg?raw'
1212
import STTSymbol from '@material-symbols/svg-700/outlined/speech_to_text.svg?raw'
13-
import { VALID_AUDIO_MIME_TYPES, VALID_TEXT_MIME_TYPES } from '../constants.js'
13+
import { VALID_AUDIO_MIME_TYPES, VALID_TEXT_MIME_TYPES, VALID_VIDEO_MIME_TYPES } from '../constants.js'
1414

1515
const actionIgnoreLists = [
1616
'trashbin',
@@ -156,6 +156,45 @@ function registerSttAction() {
156156
registerFileAction(sttAction)
157157
}
158158

159+
function registerSttSubtitlesAction() {
160+
const sttSubtitlesAction = {
161+
id: 'assistant-stt-subtitles',
162+
parent: 'assistant-group',
163+
displayName: ({ nodes }) => {
164+
return t('assistant', 'Generate subtitles using AI')
165+
},
166+
enabled({ nodes, view }) {
167+
return !actionIgnoreLists.includes(view.id)
168+
&& nodes.length === 1
169+
&& !nodes.some(({ permissions }) => (permissions & Permission.READ) === 0)
170+
&& nodes.every(({ type }) => type === FileType.File)
171+
&& nodes.every(({ mime }) => VALID_AUDIO_MIME_TYPES.includes(mime) || VALID_VIDEO_MIME_TYPES.includes(mime))
172+
},
173+
iconSvgInline: () => STTSymbol,
174+
order: 0,
175+
async exec({ nodes }) {
176+
const node = nodes[0]
177+
const { default: axios } = await import('@nextcloud/axios')
178+
const { generateOcsUrl } = await import('@nextcloud/router')
179+
const { showError, showSuccess } = await import('@nextcloud/dialogs')
180+
const url = generateOcsUrl('/apps/assistant/api/v1/file-action/{fileId}/core:audio2text:subtitles', { fileId: node.fileid })
181+
try {
182+
await axios.post(url)
183+
showSuccess(
184+
t('assistant', 'AI subtitles task submitted successfully.') + '\n'
185+
+ t('assistant', 'You will be notified when it is ready.') + '\n'
186+
+ t('assistant', 'It can also be checked in the Assistant in the "Work with audio -> Generate subtitles" menu.'),
187+
)
188+
} catch (error) {
189+
console.error(error)
190+
showError(t('assistant', 'Failed to launch the AI file action'))
191+
}
192+
return null
193+
},
194+
}
195+
registerFileAction(sttSubtitlesAction)
196+
}
197+
159198
const assistantEnabled = loadState('assistant', 'assistant-enabled', false)
160199
const summarizeAvailable = loadState('assistant', 'summarize-available', false)
161200
const sttAvailable = loadState('assistant', 'stt-available', false)
@@ -174,6 +213,7 @@ if (assistantEnabled) {
174213
}
175214
if (sttAvailable) {
176215
registerSttAction()
216+
registerSttSubtitlesAction()
177217
}
178218
if (summarizeAvailable) {
179219
registerSummarizeAction()

0 commit comments

Comments
 (0)