feat: add support for multimodal indexing

appinfo/info.xml

@@ -48,6 +48,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
 	</background-jobs>
 	<commands>
 		<command>OCA\ContextChat\Command\Prompt</command>
+		<command>OCA\ContextChat\Command\QueueMultimodalFiles</command>
 		<command>OCA\ContextChat\Command\Search</command>
 		<command>OCA\ContextChat\Command\Statistics</command>
 	</commands>

lib/AppInfo/Application.php

@@ -61,6 +61,35 @@ class Application extends App implements IBootstrap {
 		'text/org',
 	];
 
+	public const IMAGE_MIMETYPES = [
+		'image/bmp',
+		'image/bpg',
+		'image/emf',
+		'image/gif',
+		'image/heic',
+		'image/heif',
+		'image/jp2',
+		'image/jpeg',
+		'image/png',
+		'image/svg+xml',
+		'image/tga',
+		'image/tiff',
+		'image/webp',
+		'image/x-dcraw',
+		'image/x-icon',
+	];
+
+	public const AUDIO_MIMETYPES = [
+		'audio/aac',
+		'audio/flac',
+		'audio/mp4',
+		'audio/mpeg',
+		'audio/ogg',
+		'audio/wav',
+		'audio/webm',
+		'audio/x-scpls',
+	];
+
 	public function __construct(array $urlParams = []) {
 		parent::__construct(self::APP_ID, $urlParams);
 	}

lib/BackgroundJobs/StorageCrawlJob.php

@@ -15,6 +15,7 @@
 use OCA\ContextChat\Service\DiagnosticService;
 use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
+use OCA\ContextChat\Service\TaskTypeService;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
@@ -33,19 +34,22 @@ public function __construct(
 		private StorageService $storageService,
 		private DiagnosticService $diagnosticService,
 		private IAppConfig $appConfig,
+		private TaskTypeService $taskTypeService,
 	) {
 		parent::__construct($timeFactory);
 	}
 
 	/**
-	 * @param array{storage_id:int, root_id:int, overridden_root:int|null, override_root:int|null, last_file_id:int|null} $argument
+	 * @param array{storage_id:int, root_id:int, overridden_root:int|null, override_root:int|null, last_file_id:int|null, only_non_textual?:bool} $argument
 	 * @return void
 	 */
 	protected function run($argument): void {
 		$storageId = $argument['storage_id'];
 		$rootId = $argument['root_id'];
 		$overrideRoot = $argument['overridden_root'] ?? $argument['override_root'] ?? $rootId;
 		$lastFileId = ($argument['last_file_id'] ?? null) === null ? 0 : $argument['last_file_id'];
+		$onlyNonTextual = $argument['only_non_textual'] ?? false;
+		$mimeTypes = $this->taskTypeService->getMultimodalMimetypes(!$onlyNonTextual);
 
 		// Remove current iteration
 		$this->jobList->remove(self::class, $argument);
@@ -57,7 +61,7 @@ protected function run($argument): void {
 			$mountFilesCount = 0;
 			$lastSuccessfulDbId = -1;
 			$queueFile = null;
-			foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
+			foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE, $mimeTypes) as $fileId) {
 				$queueFile = new QueueFile();
 				$queueFile->setStorageId($storageId);
 				$queueFile->setRootId($rootId);

lib/Command/QueueMultimodalFiles.php

@@ -0,0 +1,76 @@
+<?php
+
+/**
+ * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\ContextChat\Command;
+
+use OCA\ContextChat\Service\MultimodalService;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class QueueMultimodalFiles extends Command {
+
+	public function __construct(
+		private MultimodalService $multimodalService,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('context_chat:queue-multimodal-files')
+			->setDescription(
+				'Queue existing multimodal files (Images and Audio) for indexation.'
+				. ' Each type of files is queued only if the required TaskProcessing task provider is available.'
+				. ' OCR for Images and Speech-to-text for Audio.'
+				. ' See https://docs.nextcloud.com/server/latest/admin_manual/ai/overview.html for more information.'
+			)
+			->addOption(
+				'force',
+				null,
+				InputOption::VALUE_NONE,
+				'Queue multimodal files regardless of whether it has been done previously already.',
+			);
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$taskTypes = $this->multimodalService->checkTaskTypes();
+
+		if (!$taskTypes['ocrAvailable']) {
+			$output->writeln('<warning>OCR task type is not configured. Image files will not be indexed.</warning>');
+		}
+		if (!$taskTypes['sttAvailable']) {
+			$output->writeln('<warning>Speech-to-text task type is not configured. Audio files will not be indexed.</warning>');
+		}
+
+		if (!$taskTypes['ocrAvailable'] && !$taskTypes['sttAvailable']) {
+			$output->writeln('<error>No multimodal task types are available. No files will be queued for indexing.</error>');
+			return 1;
+		}
+
+		try {
+			$this->multimodalService->enableMultimodal((bool)$input->getOption('force'));
+		} catch (\Exception $e) {
+			$output->writeln('<error>' . $e->getMessage() . '</error>');
+			return 1;
+		}
+
+		$errors = $this->multimodalService->queueExistingMultimodalFiles();
+
+		foreach ($errors as $error) {
+			$output->writeln('<warning>' . $error . '</warning>');
+		}
+
+		if (empty($errors)) {
+			$output->writeln(
+				'<info>Multimodal files have been scheduled to be queued for indexation. '
+				. 'They will be queued in the subsequent cron runs automatically.</info>'
+			);
+		}
+		return empty($errors) ? 0 : 1;
+	}
+}

lib/Controller/AppDataController.php

@@ -0,0 +1,140 @@
+<?php
+
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\ContextChat\Controller;
+
+use OCA\ContextChat\AppInfo\Application;
+use OCP\AppFramework\Http;
+use OCP\AppFramework\Http\Attribute\ApiRoute;
+use OCP\AppFramework\Http\Attribute\ExAppRequired;
+use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\OCSController;
+use OCP\Files\AppData\IAppDataFactory;
+use OCP\Files\NotFoundException;
+use OCP\Files\NotPermittedException;
+use OCP\Files\SimpleFS\ISimpleFolder;
+use OCP\IRequest;
+use Psr\Log\LoggerInterface;
+
+class AppDataController extends OCSController {
+	private const APP_DATA_FOLDER_NAME = 'temp_tp_files';
+	private ?ISimpleFolder $appDataFolder = null;
+
+	public function __construct(
+		string $appName,
+		IRequest $request,
+		private LoggerInterface $logger,
+		private IAppDataFactory $appDataFactory,
+	) {
+		parent::__construct($appName, $request);
+	}
+
+	/**
+	 * Stores files to the appdata and returns corresponding NC file ids
+	 * @return DataResponse
+	 */
+	#[ExAppRequired]
+	#[ApiRoute(verb: 'POST', url: '/upload_files')]
+	public function uploadTempFile() : DataResponse {
+		$files = $this->request->files;
+		if (count($files) === 0) {
+			return new DataResponse('No files provided.', Http::STATUS_BAD_REQUEST);
+		}
+
+		if (count($files) > Application::CC_MAX_FILES) {
+			return new DataResponse(
+				'No. of files exceeds permissible limit of ' . Application::CC_MAX_FILES,
+				Http::STATUS_REQUEST_ENTITY_TOO_LARGE,
+			);
+		}
+
+		/** @var array<string, int> */
+		$fileIds = [];
+		/** @var array<string, array{message: string, code: int}> */
+		$errors = [];
+
+		foreach ($files as $filename => $file) {
+			if ($file['size'] > Application::CC_MAX_SIZE) {
+				$errors[$filename] = [
+					'message' => 'Max file size exceeds permissible limit of ' . Application::CC_MAX_SIZE . ' bytes',
+					'code' => Http::STATUS_REQUEST_ENTITY_TOO_LARGE,
+				];
+			}
+
+			if ($file['error'] !== 0) {
+				$errors[$filename] = [
+					'message' => 'Error in input file upload: ' . $file['error'],
+					'code' => $file['error'] <= UPLOAD_ERR_NO_FILE
+						? Http::STATUS_BAD_REQUEST
+						: Http::STATUS_INTERNAL_SERVER_ERROR,
+				];
+			}
+
+			if (empty($file)) {
+				$errors[$filename] = [
+					'message' => 'Invalid input data received',
+					'code' => Http::STATUS_BAD_REQUEST,
+				];
+			}
+
+			try {
+				// tmp_name is the temporary filename of the file in which the uploaded file was stored on the server.
+				$fileIds[$filename] = $this->storeTempFile($file['tmp_name']);
+			} catch (NotPermittedException $e) {
+				$this->logger->error('No permission to write the appdata folder: ' . $e->getMessage(), ['exception' => $e]);
+				// this error message is fine since it's a ex-app only path
+				return new DataResponse(
+					'No permission to write the appdata folder: ' . $e->getMessage(),
+					Http::STATUS_INTERNAL_SERVER_ERROR,
+				);
+			} catch (\Exception $e) {
+				$this->logger->error('Failed to store input file: ' . $e->getMessage(), ['exception' => $e]);
+				$errors[$filename] = [
+					'message' => 'Failed to store the input file: ' . $e->getMessage(),
+					'code' => Http::STATUS_INTERNAL_SERVER_ERROR,
+				];
+			}
+		}
+
+		return new DataResponse([
+			'fileIds' => $fileIds,
+			'errors' => (object)$errors,
+		]);
+	}
+
+	/**
+	 * @param string $tempFileLocation
+	 * @return int file ID of the stored file
+	 * @throws \RuntimeException
+	 * @throws NotPermittedException
+	 * @throws \OCP\DB\Exception
+	 */
+	private function storeTempFile(string $tempFileLocation): int {
+		if ($this->appDataFolder === null) {
+			$appDataFolder = $this->appDataFactory->get(Application::APP_ID);
+			try {
+				$this->appDataFolder = $appDataFolder->getFolder(self::APP_DATA_FOLDER_NAME);
+			} catch (NotFoundException) {
+				$this->appDataFolder = $appDataFolder->newFolder(self::APP_DATA_FOLDER_NAME);
+			}
+		}
+
+		// file handle is closed in the newFile call
+		$tempFileHandle = fopen($tempFileLocation, 'rb');
+		if ($tempFileHandle === false) {
+			throw new \RuntimeException('Failed to open temporary file');
+		}
+
+		$targetFileName = 'cc_temp_' . time() . '_' . random_int(100000, 999999);
+		$targetFile = $this->appDataFolder->newFile($targetFileName, $tempFileHandle);
+
+		return $targetFile->getId(); // polymorphic call to SimpleFile
+	}
+}

lib/Controller/ConfigController.php

@@ -7,6 +7,7 @@
 
 namespace OCA\ContextChat\Controller;
 
+use OCA\ContextChat\Service\MultimodalService;
 use OCP\AppFramework\Controller;
 use OCP\AppFramework\Http\Attribute\NoAdminRequired;
 use OCP\AppFramework\Http\DataResponse;

lib/Controller/UtilController.php

@@ -11,11 +11,13 @@
 namespace OCA\ContextChat\Controller;
 
 use OCA\ContextChat\Service\MetadataService;
+use OCA\ContextChat\Service\MultimodalService;
 use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\Service\StorageService;
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\Attribute\ApiRoute;
 use OCP\AppFramework\Http\Attribute\ExAppRequired;
+use OCP\AppFramework\Http\Attribute\Route;
 use OCP\AppFramework\Http\DataResponse;
 use OCP\AppFramework\OCSController;
 use OCP\Files\Folder;
@@ -28,6 +30,7 @@ public function __construct(
 		string $appName,
 		IRequest $request,
 		private LoggerInterface $logger,
+		private MultimodalService $multimodalService,
 		string $corsMethods = 'POST',
 		string $corsAllowedHeaders = 'Authorization, Content-Type, Accept, OCS-APIRequest',
 		int $corsMaxAge = 1728000,
@@ -97,4 +100,30 @@ public function enrichSources(MetadataService $metadataService, array $sources,
 			return new DataResponse([], Http::STATUS_NOT_FOUND);
 		}
 	}
+
+	/**
+	 * Enable multimodal indexing and queue existing multimodal files.
+	 *
+	 * @param bool $force Queue multimodal files even if multimodal indexing was already enabled previously.
+	 * @return DataResponse
+	 */
+	#[Route(type: Route::TYPE_FRONTPAGE, verb: 'POST', url: '/queue-multimodal-files')]
+	public function queueMultimodalFiles(bool $force = false): DataResponse {
+		$taskTypes = $this->multimodalService->checkTaskTypes();
+
+		if (!$taskTypes['ocrAvailable'] && !$taskTypes['sttAvailable']) {
+			return new DataResponse(
+				['error' => 'No multimodal task types are available. No files will be queued for indexing.'],
+				Http::STATUS_PRECONDITION_FAILED,
+			);
+		}
+
+		try {
+			$this->multimodalService->enableMultimodal($force);
+		} catch (\Exception $e) {
+			return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
+		}
+
+		return new DataResponse(['errors' => $this->multimodalService->queueExistingMultimodalFiles()]);
+	}
 }

lib/Listener/ShareListener.php

@@ -10,12 +10,12 @@
 
 namespace OCA\ContextChat\Listener;
 
-use OCA\ContextChat\AppInfo\Application;
 use OCA\ContextChat\Logger;
 use OCA\ContextChat\Public\UpdateAccessOp;
 use OCA\ContextChat\Service\ActionScheduler;
 use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\Service\StorageService;
+use OCA\ContextChat\Service\TaskTypeService;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
 use OCP\Files\FileInfo;
@@ -37,6 +37,7 @@ public function __construct(
 		private IManager $shareManager,
 		private ActionScheduler $actionService,
 		private IGroupManager $groupManager,
+		private TaskTypeService $taskTypeService,
 	) {
 	}
 
@@ -145,6 +146,7 @@ public function handle(Event $event): void {
 
 	private function allowedMimeType(Node $file): bool {
 		$mimeType = $file->getMimeType();
-		return in_array($mimeType, Application::MIMETYPES, true);
+		$mimeTypes = $this->taskTypeService->getMultimodalMimetypes();
+		return in_array($mimeType, $mimeTypes, true);
 	}
 }