Source: Providers/Localhost/OllamaMultimodal.php

  1. <?php
  2. /**
  3.  * Ollama Multimodal integration
  4.  */
  6. namespace Classifai\Providers\Localhost;
  8. use Classifai\Features\DescriptiveTextGenerator;
  9. use Classifai\Features\ImageTagsGenerator;
  10. use Classifai\Features\ImageTextExtraction;
  11. use Classifai\Providers\OpenAI\APIRequest;
  12. use WP_Error;
  14. use function Classifai\get_largest_size_and_dimensions_image_url;
  15. use function Classifai\get_modified_image_source_url;
  16. use function Classifai\computer_vision_max_filesize;
  17. use function Classifai\get_default_prompt;
  19. /**
  20.  * Ollama Multimodal class
  21.  */
  22. class OllamaMultimodal extends Ollama {
  24. 	/**
  25. 	 * The Provider ID.
  26. 	 */
  27. 	const ID = 'ollama_multimodal';
  29. 	/**
  30. 	 * Connects to Ollama and retrieves supported models.
  31. 	 *
  32. 	 * @param array $args Overridable args.
  33. 	 * @return array
  34. 	 */
  35. 	public function get_models( array $args = [] ): array {
  36. 		$models = parent::get_models( $args );
  38. 		$supported_models = [
  39. 			'llava',
  40. 			'bakllava',
  41. 			'llama3.2-vision',
  42. 			'llava-llama3',
  43. 			'moondream',
  44. 			'minicpm-v',
  45. 			'llava-phi3',
  46. 		];
  48. 		// Ensure our model list only contains the ones we support.
  49. 		foreach ( $models as $key => $model ) {
  50. 			$model = explode( ':', $model );
  52. 			if ( ! in_array( $model[0], $supported_models, true ) ) {
  53. 				unset( $models[ $key ] );
  54. 			}
  55. 		}
  57. 		return $models;
  58. 	}
  60. 	/**
  61. 	 * Returns the default settings for this provider.
  62. 	 *
  63. 	 * @return array
  64. 	 */
  65. 	public function get_default_provider_settings(): array {
  66. 		$common_settings = parent::get_default_provider_settings();
  68. 		/**
  69. 		 * Default values for feature specific settings.
  70. 		 */
  71. 		switch ( $this->feature_instance::ID ) {
  72. 			case DescriptiveTextGenerator::ID:
  73. 			case ImageTagsGenerator::ID:
  74. 			case ImageTextExtraction::ID:
  75. 				$common_settings['prompt'] = [
  76. 					[
  77. 						'title'    => esc_html__( 'ClassifAI default', 'classifai' ),
  78. 						'prompt'   => $this->feature_instance->prompt,
  79. 						'original' => 1,
  80. 						'default'  => 1,
  81. 					],
  82. 				];
  83. 				break;
  84. 		}
  86. 		return $common_settings;
  87. 	}
  89. 	/**
  90. 	 * Generic request handler for multimodal LLMs run by Ollama.
  91. 	 *
  92. 	 * @param string $url Request URL.
  93. 	 * @param array  $body Request body.
  94. 	 * @return string|WP_Error
  95. 	 */
  96. 	public function request( string $url, array $body ) {
  97. 		// Make our API request.
  98. 		$request  = new APIRequest( 'test' );
  99. 		$response = $request->post(
  100. 			$url,
  101. 			[
  102. 				'body' => wp_json_encode( $body ),
  103. 			]
  104. 		);
  106. 		// Return the error if we have one.
  107. 		if ( is_wp_error( $response ) ) {
  108. 			return $response;
  109. 		}
  111. 		// If we have a message, return it.
  112. 		$return = '';
  113. 		if ( isset( $response['response'] ) ) {
  114. 			$return = sanitize_text_field( trim( $response['response'], ' "\'' ) );
  115. 		}
  117. 		return $return;
  118. 	}
  120. 	/**
  121. 	 * Generate descriptive alt text for an image.
  122. 	 *
  123. 	 * @param string $image_url URL of image to process.
  124. 	 * @param int    $attachment_id Post ID for the attachment.
  125. 	 * @return string|WP_Error
  126. 	 */
  127. 	public function generate_alt_text( string $image_url, int $attachment_id ) {
  128. 		if ( ! wp_attachment_is_image( $attachment_id ) ) {
  129. 			return new WP_Error( 'invalid', esc_html__( 'This attachment can\'t be processed.', 'classifai' ) );
  130. 		}
  132. 		$feature  = new DescriptiveTextGenerator();
  133. 		$settings = $feature->get_settings( static::ID );
  135. 		// Ensure things are set up properly.
  136. 		if ( empty( $settings ) || ( isset( $settings[ static::ID ]['authenticated'] ) && false === $settings[ static::ID ]['authenticated'] ) || ( ! $feature->is_feature_enabled() ) ) {
  137. 			return new WP_Error( 'not_enabled', esc_html__( 'Descriptive text generation is disabled or Ollama authentication failed. Please check your settings.', 'classifai' ) );
  138. 		}
  140. 		// Download the image so we can encode it.
  141. 		$image_data = file_get_contents( $image_url ); // phpcs:ignore
  143. 		if ( false === $image_data || ! is_string( $image_data ) ) {
  144. 			return new WP_Error( 'invalid', esc_html__( 'Image cannot be downloaded.', 'classifai' ) );
  145. 		}
  147. 		/**
  148. 		 * Filter the prompt we will send to Ollama.
  149. 		 *
  150. 		 * @since 3.3.0
  151. 		 * @hook classifai_ollama_descriptive_text_prompt
  152. 		 *
  153. 		 * @param {string} $prompt Prompt we are sending to Ollama.
  154. 		 * @param {int} $attachment_id ID of attachment.
  155. 		 *
  156. 		 * @return {string} Prompt.
  157. 		 */
  158. 		$prompt = apply_filters( 'classifai_ollama_descriptive_text_prompt', get_default_prompt( $settings[ static::ID ]['prompt'] ?? [] ) ?? $feature->prompt, $attachment_id );
  160. 		/**
  161. 		 * Filter the request body before sending to Ollama.
  162. 		 *
  163. 		 * @since 3.3.0
  164. 		 * @hook classifai_ollama_descriptive_text_request_body
  165. 		 *
  166. 		 * @param {array} $body Request body that will be sent to Ollama.
  167. 		 * @param {int} $attachment_id ID of attachment.
  168. 		 *
  169. 		 * @return {array} Request body.
  170. 		 */
  171. 		$body = apply_filters(
  172. 			'classifai_ollama_descriptive_text_request_body',
  173. 			[
  174. 				'model'  => $settings['model'] ?? '',
  175. 				'prompt' => $prompt,
  176. 				'images' => [ base64_encode( $image_data ) ], // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode
  177. 				'stream' => false,
  178. 			],
  179. 			$attachment_id
  180. 		);
  182. 		// Make our API request.
  183. 		$response = $this->request( $this->get_api_generate_url( $settings['endpoint_url'] ?? '' ), $body );
  185. 		return $response;
  186. 	}
  188. 	/**
  189. 	 * Extract text out of an image.
  190. 	 *
  191. 	 * @param string $image_url URL of image to process.
  192. 	 * @param int    $attachment_id Post ID for the attachment.
  193. 	 * @return string|WP_Error
  194. 	 */
  195. 	public function ocr_processing( string $image_url, int $attachment_id ) {
  196. 		if ( ! wp_attachment_is_image( $attachment_id ) ) {
  197. 			return new WP_Error( 'invalid', esc_html__( 'This attachment can\'t be processed.', 'classifai' ) );
  198. 		}
  200. 		$feature  = new ImageTextExtraction();
  201. 		$settings = $feature->get_settings( static::ID );
  203. 		// Ensure things are set up properly.
  204. 		if ( empty( $settings ) || ( isset( $settings[ static::ID ]['authenticated'] ) && false === $settings[ static::ID ]['authenticated'] ) || ( ! $feature->is_feature_enabled() ) ) {
  205. 			return new WP_Error( 'not_enabled', esc_html__( 'Image Text Extraction is disabled or Ollama authentication failed. Please check your settings.', 'classifai' ) );
  206. 		}
  208. 		// Download the image so we can encode it.
  209. 		$image_data = file_get_contents( $image_url ); // phpcs:ignore
  211. 		if ( false === $image_data || ! is_string( $image_data ) ) {
  212. 			return new WP_Error( 'invalid', esc_html__( 'Image cannot be downloaded.', 'classifai' ) );
  213. 		}
  215. 		/**
  216. 		 * Filter the prompt we will send to Ollama.
  217. 		 *
  218. 		 * @since 3.3.0
  219. 		 * @hook classifai_ollama_ocr_processing_prompt
  220. 		 *
  221. 		 * @param {string} $prompt Prompt we are sending to Ollama.
  222. 		 * @param {int} $attachment_id ID of attachment.
  223. 		 *
  224. 		 * @return {string} Prompt.
  225. 		 */
  226. 		$prompt = apply_filters( 'classifai_ollama_ocr_processing_prompt', get_default_prompt( $settings[ static::ID ]['prompt'] ?? [] ) ?? $feature->prompt, $attachment_id );
  228. 		/**
  229. 		 * Filter the request body before sending to Ollama.
  230. 		 *
  231. 		 * @since 3.3.0
  232. 		 * @hook classifai_ollama_ocr_processing_request_body
  233. 		 *
  234. 		 * @param {array} $body Request body that will be sent to Ollama.
  235. 		 * @param {int} $attachment_id ID of attachment.
  236. 		 *
  237. 		 * @return {array} Request body.
  238. 		 */
  239. 		$body = apply_filters(
  240. 			'classifai_ollama_ocr_processing_request_body',
  241. 			[
  242. 				'model'  => $settings['model'] ?? '',
  243. 				'prompt' => $prompt,
  244. 				'images' => [ base64_encode( $image_data ) ], // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode
  245. 				'stream' => false,
  246. 			],
  247. 			$attachment_id
  248. 		);
  250. 		// Make our API request.
  251. 		$response = $this->request( $this->get_api_generate_url( $settings['endpoint_url'] ?? '' ), $body );
  253. 		// If no text was found, return an empty string.
  254. 		if ( 'none' === strtolower( $response ) ) {
  255. 			return '';
  256. 		}
  258. 		return $response;
  259. 	}
  261. 	/**
  262. 	 * Generate tags for an image.
  263. 	 *
  264. 	 * @param string $image_url URL of image to process.
  265. 	 * @param int    $attachment_id Post ID for the attachment.
  266. 	 * @return array|WP_Error
  267. 	 */
  268. 	public function generate_image_tags( string $image_url, int $attachment_id ) {
  269. 		if ( ! wp_attachment_is_image( $attachment_id ) ) {
  270. 			return new WP_Error( 'invalid', esc_html__( 'This attachment can\'t be processed.', 'classifai' ) );
  271. 		}
  273. 		$feature  = new ImageTagsGenerator();
  274. 		$settings = $feature->get_settings( static::ID );
  276. 		// Ensure things are set up properly.
  277. 		if ( empty( $settings ) || ( isset( $settings[ static::ID ]['authenticated'] ) && false === $settings[ static::ID ]['authenticated'] ) || ( ! $feature->is_feature_enabled() ) ) {
  278. 			return new WP_Error( 'not_enabled', esc_html__( 'Image tag generation is disabled or Ollama authentication failed. Please check your settings.', 'classifai' ) );
  279. 		}
  281. 		// Download the image so we can encode it.
  282. 		$image_data = file_get_contents( $image_url ); // phpcs:ignore
  284. 		if ( false === $image_data || ! is_string( $image_data ) ) {
  285. 			return new WP_Error( 'invalid', esc_html__( 'Image cannot be downloaded.', 'classifai' ) );
  286. 		}
  288. 		/**
  289. 		 * Filter the prompt we will send to Ollama.
  290. 		 *
  291. 		 * @since 3.3.0
  292. 		 * @hook classifai_ollama_image_tag_prompt
  293. 		 *
  294. 		 * @param {string} $prompt Prompt we are sending to Ollama.
  295. 		 * @param {int} $attachment_id ID of attachment.
  296. 		 *
  297. 		 * @return {string} Prompt.
  298. 		 */
  299. 		$prompt = apply_filters( 'classifai_ollama_image_tag_prompt', get_default_prompt( $settings[ static::ID ]['prompt'] ?? [] ) ?? $feature->prompt, $attachment_id );
  301. 		/**
  302. 		 * Filter the request body before sending to Ollama.
  303. 		 *
  304. 		 * @since 3.3.0
  305. 		 * @hook classifai_ollama_image_tag_request_body
  306. 		 *
  307. 		 * @param {array} $body Request body that will be sent to Ollama.
  308. 		 * @param {int} $attachment_id ID of attachment.
  309. 		 *
  310. 		 * @return {array} Request body.
  311. 		 */
  312. 		$body = apply_filters(
  313. 			'classifai_ollama_image_tag_request_body',
  314. 			[
  315. 				'model'  => $settings['model'] ?? '',
  316. 				'prompt' => $prompt,
  317. 				'images' => [ base64_encode( $image_data ) ], // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode
  318. 				'stream' => false,
  319. 			],
  320. 			$attachment_id
  321. 		);
  323. 		// Make our API request.
  324. 		$response = $this->request( $this->get_api_generate_url( $settings['endpoint_url'] ?? '' ), $body );
  326. 		// If we have a response, clean it up.
  327. 		if ( ! is_wp_error( $response ) ) {
  328. 			$response = array_filter( explode( '- ', $response ) );
  329. 			$response = array_map( 'trim', $response );
  330. 		}
  332. 		// Ensure we have a valid response after processing.
  333. 		if ( ! is_array( $response ) || empty( $response ) ) {
  334. 			return new WP_Error( 'error', esc_html__( 'No tags found.', 'classifai' ) );
  335. 		}
  337. 		return $response;
  338. 	}
  340. 	/**
  341. 	 * Common entry point for all REST endpoints for this provider.
  342. 	 *
  343. 	 * @param int    $attachment_id The attachment ID we're processing.
  344. 	 * @param string $route_to_call The name of the route we're going to be processing.
  345. 	 * @param array  $args          Optional arguments to pass to the route.
  346. 	 * @return array|string|WP_Error|null
  347. 	 */
  348. 	public function rest_endpoint_callback( $attachment_id, string $route_to_call = '', array $args = [] ) {
  349. 		// Check to be sure the post both exists and is an attachment.
  350. 		if ( ! get_post( $attachment_id ) || 'attachment' !== get_post_type( $attachment_id ) ) {
  351. 			/* translators: %1$s: the attachment ID */
  352. 			return new WP_Error( 'incorrect_ID', sprintf( esc_html__( '%1$d is not found or is not an attachment', 'classifai' ), $attachment_id ), [ 'status' => 404 ] );
  353. 		}
  355. 		$metadata = wp_get_attachment_metadata( $attachment_id );
  357. 		if ( ! $metadata || ! is_array( $metadata ) ) {
  358. 			return new WP_Error( 'invalid', esc_html__( 'No valid metadata found.', 'classifai' ) );
  359. 		}
  361. 		$image_url = get_modified_image_source_url( $attachment_id );
  363. 		if ( empty( $image_url ) || ! filter_var( $image_url, FILTER_VALIDATE_URL ) ) {
  364. 			if ( isset( $metadata['sizes'] ) && is_array( $metadata['sizes'] ) ) {
  365. 				$image_url = get_largest_size_and_dimensions_image_url(
  366. 					get_attached_file( $attachment_id ),
  367. 					wp_get_attachment_url( $attachment_id ),
  368. 					$metadata,
  369. 					[ 50, 16000 ],
  370. 					[ 50, 16000 ],
  371. 					computer_vision_max_filesize()
  372. 				);
  373. 			} else {
  374. 				$image_url = wp_get_attachment_url( $attachment_id );
  375. 			}
  376. 		}
  378. 		if ( empty( $image_url ) ) {
  379. 			return new WP_Error( 'error', esc_html__( 'Image does not meet size requirements. Please ensure it is at least 50x50 but less than 16000x16000 and smaller than 20MB.', 'classifai' ) );
  380. 		}
  382. 		switch ( $route_to_call ) {
  383. 			case 'descriptive_text':
  384. 				return $this->generate_alt_text( $image_url, $attachment_id );
  386. 			case 'ocr':
  387. 				return $this->ocr_processing( $image_url, $attachment_id );
  389. 			case 'tags':
  390. 				return $this->generate_image_tags( $image_url, $attachment_id );
  391. 		}
  392. 	}
  393. }