From ee8a53f361451a9355b5359b3fa008c463ec0566 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 9 Jan 2026 20:37:20 +0200 Subject: [PATCH 01/10] rust: add experimental qwen3vl embedding support for llamacpp backend --- src-tauri/Cargo.toml | 2 +- src-tauri/src/ai.rs | 92 +++++++++++++++++++ src-tauri/src/ai/embedding.rs | 1 + .../src/ai/embedding/multimodal_llamacpp.rs | 14 +++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src-tauri/src/ai/embedding/multimodal_llamacpp.rs diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index da4c706..b7d90f5 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -29,7 +29,7 @@ chrono = { version = "0.4.42", features = ["serde"] } anyhow = "1.0.100" ignore = "0.4.25" dashmap = "6.1.0" -async-openai = { version = "0.31.1", features = ["responses", "embedding", "chat-completion", "chat-completion-types"] } +async-openai = { version = "0.31.1", features = ["responses", "embedding", "chat-completion", "chat-completion-types", "byot"] } specta = { version = "=2.0.0-rc.22", features = ["derive", "chrono", "serde"] } specta-typescript = "0.0.9" tauri-specta = { version = "=2.0.0-rc.21", features = ["derive", "typescript"] } diff --git a/src-tauri/src/ai.rs b/src-tauri/src/ai.rs index 3caaa82..80c28e1 100644 --- a/src-tauri/src/ai.rs +++ b/src-tauri/src/ai.rs @@ -16,6 +16,9 @@ use async_openai::{ Client, }; use base64::Engine; +use serde_json::json; + +use crate::ai::{self}; pub mod embedding; pub mod llm; @@ -78,6 +81,95 @@ impl AI { Ok(response) } + // the expected structure of qwen3vl embedding multimodal request in llamacpp + // accroding to this PR: https://github.com/ggml-org/llama.cpp/pull/18665 + // and exact this commit: https://github.com/ggml-org/llama.cpp/pull/18665/commits/56a0d87bd022ab017484523beac26cc3a946c5a4 + + // struct ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput is currently typed in this manner to prevent accidentally usage + + pub async fn create_embedding_qwen3vl_llamacpp( + &self, + input: ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput, + model: String, + ) -> Result { + let mut content = vec![json!({})]; + + if let Some(text) = input.text { + content.push(json!({ + "type": "text", + "text": text, + })); + } + + if let Some(url) = input.image_url { + content.push(json!({ + "type": "image_url", + "image_url": { "url": url }, + })); + } + + let request = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingRequest { + model: model, + input: json!(content), + encoding_format: None, + }; + + let response: CreateEmbeddingResponse = self + .client + .embeddings() + .create_byot(&request) + .await + .context("failed to generate multimodal embedding using qwen3vl and llamacpp") + .unwrap(); + + Ok(response) + } + + pub async fn create_embeddings_batch_qwen3vl_llamacpp( + &self, + inputs: Vec, + model: String, + ) -> Result { + let content_array: Vec = inputs + .into_iter() + .map(|input| { + let mut content = Vec::new(); + + if let Some(text) = input.text { + content.push(json!({ + "type": "text", + "text": text, + })); + } + + if let Some(url) = input.image_url { + content.push(json!({ + "type": "image_url", + "image_url": { "url": url }, + })); + } + + json!(content) + }) + .collect(); + + let request = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingRequest { + model: model, + input: json!(content_array), + encoding_format: None, + }; + + let response: CreateEmbeddingResponse = self + .client + .embeddings() + .create_byot(&request) + .await + .context("failed to generate batch of multimodal embedding using qwen3vl and llamacpp") + .unwrap(); + + Ok(response) + } + pub async fn request_llm(&self, input: InputParam, model: String) -> Result { let args = CreateResponseArgs::default() .input(input) diff --git a/src-tauri/src/ai/embedding.rs b/src-tauri/src/ai/embedding.rs index e69de29..5f99a38 100644 --- a/src-tauri/src/ai/embedding.rs +++ b/src-tauri/src/ai/embedding.rs @@ -0,0 +1 @@ +pub mod multimodal_llamacpp; \ No newline at end of file diff --git a/src-tauri/src/ai/embedding/multimodal_llamacpp.rs b/src-tauri/src/ai/embedding/multimodal_llamacpp.rs new file mode 100644 index 0000000..3c44cc4 --- /dev/null +++ b/src-tauri/src/ai/embedding/multimodal_llamacpp.rs @@ -0,0 +1,14 @@ +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub struct MultimodalEmbeddingRequest { + pub model: String, + pub input: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub encoding_format: Option, +} + +pub struct MultimodalEmbeddingInput { + pub text: Option, + pub image_url: Option, +} \ No newline at end of file From a6e912b7a2edc4bc0fcd8fea9dfff724da123711 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 9 Jan 2026 20:42:15 +0200 Subject: [PATCH 02/10] rust: add prompts for embeddings to models structure --- src-tauri/src/database/models.rs | 4 ++++ src/lib/vecdir/bindings.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/database/models.rs b/src-tauri/src/database/models.rs index da0a2ab..c09f758 100644 --- a/src-tauri/src/database/models.rs +++ b/src-tauri/src/database/models.rs @@ -63,6 +63,10 @@ pub struct EmbeddingConfig { pub model: String, pub dimensions: i32, + + pub text_processing_prompt: AIPrompt, + pub image_processing_prompt: AIPrompt, + pub default_processing_prompt: AIPrompt, } // SPACES diff --git a/src/lib/vecdir/bindings.ts b/src/lib/vecdir/bindings.ts index 8213c61..20c1a39 100644 --- a/src/lib/vecdir/bindings.ts +++ b/src/lib/vecdir/bindings.ts @@ -136,7 +136,7 @@ statusEvent: "status-event" export type AIPrompt = { system_prompt: string; user_prompt: string } export type AppConfig = { theme?: string; indexer_parallelism?: number; default_openai_url: string | null } export type BackendReadyEvent = null -export type EmbeddingConfig = { api_base_url: string; api_key: string; model: string; dimensions: number } +export type EmbeddingConfig = { api_base_url: string; api_key: string; model: string; dimensions: number; text_processing_prompt: AIPrompt; image_processing_prompt: AIPrompt; default_processing_prompt: AIPrompt } export type ErrorEvent = { message: string; context: string | null } export type FileMetadata = { id: number; root_id: number; absolute_path: string; filename: string; file_extension: string; file_size: number; description: string | null; modified_at_fs: string; last_indexed_at: string | null; content_hash: string | null; indexing_status: string; indexing_error_message: string | null } export type IndexedRoot = { id: number; space_id: number; path: string; status: string } From 32d3f004a534a0bcaa10886c52c968797b7679b9 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 9 Jan 2026 21:07:29 +0200 Subject: [PATCH 03/10] rust: add search_prompt and multimodal boolean to embeddingConfig model --- src-tauri/src/database/models.rs | 4 ++++ src/lib/vecdir/bindings.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/database/models.rs b/src-tauri/src/database/models.rs index c09f758..ec16401 100644 --- a/src-tauri/src/database/models.rs +++ b/src-tauri/src/database/models.rs @@ -67,6 +67,10 @@ pub struct EmbeddingConfig { pub text_processing_prompt: AIPrompt, pub image_processing_prompt: AIPrompt, pub default_processing_prompt: AIPrompt, + + pub search_prompt: AIPrompt, + + pub multimodal: bool, } // SPACES diff --git a/src/lib/vecdir/bindings.ts b/src/lib/vecdir/bindings.ts index 20c1a39..ecbcc06 100644 --- a/src/lib/vecdir/bindings.ts +++ b/src/lib/vecdir/bindings.ts @@ -136,7 +136,7 @@ statusEvent: "status-event" export type AIPrompt = { system_prompt: string; user_prompt: string } export type AppConfig = { theme?: string; indexer_parallelism?: number; default_openai_url: string | null } export type BackendReadyEvent = null -export type EmbeddingConfig = { api_base_url: string; api_key: string; model: string; dimensions: number; text_processing_prompt: AIPrompt; image_processing_prompt: AIPrompt; default_processing_prompt: AIPrompt } +export type EmbeddingConfig = { api_base_url: string; api_key: string; model: string; dimensions: number; text_processing_prompt: AIPrompt; image_processing_prompt: AIPrompt; default_processing_prompt: AIPrompt; search_prompt: AIPrompt; multimodal: boolean } export type ErrorEvent = { message: string; context: string | null } export type FileMetadata = { id: number; root_id: number; absolute_path: string; filename: string; file_extension: string; file_size: number; description: string | null; modified_at_fs: string; last_indexed_at: string | null; content_hash: string | null; indexing_status: string; indexing_error_message: string | null } export type IndexedRoot = { id: number; space_id: number; path: string; status: string } From 7d74f96320be5b82dbdcb477a36faf63038061f1 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 9 Jan 2026 21:08:17 +0200 Subject: [PATCH 04/10] fe: satisfy updated embeddingConfig schema in createSpace and Settings --- src/components/settings/createSpace.tsx | 190 +++++++++++++++++++++++- src/components/settings/settings.tsx | 170 ++++++++++++++++++++- 2 files changed, 358 insertions(+), 2 deletions(-) diff --git a/src/components/settings/createSpace.tsx b/src/components/settings/createSpace.tsx index 9207f07..79d08e4 100644 --- a/src/components/settings/createSpace.tsx +++ b/src/components/settings/createSpace.tsx @@ -2,10 +2,11 @@ import type { EmbeddingConfig, LLMConfig } from "@/lib/vecdir/bindings"; import { useForm } from "@tanstack/react-form"; import { useNavigate } from "@tanstack/react-router"; -import { Brain, FileImage, FileType, FolderPen, SquaresIntersect } from "lucide-react"; +import { Brain, FileImage, FileType, FolderPen, Search, SquaresIntersect } from "lucide-react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { createSpace } from "@/lib/vecdir/spaces/createSpace"; @@ -53,6 +54,26 @@ export function CreateSpace() { api_base_url: "http://127.0.0.1:1234/v1", // LM Studio api_key: "lmstudio", + + text_processing_prompt: { + system_prompt: "you are a text processing LLM for RAG purposes", + user_prompt: "describe this text file", + }, + image_processing_prompt: { + system_prompt: "you are an image descriptor. you describe images very precisely to use these descriptions in RAG", + user_prompt: "describe this image", + }, + default_processing_prompt: { + system_prompt: "you are a file processing LLM for RAG purposes", + user_prompt: "describe this file based on its metadata: ", + }, + + search_prompt: { + system_prompt: "you are a file processing LLM for RAG purposes", + user_prompt: "describe this file based on its metadata: ", + }, + + multimodal: true, }, }; const form = useForm({ @@ -345,6 +366,173 @@ export function CreateSpace() { )} + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(!field.state.value)} + className="border-border" + /> +
+ )} + + +
+ +

Text Processing Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + + +
+ +

Image Processing Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + + +
+ +

Default Processing Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + + +
+ +

Search Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + [state.canSubmit, state.isSubmitting]} diff --git a/src/components/settings/settings.tsx b/src/components/settings/settings.tsx index 3244438..c2e4a94 100644 --- a/src/components/settings/settings.tsx +++ b/src/components/settings/settings.tsx @@ -1,11 +1,12 @@ import type { EmbeddingConfig, IndexedRoot, LLMConfig } from "@/lib/vecdir/bindings"; import { useForm } from "@tanstack/react-form"; import { open } from "@tauri-apps/plugin-dialog"; -import { Brain, FileImage, FileType, FolderPen, SquareEqual, X } from "lucide-react"; +import { Brain, FileImage, FileType, FolderPen, Search, SquareEqual, X } from "lucide-react"; import { useEffect, useState } from "react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardFooter } from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { addRoot } from "@/lib/vecdir/roots/createRoot"; @@ -408,6 +409,173 @@ export function Settings() { )} + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(!field.state.value)} + className="border-border" + /> +
+ )} + + +
+ +

Text Processing Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + + +
+ +

Image Processing Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + + +
+ +

Default Processing Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + + +
+ +

Search Prompt

+
+ !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="System Prompt" + className="border-border" + /> +
+ )} + + !value ? "This field is required!" : undefined }}> + {field => ( +
+ + field.handleChange(e.target.value)} + placeholder="User Prompt" + className="border-border" + /> +
+ )} + + [state.canSubmit, state.isSubmitting]} From a0b72ae1d85ada19bf179f68eb29739d3a5f3ca6 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 9 Jan 2026 23:43:36 +0200 Subject: [PATCH 05/10] rust: FileChunk's content is now Optional --- src-tauri/migrations/20251205113555_init_schema.sql | 2 +- src-tauri/src/database/chunks.rs | 2 +- src-tauri/src/database/models.rs | 4 ++-- src/lib/vecdir/bindings.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src-tauri/migrations/20251205113555_init_schema.sql b/src-tauri/migrations/20251205113555_init_schema.sql index 0f20a47..eabaa51 100644 --- a/src-tauri/migrations/20251205113555_init_schema.sql +++ b/src-tauri/migrations/20251205113555_init_schema.sql @@ -59,7 +59,7 @@ CREATE TABLE IF NOT EXISTS file_chunks ( file_id INTEGER NOT NULL, chunk_index INTEGER NOT NULL, -- index of chunk in file - content TEXT NOT NULL, -- text part + content TEXT, -- text part -- optional for syntax highlighting start_char_idx INTEGER, diff --git a/src-tauri/src/database/chunks.rs b/src-tauri/src/database/chunks.rs index d0a56e4..fcecff4 100644 --- a/src-tauri/src/database/chunks.rs +++ b/src-tauri/src/database/chunks.rs @@ -7,7 +7,7 @@ pub struct AddFileChunk { pub file_id: i32, pub chunk_index: i32, - pub content: String, + pub content: Option, pub start_char_idx: Option, pub end_char_idx: Option, diff --git a/src-tauri/src/database/models.rs b/src-tauri/src/database/models.rs index ec16401..0858e18 100644 --- a/src-tauri/src/database/models.rs +++ b/src-tauri/src/database/models.rs @@ -132,7 +132,7 @@ pub struct FileChunk { pub file_id: i32, pub chunk_index: i32, - pub content: String, + pub content: Option, pub start_char_idx: Option, pub end_char_idx: Option @@ -142,7 +142,7 @@ pub struct FileChunk { #[derive(Debug, FromRow, Serialize, Type)] pub struct VectorSearchResult { pub chunk_id: i32, - pub content: String, + pub content: Option, pub file_id: i32, pub absolute_path: String, diff --git a/src/lib/vecdir/bindings.ts b/src/lib/vecdir/bindings.ts index ecbcc06..fb652f2 100644 --- a/src/lib/vecdir/bindings.ts +++ b/src/lib/vecdir/bindings.ts @@ -144,7 +144,7 @@ export type LLMConfig = { api_base_url: string; api_key: string; model: string; export type Space = { id: number; name: string; description: string | null; embedding_config: EmbeddingConfig; llm_config: LLMConfig; created_at: string } export type StatusEvent = { status: StatusType; message: string | null; total: number | null; processed: number | null } export type StatusType = "Idle" | "Indexing" | "Processing" | "Notification" | "Error" -export type VectorSearchResult = { chunk_id: number; content: string; file_id: number; absolute_path: string; filename: string; distance: number } +export type VectorSearchResult = { chunk_id: number; content: string | null; file_id: number; absolute_path: string; filename: string; distance: number } /** tauri-specta globals **/ From 8178351d4f91e3f555e2b93e009d322431352b36 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 9 Jan 2026 23:50:54 +0200 Subject: [PATCH 06/10] rust: processor now supports multimodal embedding space processing --- src-tauri/src/indexer/processor.rs | 183 ++++++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 6 deletions(-) diff --git a/src-tauri/src/indexer/processor.rs b/src-tauri/src/indexer/processor.rs index 9154786..daa277f 100644 --- a/src-tauri/src/indexer/processor.rs +++ b/src-tauri/src/indexer/processor.rs @@ -6,7 +6,7 @@ use tauri::{AppHandle, Emitter}; use tauri_specta::Event; use crate::{ - ai::AI, + ai::{self, embedding, AI}, database::{ self, chunks::{add_chunk, add_chunks_batch, AddFileChunk}, @@ -14,7 +14,7 @@ use crate::{ get_pending_files_for_space, mark_file_as_indexed, mark_file_as_indexed_batch, MarkFileAsIndexed, }, - models::LLMConfig, + models::{EmbeddingConfig, LLMConfig, Space}, spaces::get_space_by_id, DbPool, }, @@ -59,12 +59,36 @@ pub async fn process_space( space_id: i32, limit: i32, ) -> Result<()> { - let mut descriptions: Vec = Vec::new(); - let mut processed_files: Vec = Vec::new(); - let space = get_space_by_id(pool, space_id) .await .context("failed to get space in process_space")?; + let embedding_config = &space.embedding_config.0; + + if embedding_config.multimodal { + process_space_multimodal_embedding(app_handle, pool, space, limit) + .await + .context("failed to process space using multimodal embedding") + .unwrap(); + } else { + process_space_llm(app_handle, pool, space, limit) + .await + .context("failed to process space using LLM description embedding") + .unwrap(); + } + + Ok(()) +} + +pub async fn process_space_llm( + app_handle: AppHandle, + pool: &DbPool, + space: Space, + limit: i32, +) -> Result<()> { + let mut descriptions: Vec = Vec::new(); + let mut processed_files: Vec = Vec::new(); + + let space_id = space.id; let llm_config = space.llm_config.0; let embedding_config = space.embedding_config.0; @@ -166,7 +190,7 @@ pub async fn process_space( file_id: file_id, chunk_index: 0, - content: content.clone(), + content: Some(content.clone()), start_char_idx: None, end_char_idx: None, @@ -220,3 +244,150 @@ pub async fn process_space( Ok(()) } + +pub async fn process_space_multimodal_embedding( + app_handle: AppHandle, + pool: &DbPool, + space: Space, + limit: i32, +) -> Result<()> { + let space_id = space.id; + + let embedding_config = space.embedding_config.0; + + let ai_client_emdedding = AI::new(&embedding_config.api_base_url, &embedding_config.api_key) + .context("failed to create openai client")?; + + let pending_files = get_pending_files_for_space(pool, space_id, limit) + .await + .context("failed to get pending indexed files")?; + + if pending_files.is_empty() { + return Ok(()); + } + + const BATCH_SIZE: usize = 50; + let total_files = pending_files.len() as i32; + + for (i, chunk) in pending_files.chunks(BATCH_SIZE).enumerate() { + let mut inputs: Vec = + Vec::new(); + + for file in chunk.iter() { + let file_path = file.absolute_path.clone(); + let guess = mime_guess::from_path(&file_path); + let mime_type = guess.first_or_octet_stream(); + + match mime_type.type_() { + mime::IMAGE => { + let image_url = ai_client_emdedding + .image_to_base64(&file_path) + .await + .context("failed to get image base64 url")?; + let input = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { + text: Some( + embedding_config + .image_processing_prompt + .system_prompt + .clone(), + ), + image_url: Some(image_url), + }; + + inputs.push(input); + } + _ => {} + }; + } + + let embeddings_response = ai_client_emdedding + .create_embeddings_batch_qwen3vl_llamacpp(inputs, embedding_config.model.clone()) + .await + .context("failed to create multimodal embeddings in batch during processing space")?; + + if embeddings_response.data.is_empty() { + continue; + } + + let mut chunks_to_add: Vec = Vec::new(); + let mut updates_to_add: Vec = Vec::new(); + + for embedding_item in embeddings_response.data { + let idx = embedding_item.index as usize; + + if idx >= chunk.len() { + continue; + } + + let file = &chunk[idx]; + let file_id = file.id; + + // TODO: make default dimension const + let embedding_result = ai_client_emdedding + .prepare_matroshka(embedding_item.embedding.clone(), 768) + .context("failed to prepare matroshka to 768 dim"); + + if embedding_result.is_err() { + println!("failed to create batch embedding {:?}", embedding_result); + continue; + } + + let embedding = embedding_result.unwrap(); + + let file_chunk = AddFileChunk { + file_id: file_id, + + chunk_index: 0, + content: None, + + start_char_idx: None, + end_char_idx: None, + + embedding: embedding, + }; + + chunks_to_add.push(file_chunk); + + let update: MarkFileAsIndexed = MarkFileAsIndexed { + file_id: file_id, + description: None, + }; + + updates_to_add.push(update); + } + + add_chunks_batch(pool, chunks_to_add) + .await + .context("failed to add chunks in batch in process_space")?; + + mark_file_as_indexed_batch(pool, updates_to_add) + .await + .context("failed to update file indexing status in batch in process_space")?; + + StatusEvent { + status: StatusType::Processing, + message: Some("Processing Space".to_string()), + total: Some(total_files), + processed: Some(i as i32), + } + .emit(&app_handle)?; + } + + StatusEvent { + status: StatusType::Idle, + message: None, + total: None, + processed: None, + } + .emit(&app_handle)?; + + StatusEvent { + status: StatusType::Notification, + message: Some("Space Processed".to_string()), + total: None, + processed: None, + } + .emit(&app_handle)?; + + Ok(()) +} From 97418912367c5b1a4c395ef6129a158e78c439b5 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Sat, 10 Jan 2026 00:05:50 +0200 Subject: [PATCH 07/10] rust: ai create_embedding_qwen3vl_llamacpp content initialization fix --- src-tauri/src/ai.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/src/ai.rs b/src-tauri/src/ai.rs index 80c28e1..220d530 100644 --- a/src-tauri/src/ai.rs +++ b/src-tauri/src/ai.rs @@ -92,7 +92,7 @@ impl AI { input: ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput, model: String, ) -> Result { - let mut content = vec![json!({})]; + let mut content = Vec::new(); if let Some(text) = input.text { content.push(json!({ From e39e0f35002fc6c76c615696f6553e28f46dcfab Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Sat, 10 Jan 2026 00:13:39 +0200 Subject: [PATCH 08/10] rust: processor changes due to llamacpp server doesnt support batching multimodal embedding input yet --- src-tauri/src/indexer/processor.rs | 173 ++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 40 deletions(-) diff --git a/src-tauri/src/indexer/processor.rs b/src-tauri/src/indexer/processor.rs index daa277f..16abef8 100644 --- a/src-tauri/src/indexer/processor.rs +++ b/src-tauri/src/indexer/processor.rs @@ -266,45 +266,36 @@ pub async fn process_space_multimodal_embedding( return Ok(()); } - const BATCH_SIZE: usize = 50; let total_files = pending_files.len() as i32; + for (i, file) in pending_files.iter().enumerate() { + let file_path = file.absolute_path.clone(); + let guess = mime_guess::from_path(&file_path); + let mime_type = guess.first_or_octet_stream(); - for (i, chunk) in pending_files.chunks(BATCH_SIZE).enumerate() { - let mut inputs: Vec = - Vec::new(); - - for file in chunk.iter() { - let file_path = file.absolute_path.clone(); - let guess = mime_guess::from_path(&file_path); - let mime_type = guess.first_or_octet_stream(); - - match mime_type.type_() { - mime::IMAGE => { - let image_url = ai_client_emdedding + let input = match mime_type.type_() { + mime::IMAGE => ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { + text: Some( + embedding_config + .image_processing_prompt + .system_prompt + .clone(), + ), + image_url: Some( + ai_client_emdedding .image_to_base64(&file_path) .await - .context("failed to get image base64 url")?; - let input = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { - text: Some( - embedding_config - .image_processing_prompt - .system_prompt - .clone(), - ), - image_url: Some(image_url), - }; - - inputs.push(input); - } - _ => {} - }; - } - + .context("failed to get image base64 url")?, + ), + }, + _ => ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { + text: None, + image_url: None, + }, + }; let embeddings_response = ai_client_emdedding - .create_embeddings_batch_qwen3vl_llamacpp(inputs, embedding_config.model.clone()) + .create_embedding_qwen3vl_llamacpp(input, embedding_config.model.clone()) .await - .context("failed to create multimodal embeddings in batch during processing space")?; - + .context("failed to create multimodal embedding during processing space")?; if embeddings_response.data.is_empty() { continue; } @@ -313,13 +304,6 @@ pub async fn process_space_multimodal_embedding( let mut updates_to_add: Vec = Vec::new(); for embedding_item in embeddings_response.data { - let idx = embedding_item.index as usize; - - if idx >= chunk.len() { - continue; - } - - let file = &chunk[idx]; let file_id = file.id; // TODO: make default dimension const @@ -373,6 +357,115 @@ pub async fn process_space_multimodal_embedding( .emit(&app_handle)?; } + // LLama.cpp server implementation currently doesn't support batching + + // const BATCH_SIZE: usize = 50; + // let total_files = pending_files.len() as i32; + + // for (i, chunk) in pending_files.chunks(BATCH_SIZE).enumerate() { + // let mut inputs: Vec = + // Vec::new(); + + // for file in chunk.iter() { + // let file_path = file.absolute_path.clone(); + // let guess = mime_guess::from_path(&file_path); + // let mime_type = guess.first_or_octet_stream(); + + // match mime_type.type_() { + // mime::IMAGE => { + // let image_url = ai_client_emdedding + // .image_to_base64(&file_path) + // .await + // .context("failed to get image base64 url")?; + // let input = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { + // text: Some( + // embedding_config + // .image_processing_prompt + // .system_prompt + // .clone(), + // ), + // image_url: Some(image_url), + // }; + + // inputs.push(input); + // } + // _ => {} + // }; + // } + + // let embeddings_response = ai_client_emdedding + // .create_embeddings_batch_qwen3vl_llamacpp(inputs, embedding_config.model.clone()) + // .await + // .context("failed to create multimodal embeddings in batch during processing space")?; + + // if embeddings_response.data.is_empty() { + // continue; + // } + + // let mut chunks_to_add: Vec = Vec::new(); + // let mut updates_to_add: Vec = Vec::new(); + + // for embedding_item in embeddings_response.data { + // let idx = embedding_item.index as usize; + + // if idx >= chunk.len() { + // continue; + // } + + // let file = &chunk[idx]; + // let file_id = file.id; + + // // TODO: make default dimension const + // let embedding_result = ai_client_emdedding + // .prepare_matroshka(embedding_item.embedding.clone(), 768) + // .context("failed to prepare matroshka to 768 dim"); + + // if embedding_result.is_err() { + // println!("failed to create batch embedding {:?}", embedding_result); + // continue; + // } + + // let embedding = embedding_result.unwrap(); + + // let file_chunk = AddFileChunk { + // file_id: file_id, + + // chunk_index: 0, + // content: None, + + // start_char_idx: None, + // end_char_idx: None, + + // embedding: embedding, + // }; + + // chunks_to_add.push(file_chunk); + + // let update: MarkFileAsIndexed = MarkFileAsIndexed { + // file_id: file_id, + // description: None, + // }; + + // updates_to_add.push(update); + // } + + // add_chunks_batch(pool, chunks_to_add) + // .await + // .context("failed to add chunks in batch in process_space")?; + + // mark_file_as_indexed_batch(pool, updates_to_add) + // .await + // .context("failed to update file indexing status in batch in process_space")?; + + // StatusEvent { + // status: StatusType::Processing, + // message: Some("Processing Space".to_string()), + // total: Some(total_files), + // processed: Some(i as i32), + // } + // .emit(&app_handle)?; + // } + StatusEvent { status: StatusType::Idle, message: None, From 94a6623e0f096d885a2796e467fe964e048bb468 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:01:42 +0200 Subject: [PATCH 09/10] add vecBox support --- src-tauri/src/ai.rs | 83 ++------------ src-tauri/src/ai/embedding.rs | 2 +- src-tauri/src/database/models.rs | 38 +++++-- src-tauri/src/indexer/processor.rs | 176 ++++++----------------------- src/lib/vecdir/bindings.ts | 3 +- 5 files changed, 74 insertions(+), 228 deletions(-) diff --git a/src-tauri/src/ai.rs b/src-tauri/src/ai.rs index 220d530..d763467 100644 --- a/src-tauri/src/ai.rs +++ b/src-tauri/src/ai.rs @@ -16,7 +16,6 @@ use async_openai::{ Client, }; use base64::Engine; -use serde_json::json; use crate::ai::{self}; @@ -81,82 +80,17 @@ impl AI { Ok(response) } - // the expected structure of qwen3vl embedding multimodal request in llamacpp - // accroding to this PR: https://github.com/ggml-org/llama.cpp/pull/18665 - // and exact this commit: https://github.com/ggml-org/llama.cpp/pull/18665/commits/56a0d87bd022ab017484523beac26cc3a946c5a4 - - // struct ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput is currently typed in this manner to prevent accidentally usage - - pub async fn create_embedding_qwen3vl_llamacpp( + pub async fn create_embedding_vecbox( &self, - input: ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput, + input: ai::embedding::vecbox::VecBoxEmbeddingInput, model: String, ) -> Result { - let mut content = Vec::new(); - - if let Some(text) = input.text { - content.push(json!({ - "type": "text", - "text": text, - })); - } - - if let Some(url) = input.image_url { - content.push(json!({ - "type": "image_url", - "image_url": { "url": url }, - })); - } - - let request = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingRequest { - model: model, - input: json!(content), - encoding_format: None, - }; - - let response: CreateEmbeddingResponse = self - .client - .embeddings() - .create_byot(&request) - .await - .context("failed to generate multimodal embedding using qwen3vl and llamacpp") - .unwrap(); - - Ok(response) - } + let content_parts = input.to_content_parts(); - pub async fn create_embeddings_batch_qwen3vl_llamacpp( - &self, - inputs: Vec, - model: String, - ) -> Result { - let content_array: Vec = inputs - .into_iter() - .map(|input| { - let mut content = Vec::new(); - - if let Some(text) = input.text { - content.push(json!({ - "type": "text", - "text": text, - })); - } - - if let Some(url) = input.image_url { - content.push(json!({ - "type": "image_url", - "image_url": { "url": url }, - })); - } - - json!(content) - }) - .collect(); - - let request = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingRequest { - model: model, - input: json!(content_array), - encoding_format: None, + let request = ai::embedding::vecbox::VecBoxEmbeddingRequest { + model: Some(model), + input: content_parts, + instruction: input.instruction, }; let response: CreateEmbeddingResponse = self @@ -164,8 +98,7 @@ impl AI { .embeddings() .create_byot(&request) .await - .context("failed to generate batch of multimodal embedding using qwen3vl and llamacpp") - .unwrap(); + .context("failed to generate vecBox embedding")?; Ok(response) } diff --git a/src-tauri/src/ai/embedding.rs b/src-tauri/src/ai/embedding.rs index 5f99a38..c5dbdd0 100644 --- a/src-tauri/src/ai/embedding.rs +++ b/src-tauri/src/ai/embedding.rs @@ -1 +1 @@ -pub mod multimodal_llamacpp; \ No newline at end of file +pub mod vecbox; diff --git a/src-tauri/src/database/models.rs b/src-tauri/src/database/models.rs index 0858e18..d95cda4 100644 --- a/src-tauri/src/database/models.rs +++ b/src-tauri/src/database/models.rs @@ -1,9 +1,20 @@ use specta::Type; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; use sqlx::types::Json; +use sqlx::FromRow; + +// EMBEDDING BACKEND TYPE + +#[derive(Debug, Serialize, Deserialize, Clone, Type, PartialEq, Default)] +pub enum EmbeddingBackendType { + #[default] + #[serde(rename = "openai_compat")] + OpenAICompat, + #[serde(rename = "vecbox")] + VecBox, +} // APP CONFIG #[derive(Debug, Serialize, Deserialize, Clone, Type)] @@ -16,21 +27,25 @@ pub struct AppConfig { #[serde(default = "default_parallelism")] #[specta(type = i32)] pub indexer_parallelism: usize, // How many files vecDir needs to index in parallel (2-4) - + // AI Settings (Global defaults) pub default_openai_url: Option, } // Default values -fn default_theme() -> String { "system".to_string() } -fn default_parallelism() -> usize { 2 } +fn default_theme() -> String { + "system".to_string() +} +fn default_parallelism() -> usize { + 2 +} impl Default for AppConfig { fn default() -> Self { Self { theme: default_theme(), indexer_parallelism: default_parallelism(), - default_openai_url: None + default_openai_url: None, } } } @@ -49,7 +64,7 @@ pub struct LLMConfig { pub api_key: String, pub model: String, - + pub text_processing_prompt: AIPrompt, pub image_processing_prompt: AIPrompt, pub default_processing_prompt: AIPrompt, @@ -57,6 +72,9 @@ pub struct LLMConfig { #[derive(Debug, Serialize, Deserialize, Clone, Type)] pub struct EmbeddingConfig { + #[serde(default)] + pub backend_type: EmbeddingBackendType, + pub api_base_url: String, pub api_key: String, @@ -85,7 +103,7 @@ pub struct Space { #[specta(type = EmbeddingConfig)] pub embedding_config: Json, - #[specta(type = LLMConfig)] + #[specta(type = LLMConfig)] pub llm_config: Json, pub created_at: DateTime, @@ -120,7 +138,7 @@ pub struct FileMetadata { pub modified_at_fs: DateTime, pub last_indexed_at: Option>, pub content_hash: Option, - + pub indexing_status: String, pub indexing_error_message: Option, } @@ -135,7 +153,7 @@ pub struct FileChunk { pub content: Option, pub start_char_idx: Option, - pub end_char_idx: Option + pub end_char_idx: Option, } // VECTOR SEARCH RESULT diff --git a/src-tauri/src/indexer/processor.rs b/src-tauri/src/indexer/processor.rs index 16abef8..967b4c2 100644 --- a/src-tauri/src/indexer/processor.rs +++ b/src-tauri/src/indexer/processor.rs @@ -6,22 +6,18 @@ use tauri::{AppHandle, Emitter}; use tauri_specta::Event; use crate::{ - ai::{self, embedding, AI}, + ai::{self, AI}, database::{ - self, - chunks::{add_chunk, add_chunks_batch, AddFileChunk}, + chunks::{add_chunks_batch, AddFileChunk}, files::{ get_pending_files_for_space, mark_file_as_indexed, mark_file_as_indexed_batch, MarkFileAsIndexed, }, - models::{EmbeddingConfig, LLMConfig, Space}, + models::{EmbeddingBackendType, EmbeddingConfig, LLMConfig, Space}, spaces::get_space_by_id, DbPool, }, - status::{ - self, - events::{StatusEvent, StatusType}, - }, + status::events::{StatusEvent, StatusType}, }; async fn process_image( @@ -64,16 +60,17 @@ pub async fn process_space( .context("failed to get space in process_space")?; let embedding_config = &space.embedding_config.0; - if embedding_config.multimodal { - process_space_multimodal_embedding(app_handle, pool, space, limit) - .await - .context("failed to process space using multimodal embedding") - .unwrap(); - } else { - process_space_llm(app_handle, pool, space, limit) - .await - .context("failed to process space using LLM description embedding") - .unwrap(); + match embedding_config.backend_type { + EmbeddingBackendType::VecBox if embedding_config.multimodal => { + process_space_vecbox_multimodal(app_handle, pool, space, limit) + .await + .context("failed to process space using vecBox multimodal embedding")?; + } + EmbeddingBackendType::OpenAICompat | EmbeddingBackendType::VecBox => { + process_space_llm(app_handle, pool, space, limit) + .await + .context("failed to process space using LLM description embedding")?; + } } Ok(()) @@ -245,7 +242,7 @@ pub async fn process_space_llm( Ok(()) } -pub async fn process_space_multimodal_embedding( +pub async fn process_space_vecbox_multimodal( app_handle: AppHandle, pool: &DbPool, space: Space, @@ -255,7 +252,7 @@ pub async fn process_space_multimodal_embedding( let embedding_config = space.embedding_config.0; - let ai_client_emdedding = AI::new(&embedding_config.api_base_url, &embedding_config.api_key) + let ai_client_embedding = AI::new(&embedding_config.api_base_url, &embedding_config.api_key) .context("failed to create openai client")?; let pending_files = get_pending_files_for_space(pool, space_id, limit) @@ -273,29 +270,36 @@ pub async fn process_space_multimodal_embedding( let mime_type = guess.first_or_octet_stream(); let input = match mime_type.type_() { - mime::IMAGE => ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { - text: Some( + mime::IMAGE => ai::embedding::vecbox::VecBoxEmbeddingInput { + instruction: Some( embedding_config .image_processing_prompt .system_prompt .clone(), ), + text: Some( + embedding_config + .image_processing_prompt + .user_prompt + .clone(), + ), image_url: Some( - ai_client_emdedding + ai_client_embedding .image_to_base64(&file_path) .await .context("failed to get image base64 url")?, ), }, - _ => ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { - text: None, - image_url: None, - }, + _ => { + continue; + } }; - let embeddings_response = ai_client_emdedding - .create_embedding_qwen3vl_llamacpp(input, embedding_config.model.clone()) + + let embeddings_response = ai_client_embedding + .create_embedding_vecbox(input, embedding_config.model.clone()) .await - .context("failed to create multimodal embedding during processing space")?; + .context("failed to create vecBox embedding during processing space")?; + if embeddings_response.data.is_empty() { continue; } @@ -306,8 +310,7 @@ pub async fn process_space_multimodal_embedding( for embedding_item in embeddings_response.data { let file_id = file.id; - // TODO: make default dimension const - let embedding_result = ai_client_emdedding + let embedding_result = ai_client_embedding .prepare_matroshka(embedding_item.embedding.clone(), 768) .context("failed to prepare matroshka to 768 dim"); @@ -357,115 +360,6 @@ pub async fn process_space_multimodal_embedding( .emit(&app_handle)?; } - // LLama.cpp server implementation currently doesn't support batching - - // const BATCH_SIZE: usize = 50; - // let total_files = pending_files.len() as i32; - - // for (i, chunk) in pending_files.chunks(BATCH_SIZE).enumerate() { - // let mut inputs: Vec = - // Vec::new(); - - // for file in chunk.iter() { - // let file_path = file.absolute_path.clone(); - // let guess = mime_guess::from_path(&file_path); - // let mime_type = guess.first_or_octet_stream(); - - // match mime_type.type_() { - // mime::IMAGE => { - // let image_url = ai_client_emdedding - // .image_to_base64(&file_path) - // .await - // .context("failed to get image base64 url")?; - // let input = ai::embedding::multimodal_llamacpp::MultimodalEmbeddingInput { - // text: Some( - // embedding_config - // .image_processing_prompt - // .system_prompt - // .clone(), - // ), - // image_url: Some(image_url), - // }; - - // inputs.push(input); - // } - // _ => {} - // }; - // } - - // let embeddings_response = ai_client_emdedding - // .create_embeddings_batch_qwen3vl_llamacpp(inputs, embedding_config.model.clone()) - // .await - // .context("failed to create multimodal embeddings in batch during processing space")?; - - // if embeddings_response.data.is_empty() { - // continue; - // } - - // let mut chunks_to_add: Vec = Vec::new(); - // let mut updates_to_add: Vec = Vec::new(); - - // for embedding_item in embeddings_response.data { - // let idx = embedding_item.index as usize; - - // if idx >= chunk.len() { - // continue; - // } - - // let file = &chunk[idx]; - // let file_id = file.id; - - // // TODO: make default dimension const - // let embedding_result = ai_client_emdedding - // .prepare_matroshka(embedding_item.embedding.clone(), 768) - // .context("failed to prepare matroshka to 768 dim"); - - // if embedding_result.is_err() { - // println!("failed to create batch embedding {:?}", embedding_result); - // continue; - // } - - // let embedding = embedding_result.unwrap(); - - // let file_chunk = AddFileChunk { - // file_id: file_id, - - // chunk_index: 0, - // content: None, - - // start_char_idx: None, - // end_char_idx: None, - - // embedding: embedding, - // }; - - // chunks_to_add.push(file_chunk); - - // let update: MarkFileAsIndexed = MarkFileAsIndexed { - // file_id: file_id, - // description: None, - // }; - - // updates_to_add.push(update); - // } - - // add_chunks_batch(pool, chunks_to_add) - // .await - // .context("failed to add chunks in batch in process_space")?; - - // mark_file_as_indexed_batch(pool, updates_to_add) - // .await - // .context("failed to update file indexing status in batch in process_space")?; - - // StatusEvent { - // status: StatusType::Processing, - // message: Some("Processing Space".to_string()), - // total: Some(total_files), - // processed: Some(i as i32), - // } - // .emit(&app_handle)?; - // } - StatusEvent { status: StatusType::Idle, message: None, diff --git a/src/lib/vecdir/bindings.ts b/src/lib/vecdir/bindings.ts index fb652f2..f441a0c 100644 --- a/src/lib/vecdir/bindings.ts +++ b/src/lib/vecdir/bindings.ts @@ -136,7 +136,8 @@ statusEvent: "status-event" export type AIPrompt = { system_prompt: string; user_prompt: string } export type AppConfig = { theme?: string; indexer_parallelism?: number; default_openai_url: string | null } export type BackendReadyEvent = null -export type EmbeddingConfig = { api_base_url: string; api_key: string; model: string; dimensions: number; text_processing_prompt: AIPrompt; image_processing_prompt: AIPrompt; default_processing_prompt: AIPrompt; search_prompt: AIPrompt; multimodal: boolean } +export type EmbeddingBackendType = "openai_compat" | "vecbox" +export type EmbeddingConfig = { backend_type?: EmbeddingBackendType; api_base_url: string; api_key: string; model: string; dimensions: number; text_processing_prompt: AIPrompt; image_processing_prompt: AIPrompt; default_processing_prompt: AIPrompt; search_prompt: AIPrompt; multimodal: boolean } export type ErrorEvent = { message: string; context: string | null } export type FileMetadata = { id: number; root_id: number; absolute_path: string; filename: string; file_extension: string; file_size: number; description: string | null; modified_at_fs: string; last_indexed_at: string | null; content_hash: string | null; indexing_status: string; indexing_error_message: string | null } export type IndexedRoot = { id: number; space_id: number; path: string; status: string } From d964689dc9d7612e05d832711b04e73eb4008316 Mon Sep 17 00:00:00 2001 From: alpaim <156456776+alpaim@users.noreply.github.com> Date: Fri, 13 Mar 2026 16:11:03 +0200 Subject: [PATCH 10/10] vecBox module --- .../src/ai/embedding/multimodal_llamacpp.rs | 14 ------ src-tauri/src/ai/embedding/vecbox.rs | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+), 14 deletions(-) delete mode 100644 src-tauri/src/ai/embedding/multimodal_llamacpp.rs create mode 100644 src-tauri/src/ai/embedding/vecbox.rs diff --git a/src-tauri/src/ai/embedding/multimodal_llamacpp.rs b/src-tauri/src/ai/embedding/multimodal_llamacpp.rs deleted file mode 100644 index 3c44cc4..0000000 --- a/src-tauri/src/ai/embedding/multimodal_llamacpp.rs +++ /dev/null @@ -1,14 +0,0 @@ -use serde::Serialize; - -#[derive(Debug, Serialize)] -pub struct MultimodalEmbeddingRequest { - pub model: String, - pub input: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub encoding_format: Option, -} - -pub struct MultimodalEmbeddingInput { - pub text: Option, - pub image_url: Option, -} \ No newline at end of file diff --git a/src-tauri/src/ai/embedding/vecbox.rs b/src-tauri/src/ai/embedding/vecbox.rs new file mode 100644 index 0000000..fd7a9fa --- /dev/null +++ b/src-tauri/src/ai/embedding/vecbox.rs @@ -0,0 +1,46 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EmbeddingContentPart { + Text { text: String }, + ImageUrl { image_url: EmbeddingImageUrl }, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct EmbeddingImageUrl { + pub url: String, +} + +#[derive(Debug, Serialize)] +pub struct VecBoxEmbeddingRequest { + pub model: Option, + pub input: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub instruction: Option, +} + +#[derive(Debug, Clone)] +pub struct VecBoxEmbeddingInput { + pub text: Option, + pub instruction: Option, + pub image_url: Option, +} + +impl VecBoxEmbeddingInput { + pub fn to_content_parts(&self) -> Vec { + let mut parts = Vec::new(); + + if let Some(ref text) = self.text { + parts.push(EmbeddingContentPart::Text { text: text.clone() }); + } + + if let Some(ref url) = self.image_url { + parts.push(EmbeddingContentPart::ImageUrl { + image_url: EmbeddingImageUrl { url: url.clone() }, + }); + } + + parts + } +}