From f7312bf240498110c84afa7aeb4f97368d00b031 Mon Sep 17 00:00:00 2001 From: "Jansen@home" Date: Fri, 27 Mar 2026 19:48:36 +0800 Subject: [PATCH 1/4] feat: separate internal and request dimensions for embeddings --- index.ts | 8 ++++++ openclaw.plugin.json | 5 ++++ src/embedder.ts | 7 ++++- test/plugin-manifest-regression.mjs | 44 ++++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/index.ts b/index.ts index b9d9ee68..95235e69 100644 --- a/index.ts +++ b/index.ts @@ -84,7 +84,10 @@ interface PluginConfig { apiKey: string | string[]; model?: string; baseURL?: string; + /** Internal schema/validation dimension (LanceDB + local checks). */ dimensions?: number; + /** Optional provider request dimension (dimensions/output_dimension). */ + requestDimensions?: number; omitDimensions?: boolean; taskQuery?: string; taskPassage?: string; @@ -1639,7 +1642,10 @@ const memoryLanceDBProPlugin = { apiKey: config.embedding.apiKey, model: config.embedding.model || "text-embedding-3-small", baseURL: config.embedding.baseURL, + // Internal dimension for local schema/validation checks. dimensions: config.embedding.dimensions, + // Optional request hint sent to providers that support variable dimensions. + requestDimensions: config.embedding.requestDimensions, omitDimensions: config.embedding.omitDimensions, taskQuery: config.embedding.taskQuery, taskPassage: config.embedding.taskPassage, @@ -3776,6 +3782,8 @@ export function parsePluginConfig(value: unknown): PluginConfig { // Accept number, numeric string, or env-var string (e.g. "${EMBED_DIM}"). // Also accept legacy top-level `dimensions` for convenience. dimensions: parsePositiveInt(embedding.dimensions ?? cfg.dimensions), + // Request dimension is intentionally separate from internal schema sizing. + requestDimensions: parsePositiveInt(embedding.requestDimensions), omitDimensions: typeof embedding.omitDimensions === "boolean" ? embedding.omitDimensions diff --git a/openclaw.plugin.json b/openclaw.plugin.json index a2cfb1f5..f3df07c2 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -46,6 +46,11 @@ "type": "integer", "minimum": 1 }, + "requestDimensions": { + "type": "integer", + "minimum": 1, + "description": "Optional output dimension sent to embedding API requests only (for providers supporting variable dimensions)" + }, "omitDimensions": { "type": "boolean", "description": "When true, omit the dimensions parameter from embedding requests even if dimensions is configured" diff --git a/src/embedder.ts b/src/embedder.ts index bcbbaa76..99745173 100644 --- a/src/embedder.ts +++ b/src/embedder.ts @@ -90,7 +90,10 @@ export interface EmbeddingConfig { apiKey: string | string[]; model: string; baseURL?: string; + /** Internal vector dimension for schema/validation. This does NOT imply sending API dimensions. */ dimensions?: number; + /** Optional API request output dimension for providers that support variable dimensions. */ + requestDimensions?: number; /** Optional task type for query embeddings (e.g. "retrieval.query") */ taskQuery?: string; @@ -428,7 +431,8 @@ export class Embedder { this._taskQuery = config.taskQuery; this._taskPassage = config.taskPassage; this._normalized = config.normalized; - this._requestDimensions = config.dimensions; + // Request-side dimension hint is isolated from internal schema dimension. + this._requestDimensions = config.requestDimensions; this._omitDimensions = config.omitDimensions === true; // Enable auto-chunking by default for better handling of long documents this._autoChunk = config.chunking !== false; @@ -472,6 +476,7 @@ export class Embedder { console.log(`[memory-lancedb-pro] Initialized ${this.clients.length} API keys for round-robin rotation`); } + // Internal dimension remains the single source of truth for local validation. this.dimensions = getVectorDimensions(config.model, config.dimensions); this._cache = new EmbeddingCache(256, 30); // 256 entries, 30 min TTL } diff --git a/test/plugin-manifest-regression.mjs b/test/plugin-manifest-regression.mjs index 65e9ec23..461d2e76 100644 --- a/test/plugin-manifest-regression.mjs +++ b/test/plugin-manifest-regression.mjs @@ -109,6 +109,11 @@ assert.equal( "boolean", "embedding.omitDimensions should be declared in the plugin schema", ); +assert.equal( + manifest.configSchema.properties.embedding.properties.requestDimensions?.type, + "integer", + "embedding.requestDimensions should be declared in the plugin schema", +); assert.equal( manifest.configSchema.properties.sessionMemory.properties.enabled.default, false, @@ -325,14 +330,44 @@ try { }); const requestCountBeforeWithDimensions = embeddingRequests.length; await withDimensionsTool.execute("tool-3", { - text: "dimensions should be sent by default", + text: "dimensions should not be sent by default", scope: "global", }); const withDimensionsRequest = embeddingRequests.at(requestCountBeforeWithDimensions); assert.equal( - withDimensionsRequest?.dimensions, + Object.prototype.hasOwnProperty.call(withDimensionsRequest ?? {}, "dimensions"), + false, + "embedding.dimensions should be used for internal schema sizing, not forwarded by default", + ); + + const withRequestDimensionsApi = createMockApi({ + dbPath: path.join(workDir, "db-with-request-dimensions"), + autoCapture: false, + autoRecall: false, + embedding: { + provider: "openai-compatible", + apiKey: "dummy", + model: "text-embedding-3-small", + baseURL: embeddingBaseURL, + dimensions: 4, + requestDimensions: 4, + }, + }); + plugin.register(withRequestDimensionsApi); + const withRequestDimensionsTool = withRequestDimensionsApi.toolFactories.memory_store({ + agentId: "main", + sessionKey: "agent:main:test", + }); + const requestCountBeforeRequestDimensions = embeddingRequests.length; + await withRequestDimensionsTool.execute("tool-3b", { + text: "requestDimensions should be forwarded", + scope: "global", + }); + const withRequestDimensionsRequest = embeddingRequests.at(requestCountBeforeRequestDimensions); + assert.equal( + withRequestDimensionsRequest?.dimensions, 4, - "embedding.dimensions should be forwarded by default", + "embedding.requestDimensions should be forwarded to embedding requests", ); const omitDimensionsApi = createMockApi({ @@ -345,6 +380,7 @@ try { model: "text-embedding-3-small", baseURL: embeddingBaseURL, dimensions: 4, + requestDimensions: 4, omitDimensions: true, }, }); @@ -362,7 +398,7 @@ try { assert.equal( Object.prototype.hasOwnProperty.call(omitDimensionsRequest, "dimensions"), false, - "embedding.omitDimensions=true should omit dimensions from embedding requests", + "embedding.omitDimensions=true should omit dimensions from embedding requests even when requestDimensions is set", ); } finally { await new Promise((resolve) => embeddingServer.close(resolve)); From fe65f9341b16b35e8e6587faf430313b045e1508 Mon Sep 17 00:00:00 2001 From: "Jansen@home" Date: Sun, 29 Mar 2026 19:54:21 +0800 Subject: [PATCH 2/4] test: align embedder dimension assertions with requestDimensions --- test/embedder-error-hints.test.mjs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/test/embedder-error-hints.test.mjs b/test/embedder-error-hints.test.mjs index 38db8ae1..767771ff 100644 --- a/test/embedder-error-hints.test.mjs +++ b/test/embedder-error-hints.test.mjs @@ -130,7 +130,7 @@ async function run() { installMockEmbeddingClient(jinaEmbedder, async (payload) => { assert.equal(payload.task, "retrieval.passage"); assert.equal(payload.normalized, true); - assert.equal(payload.dimensions, 1024); + assert.equal(payload.dimensions, undefined, "jina should not send dimensions unless requestDimensions is set"); return createEmbeddingResponse(1024); }); await jinaEmbedder.embedPassage("hello"); @@ -144,7 +144,7 @@ async function run() { }); installMockEmbeddingClient(genericEmbedder, async (payload) => { assert.equal(payload.encoding_format, "float"); - assert.equal(payload.dimensions, 384); + assert.equal(payload.dimensions, undefined, "generic profile should not send dimensions unless requestDimensions is set"); return createEmbeddingResponse(384); }); await genericEmbedder.embedPassage("hello"); @@ -189,7 +189,7 @@ async function run() { }); await voyageTaskEmbedder.embedQuery("hello"); - // Voyage: configured dimensions should be sent as output_dimension, not dimensions. + // Voyage: requestDimensions should be sent as output_dimension, not dimensions. // voyage-4-lite is a recommended Voyage model that supports output_dimension. const voyageDimEmbedder = new Embedder({ provider: "openai-compatible", @@ -197,6 +197,7 @@ async function run() { model: "voyage-4-lite", baseURL: "https://api.voyageai.com/v1", dimensions: 512, + requestDimensions: 512, }); installMockEmbeddingClient(voyageDimEmbedder, async (payload) => { assert.equal(payload.output_dimension, 512, "voyage should send output_dimension"); @@ -211,7 +212,7 @@ async function run() { await withEmbeddingCaptureServer( (payload) => { assert.equal(payload.encoding_format, "float", "generic profile should send encoding_format"); - assert.equal(payload.dimensions, 384, "generic profile should send dimensions"); + assert.equal(payload.dimensions, undefined, "generic profile should not send dimensions by default"); assert.equal(payload.task, undefined, "generic profile should not send task"); assert.equal(payload.normalized, undefined, "generic profile should not send normalized"); return { body: createEmbeddingResponse(384) }; @@ -228,6 +229,25 @@ async function run() { }, ); + await withEmbeddingCaptureServer( + (payload) => { + assert.equal(payload.encoding_format, "float", "generic profile should send encoding_format"); + assert.equal(payload.dimensions, 384, "generic profile should send dimensions when requestDimensions is set"); + return { body: createEmbeddingResponse(384) }; + }, + async ({ baseURL }) => { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "custom-embed-model", + baseURL, + dimensions: 384, + requestDimensions: 384, + }); + await embedder.embedPassage("hello world"); + }, + ); + await withJsonServer( 403, { error: { message: "Invalid API key", code: "invalid_api_key" } }, From 82748fce267f9729fdbada987c9aa43cad6e9d4c Mon Sep 17 00:00:00 2001 From: "Jansen@home" Date: Sun, 29 Mar 2026 21:18:30 +0800 Subject: [PATCH 3/4] chore: add uiHints for embedding.requestDimensions --- openclaw.plugin.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openclaw.plugin.json b/openclaw.plugin.json index f3df07c2..da814d34 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -867,6 +867,12 @@ "help": "Override vector dimensions for custom models not in the built-in lookup table", "advanced": true }, + "embedding.requestDimensions": { + "label": "Request Dimensions", + "placeholder": "unset (do not send)", + "help": "Optional output dimension sent to the embedding API request only (dimensions/output_dimension depending on provider). If unset, no request dimension field is sent.", + "advanced": true + }, "embedding.omitDimensions": { "label": "Omit Request Dimensions", "help": "Do not send the dimensions parameter to the embedding API even if embedding.dimensions is configured. Useful for local models like Qwen3-Embedding that reject the field.", From f234f956b1cea959262d816e3a6b4b5f5b7f58ef Mon Sep 17 00:00:00 2001 From: "Jansen@home" Date: Mon, 30 Mar 2026 19:13:34 +0800 Subject: [PATCH 4/4] fix: internal validation uses requestDimensions if present (review feedback) --- src/embedder.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/embedder.ts b/src/embedder.ts index 99745173..4e609eb5 100644 --- a/src/embedder.ts +++ b/src/embedder.ts @@ -476,8 +476,12 @@ export class Embedder { console.log(`[memory-lancedb-pro] Initialized ${this.clients.length} API keys for round-robin rotation`); } - // Internal dimension remains the single source of truth for local validation. - this.dimensions = getVectorDimensions(config.model, config.dimensions); + // Internal dimension用于本地校验，优先使用requestDimensions（如有），否则fallback到dimensions。 + // 这样可变维度模型（如text-embedding-3-large + requestDimensions: 1024）本地校验与API一致。 + const effectiveDims = this._requestDimensions && this._requestDimensions > 0 + ? this._requestDimensions + : config.dimensions; + this.dimensions = getVectorDimensions(config.model, effectiveDims); this._cache = new EmbeddingCache(256, 30); // 256 entries, 30 min TTL }