fix: parse structured vectors.space responses

Read output_dimension, data[0].embedding, data[0].truncated, and nested usage token fields from the documented vectors.space response shape.

This makes the remote embedding parser match the current API envelope instead of relying on flat key scans across the whole JSON payload.

Author: andinux

src/dbmem-rembed.c

@@ -210,6 +210,55 @@ static int set_json_error_message (dbmem_remote_engine_t *engine) {
     return -1;
 }
 
+static int dbmem_json_skip_token (const jsmntok_t *tokens, int index) {
+    int next = index + 1;
+
+    if (tokens[index].type == JSMN_ARRAY) {
+        for (int i = 0; i < tokens[index].size; i++) {
+            next = dbmem_json_skip_token(tokens, next);
+        }
+        return next;
+    }
+
+    if (tokens[index].type == JSMN_OBJECT) {
+        for (int i = 0; i < tokens[index].size; i++) {
+            next += 1; // skip key token
+            next = dbmem_json_skip_token(tokens, next);
+        }
+        return next;
+    }
+
+    return next;
+}
+
+static bool dbmem_json_token_equals (const char *json, const jsmntok_t *token, const char *text) {
+    size_t len = strlen(text);
+    size_t token_len = (size_t)(token->end - token->start);
+    return token_len == len && memcmp(json + token->start, text, len) == 0;
+}
+
+static int dbmem_json_object_find (const char *json, const jsmntok_t *tokens, int object_index, const char *key) {
+    if (object_index < 0 || tokens[object_index].type != JSMN_OBJECT) return -1;
+
+    int index = object_index + 1;
+    for (int i = 0; i < tokens[object_index].size; i++) {
+        int key_index = index;
+        int value_index = key_index + 1;
+
+        if (tokens[key_index].type != JSMN_STRING) return -1;
+        if (dbmem_json_token_equals(json, &tokens[key_index], key)) return value_index;
+
+        index = dbmem_json_skip_token(tokens, value_index);
+    }
+
+    return -1;
+}
+
+static bool dbmem_json_parse_bool (const char *json, const jsmntok_t *token) {
+    size_t len = (size_t)(token->end - token->start);
+    return token->type == JSMN_PRIMITIVE && len == 4 && memcmp(json + token->start, "true", 4) == 0;
+}
+
 #if ENABLE_DBMEM_DEBUG_EMBEDDING
 static void dbmem_remote_debug_log_response(dbmem_remote_engine_t *engine, long http_code) {
     const char *response = engine->data ? engine->data : "";
@@ -498,29 +547,60 @@ int dbmem_remote_compute_embedding (dbmem_remote_engine_t *engine, const char *t
     int emb_start = -1;
     size_t emb_count = 0;
 
-    for (int i = 0; i < ntokens - 1; i++) {
-        if (tokens[i].type != JSMN_STRING) continue;
-        int klen = tokens[i].end - tokens[i].start;
-        const char *key = engine->data + tokens[i].start;
-
-        if (klen == 9 && memcmp(key, "embedding", 9) == 0 && tokens[i + 1].type == JSMN_ARRAY) {
-            if (tokens[i + 1].size <= 0) {
-                dbmem_context_set_error(engine->context, "Invalid embedding array size in API response");
-                return -1;
-            }
-            emb_count = (size_t)tokens[i + 1].size;
-            emb_start = i + 2;
-        } else if (klen == 16 && memcmp(key, "output_dimension", 16) == 0) {
-            n_embd = atoi(engine->data + tokens[i + 1].start);
-        } else if (klen == 13 && memcmp(key, "prompt_tokens", 13) == 0 && tokens[i + 1].type == JSMN_PRIMITIVE) {
-            prompt_tokens = atoi(engine->data + tokens[i + 1].start);
-        } else if (klen == 19 && memcmp(key, "exact_prompt_tokens", 19) == 0 && tokens[i + 1].type == JSMN_PRIMITIVE) {
-            exact_prompt_tokens = atoi(engine->data + tokens[i + 1].start);
-        } else if (klen == 23 && memcmp(key, "estimated_prompt_tokens", 23) == 0) {
-            estimated_prompt_tokens = atoi(engine->data + tokens[i + 1].start);
-        } else if (klen == 9 && memcmp(key, "truncated", 9) == 0 && tokens[i + 1].type == JSMN_PRIMITIVE) {
-            truncated = (tokens[i + 1].end - tokens[i + 1].start == 4) &&
-                        (memcmp(engine->data + tokens[i + 1].start, "true", 4) == 0);
+    if (tokens[0].type != JSMN_OBJECT) {
+        dbmem_context_set_error(engine->context, "Invalid API response shape");
+        return -1;
+    }
+
+    int output_dimension_index = dbmem_json_object_find(engine->data, tokens, 0, "output_dimension");
+    if (output_dimension_index >= 0 && tokens[output_dimension_index].type == JSMN_PRIMITIVE) {
+        n_embd = atoi(engine->data + tokens[output_dimension_index].start);
+    }
+
+    int data_index = dbmem_json_object_find(engine->data, tokens, 0, "data");
+    if (data_index < 0 || tokens[data_index].type != JSMN_ARRAY || tokens[data_index].size <= 0) {
+        dbmem_context_set_error(engine->context, "Missing embedding data in API response");
+        return -1;
+    }
+
+    int item_index = data_index + 1;
+    if (tokens[item_index].type != JSMN_OBJECT) {
+        dbmem_context_set_error(engine->context, "Invalid embedding item in API response");
+        return -1;
+    }
+
+    int embedding_index = dbmem_json_object_find(engine->data, tokens, item_index, "embedding");
+    if (embedding_index < 0 || tokens[embedding_index].type != JSMN_ARRAY) {
+        dbmem_context_set_error(engine->context, "Missing embedding data in API response");
+        return -1;
+    }
+    if (tokens[embedding_index].size <= 0) {
+        dbmem_context_set_error(engine->context, "Invalid embedding array size in API response");
+        return -1;
+    }
+    emb_count = (size_t)tokens[embedding_index].size;
+    emb_start = embedding_index + 1;
+
+    int truncated_index = dbmem_json_object_find(engine->data, tokens, item_index, "truncated");
+    if (truncated_index >= 0) {
+        truncated = dbmem_json_parse_bool(engine->data, &tokens[truncated_index]);
+    }
+
+    int usage_index = dbmem_json_object_find(engine->data, tokens, 0, "usage");
+    if (usage_index >= 0 && tokens[usage_index].type == JSMN_OBJECT) {
+        int prompt_tokens_index = dbmem_json_object_find(engine->data, tokens, usage_index, "prompt_tokens");
+        if (prompt_tokens_index >= 0 && tokens[prompt_tokens_index].type == JSMN_PRIMITIVE) {
+            prompt_tokens = atoi(engine->data + tokens[prompt_tokens_index].start);
+        }
+
+        int exact_prompt_tokens_index = dbmem_json_object_find(engine->data, tokens, usage_index, "exact_prompt_tokens");
+        if (exact_prompt_tokens_index >= 0 && tokens[exact_prompt_tokens_index].type == JSMN_PRIMITIVE) {
+            exact_prompt_tokens = atoi(engine->data + tokens[exact_prompt_tokens_index].start);
+        }
+
+        int estimated_prompt_tokens_index = dbmem_json_object_find(engine->data, tokens, usage_index, "estimated_prompt_tokens");
+        if (estimated_prompt_tokens_index >= 0 && tokens[estimated_prompt_tokens_index].type == JSMN_PRIMITIVE) {
+            estimated_prompt_tokens = atoi(engine->data + tokens[estimated_prompt_tokens_index].start);
         }
     }