feat(grok): update API fetch logic and response handling; improve pagination and data normalization

Author: vchan-in

src/extractors/grok.ts

@@ -1,32 +1,23 @@
 /* ──────────────────────────────────────────────
- *  Extractor – Grok (grok.com / x.com/i/grok)
+ *  Extractor – Grok (grok.com)
  *
- *  Uses X/Twitter infrastructure.
- *  Requires x-csrf-token from ct0 cookie.
+ *  Endpoint: /rest/app-chat/conversations
+ *  Auth: cookie-based (sso + sso-rw). No CSRF header required.
+ *  Pagination: pageSize + cursor (X/Twitter API style).
  * ────────────────────────────────────────────── */
 
 import type { Extractor } from "./base";
 import type { Message, Thread } from "../types/schema";
 import { uuid, jitteredDelay } from "../utils/helpers";
 
-const BASE = ""; // Rely on relative path since we inject into either grok.com or x.com
-
-// In browser context content-scripts, getting cookies dynamically is harder via chrome.cookies unless it's sent from BG.
-// However, since we are doing a same-origin fetch from the content-script, 
-// the browser handles `credentials` and cookies automatically for most cases. 
-// We still need the CSRF/ct0 header if required.
-function getCsrfFromDocument(): string {
-  const match = document.cookie.match(/(?:^|;\s*)ct0=([^;]*)/);
-  if (!match) throw new Error("Grok: ct0 CSRF cookie not found in document.cookie");
-  return match[1];
-}
+const BASE = "";
+const PAGE_SIZE = 60;
 
 async function apiFetch<T>(path: string): Promise<T> {
-  const csrf = getCsrfFromDocument();
   const res = await fetch(`${BASE}${path}`, {
+    credentials: "include",
     headers: {
       "Content-Type": "application/json",
-      "x-csrf-token": csrf,
     },
   });
   if (!res.ok) {
@@ -40,85 +31,132 @@ async function apiFetch<T>(path: string): Promise<T> {
 /* ── raw response shapes ──────────────────────────────────── */
 
 interface RawConvListItem {
-  id: string;
+  conversationId: string;
   title?: string;
-  created_at: string;
-  updated_at: string;
+  createTime?: string;
+  modifyTime?: string;
+  // fallback snake_case (older API versions)
+  id?: string;
+  created_at?: string;
+  updated_at?: string;
 }
 
+/** The list endpoint returns either a plain array or a wrapped object. */
+type RawConvListResponse =
+  | RawConvListItem[]
+  | { conversations: RawConvListItem[]; nextCursor?: string; cursor?: string };
+
 interface RawGrokMessage {
-  id: string;
-  role: "user" | "assistant";
-  text: string;
-  created_at: string;
+  id?: string;
+  messageId?: string;
+  sender?: "human" | "assistant";
+  role?: "user" | "assistant";
+  message?: string;
+  text?: string;
+  createTime?: string;
+  created_at?: string;
   model?: string;
 }
 
 interface RawConversation {
-  id: string;
+  conversationId?: string;
+  id?: string;
   title?: string;
-  created_at: string;
-  updated_at: string;
-  messages: RawGrokMessage[];
+  createTime?: string;
+  modifyTime?: string;
+  created_at?: string;
+  updated_at?: string;
+  responses?: RawGrokMessage[];
+  messages?: RawGrokMessage[];
 }
 
 /* ── helpers ──────────────────────────────────────────────── */
 
+function normaliseItem(item: RawConvListItem): { providerId: string; title: string; createdAt: string; updatedAt: string } {
+  return {
+    providerId: item.conversationId ?? item.id ?? "",
+    title: item.title || "Untitled",
+    createdAt: item.createTime ?? item.created_at ?? "",
+    updatedAt: item.modifyTime ?? item.updated_at ?? item.createTime ?? item.created_at ?? "",
+  };
+}
+
 function mapMessages(raw: RawGrokMessage[]): Message[] {
-  return raw.map((m) => ({
-    id: m.id,
-    role: m.role,
-    content: m.text,
-    createdAt: m.created_at,
-    model: m.model,
-  }));
+  return raw.map((m) => {
+    const role: "user" | "assistant" =
+      m.role === "user" || m.sender === "human" ? "user" : "assistant";
+    return {
+      id: m.messageId ?? m.id ?? uuid(),
+      role,
+      content: m.message ?? m.text ?? "",
+      createdAt: m.createTime ?? m.created_at ?? "",
+      model: m.model,
+    };
+  });
 }
 
 /* ── Extractor implementation ─────────────────────────────── */
 
 export const grokExtractor: Extractor = {
   async listConversations() {
     const all: { providerId: string; title: string; createdAt: string; updatedAt: string }[] = [];
-    let offset = 0;
-    const limit = 50;
+    let cursor: string | undefined;
 
     // eslint-disable-next-line no-constant-condition
     while (true) {
-      const list = await apiFetch<RawConvListItem[]>(
-        `/rest/api/conversations?limit=${limit}&offset=${offset}`
+      const qs = cursor
+        ? `pageSize=${PAGE_SIZE}&cursor=${encodeURIComponent(cursor)}`
+        : `pageSize=${PAGE_SIZE}`;
+
+      const raw = await apiFetch<RawConvListResponse>(
+        `/rest/app-chat/conversations?${qs}`
       );
-      if (!list.length) break;
-
-      for (const item of list) {
-        all.push({
-          providerId: item.id,
-          title: item.title || "Untitled",
-          createdAt: item.created_at,
-          updatedAt: item.updated_at,
-        });
+
+      let items: RawConvListItem[];
+      let nextCursor: string | undefined;
+
+      if (Array.isArray(raw)) {
+        items = raw;
+      } else {
+        items = raw.conversations ?? [];
+        nextCursor = raw.nextCursor ?? raw.cursor;
+      }
+
+      if (!items.length) break;
+
+      for (const item of items) {
+        all.push(normaliseItem(item));
       }
 
-      offset += limit;
-      if (list.length < limit) break;
-      await jitteredDelay(500, 1200);
+      if (nextCursor) {
+        cursor = nextCursor;
+        await jitteredDelay(500, 1200);
+      } else if (items.length < PAGE_SIZE) {
+        break;
+      } else {
+        // No cursor returned but page was full — stop to avoid infinite loop.
+        break;
+      }
     }
 
     return all;
   },
 
   async fetchThread(conversationId: string): Promise<Thread> {
     const raw = await apiFetch<RawConversation>(
-      `/rest/api/conversations/${conversationId}`
+      `/rest/app-chat/conversations/${conversationId}`
     );
 
+    const messages = mapMessages(raw.responses ?? raw.messages ?? []);
+
     return {
       id: uuid(),
       providerId: conversationId,
       provider: "grok",
       title: raw.title || "Untitled",
-      createdAt: raw.created_at,
-      updatedAt: raw.updated_at,
-      messages: mapMessages(raw.messages ?? []),
+      createdAt: raw.createTime ?? raw.created_at ?? "",
+      updatedAt: raw.modifyTime ?? raw.updated_at ?? raw.createTime ?? raw.created_at ?? "",
+      messages,
       tags: [],
     };
   },