@@ -388,6 +399,47 @@ function getDisplayImageCitations(
return imageCitations;
}
+function getDisplayImageArtifacts(
+ message: ChatMessageView,
+ sourceTitlesByDocumentId: Readonly
>,
+): readonly DisplayImageArtifact[] {
+ const seenAssetUrls = new Set();
+ const imageArtifacts: DisplayImageArtifact[] = [];
+
+ for (const [index, artifact] of (message.artifacts ?? []).entries()) {
+ if (artifact.display === false || artifact.type !== "image") continue;
+
+ const assetUrl = getTrimmedCitationField(artifact.assetUrl);
+ if (!assetUrl || seenAssetUrls.has(assetUrl)) continue;
+
+ seenAssetUrls.add(assetUrl);
+ imageArtifacts.push({
+ assetUrl,
+ citationId: `${message.id}:artifact:${index}`,
+ label: getArtifactLabel(artifact, sourceTitlesByDocumentId),
+ });
+ }
+
+ return imageArtifacts;
+}
+
+function getArtifactLabel(
+ artifact: ChatArtifactView,
+ sourceTitlesByDocumentId: Readonly>,
+): string {
+ const label = getTrimmedCitationField(artifact.label);
+ if (label) return label;
+
+ if (artifact.citation) {
+ return chatPanelModel.getCitationLabel(
+ artifact.citation,
+ sourceTitlesByDocumentId,
+ );
+ }
+
+ return getTrimmedCitationField(artifact.reason) ?? "Selected image";
+}
+
function isImageCitation(
citation: ChatCitationView,
assetUrl: string,
diff --git a/src/domains/chat/chat-citation-persistence.ts b/src/domains/chat/chat-citation-persistence.ts
index a705362..1b160f0 100644
--- a/src/domains/chat/chat-citation-persistence.ts
+++ b/src/domains/chat/chat-citation-persistence.ts
@@ -1,4 +1,5 @@
import type {
+ ChatArtifactView,
ChatCitationView,
CitationView,
RetrievalResultView,
@@ -11,6 +12,9 @@ type ChatCitationPersistence = {
| null
| undefined,
) => CitationView[] | null
+ readonly normalizeArtifacts: (
+ artifacts: readonly ChatArtifactView[] | null | undefined,
+ ) => ChatArtifactView[] | null
readonly replaceDemoCitationDocumentId: (
citations: readonly ChatCitationView[] | undefined,
documentIdMap: ReadonlyMap,
@@ -27,6 +31,27 @@ function normalizeCitations(
return citations.map(toCitationView)
}
+function normalizeArtifacts(
+ artifacts: readonly ChatArtifactView[] | null | undefined,
+): ChatArtifactView[] | null {
+ if (!artifacts || artifacts.length === 0) return null
+ return artifacts.map(toArtifactView)
+}
+
+function toArtifactView(artifact: ChatArtifactView): ChatArtifactView {
+ return {
+ type: artifact.type,
+ ref: artifact.ref,
+ assetUrl: artifact.assetUrl,
+ label: artifact.label,
+ display: artifact.display,
+ reason: artifact.reason,
+ citation: artifact.citation
+ ? toCitationView(artifact.citation)
+ : undefined,
+ }
+}
+
function replaceDemoCitationDocumentId(
citations: readonly ChatCitationView[] | undefined,
documentIdMap: ReadonlyMap,
@@ -67,5 +92,6 @@ function toCitationView(
export const chatCitationPersistence: ChatCitationPersistence = {
normalizeCitations,
+ normalizeArtifacts,
replaceDemoCitationDocumentId,
}
diff --git a/src/domains/chat/chat-message-repository.ts b/src/domains/chat/chat-message-repository.ts
index 4cc04f1..e0f2584 100644
--- a/src/domains/chat/chat-message-repository.ts
+++ b/src/domains/chat/chat-message-repository.ts
@@ -9,6 +9,7 @@ import { deriveChatThreadTitle } from "./title"
import { DbClient } from "@/infrastructure/db"
import { chatMessages, chatThreads, type ChatMessage } from "@/infrastructure/db/schema"
import type {
+ ChatArtifactView,
ChatCitationView,
CitationView,
RetrievalResultView,
@@ -21,6 +22,7 @@ type AppendChatMessageInput = {
readonly citations?:
| readonly (ChatCitationView | CitationView | RetrievalResultView)[]
| null
+ readonly artifacts?: readonly ChatArtifactView[] | null
}
type ChatMessageRepository = {
@@ -78,6 +80,9 @@ const appendMessageToThreadEffect: ChatMessageRepository["appendMessageToThreadE
citations: chatCitationPersistence.normalizeCitations(
input.citations,
),
+ artifacts: chatCitationPersistence.normalizeArtifacts(
+ input.artifacts,
+ ),
})
.returning()
diff --git a/src/domains/chat/chat-turn-persistence.test.ts b/src/domains/chat/chat-turn-persistence.test.ts
index de8b3b9..3dbb85c 100644
--- a/src/domains/chat/chat-turn-persistence.test.ts
+++ b/src/domains/chat/chat-turn-persistence.test.ts
@@ -61,6 +61,7 @@ function makeMessage(id: string): ChatMessage {
role: "user",
content: "Question",
citations: null,
+ artifacts: null,
createdAt: new Date("2026-05-10T00:00:00.000Z"),
};
}
diff --git a/src/domains/chat/chat-turn-persistence.ts b/src/domains/chat/chat-turn-persistence.ts
index fbb759c..7d70b78 100644
--- a/src/domains/chat/chat-turn-persistence.ts
+++ b/src/domains/chat/chat-turn-persistence.ts
@@ -4,6 +4,7 @@ import { chatThreadService } from "./thread-service"
import type { ChatRepository } from "./service"
import type { ChatMessage, ChatThread } from "@/infrastructure/db/schema"
import type {
+ ChatArtifactView,
ChatCitationView,
CitationView,
RetrievalResultView,
@@ -16,6 +17,7 @@ type AppendMessageInput = {
readonly citations?:
| readonly (ChatCitationView | CitationView | RetrievalResultView)[]
| null
+ readonly artifacts?: readonly ChatArtifactView[] | null
}
type ChatThreadPersistenceAdapter = {
diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts
index 0bef342..eaf5f57 100644
--- a/src/domains/chat/contracts.ts
+++ b/src/domains/chat/contracts.ts
@@ -1,11 +1,14 @@
import type {
RetrievalQueryParams,
RetrievalQueryResponse,
- RetrievalSource,
} from "@ontos-ai/knowhere-sdk"
import type { Source } from "@/infrastructure/db/schema"
-import type { ChatCitationView } from "@/domains/chat/types"
+import type { HarnessRunResult } from "@/agent-harness"
+import type {
+ ChatArtifactView,
+ ChatCitationView,
+} from "@/domains/chat/types"
import type { LoadSourceAssetUrls } from "./media-assets"
export type RetrievalClient = {
@@ -39,22 +42,7 @@ export type AgenticRetrievalQuery = Pick<
readonly purpose?: string
}
-export type RetrievedChunkReference = {
- id: string
- chunkId: string | null
- kind: "result" | "referencedChunk"
- resultIndex: number | null
- chunkType: string
- score: number | null
- source: RetrievalSource
- hasAssetUrl: boolean
- contentLength: number
- contentPreview: string
- contentTruncated: boolean
-}
-
export type AgenticRetrievalResponse = RetrievalQueryResponse & {
- chunkReferences: readonly RetrievedChunkReference[]
retrievalPlan?: AgenticRetrievalPlan
}
@@ -62,40 +50,13 @@ export type SearchSources = (
input: AgenticRetrievalQuery,
) => Promise
-export type ReadRetrievedChunkInput = {
- id: string
- offset?: number
- limit?: number
-}
-
-export type ReadRetrievedChunkResult = {
- id: string
- chunkId: string | null
- found: boolean
- chunkType: string | null
- score: number | null
- source: RetrievalSource | null
- hasAssetUrl: boolean
- offset: number
- limit: number
- contentLength: number
- contentSlice: string
- hasMoreContent: boolean
- nextOffset: number | null
-}
-
-export type ReadRetrievedChunk = (
- input: ReadRetrievedChunkInput,
-) => Promise
-
export type GenerateAnswer = (input: {
question: string
messages: readonly ChatHistoryMessage[]
sources: readonly Source[]
excludedSourceIds: readonly string[]
searchSources: SearchSources
- readRetrievedChunk: ReadRetrievedChunk
-}) => Promise
+}) => Promise
export type AnswerQuestionInput = {
question: string
@@ -111,4 +72,5 @@ export type AnswerQuestionInput = {
export type AnswerQuestionResult = {
answer: string
citations: ChatCitationView[]
+ artifacts?: ChatArtifactView[]
}
diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts
index a60f3c1..0640de2 100644
--- a/src/domains/chat/index.test.ts
+++ b/src/domains/chat/index.test.ts
@@ -1,23 +1,15 @@
import { afterEach, describe, expect, it, vi } from "vitest"
import type { RetrievalResult } from "@ontos-ai/knowhere-sdk"
import { Effect } from "effect"
-import { generateText, ToolLoopAgent, type ModelMessage } from "ai"
+import { ToolLoopAgent } from "ai"
+import type { HarnessRunResult } from "@/agent-harness"
import {
answerQuestionWithRetrieval,
- buildAgenticChatSystemPrompt,
- buildGroundedPrompt,
- buildRetrievalQueryPrompt,
- generateAgenticGroundedAnswer,
- generateContextualRetrievalQuery,
- generateGroundedAnswer,
+ generateAgenticOutputManifest,
parseChatRequestBody,
} from "."
import type { Source } from "@/infrastructure/db/schema"
-import type {
- AgenticRetrievalQuery,
- ReadRetrievedChunkInput,
-} from "./contracts"
const loggerMock = vi.hoisted(() => ({
info: vi.fn(),
@@ -25,11 +17,6 @@ const loggerMock = vi.hoisted(() => ({
error: vi.fn(),
}));
-vi.mock("ai", async (importOriginal) => ({
- ...(await importOriginal()),
- generateText: vi.fn(),
-}));
-
vi.mock("@/lib/logger", () => ({
logger: {
info: loggerMock.info,
@@ -40,7 +27,6 @@ vi.mock("@/lib/logger", () => ({
afterEach(() => {
vi.restoreAllMocks();
- vi.mocked(generateText).mockReset();
loggerMock.info.mockReset();
loggerMock.warn.mockReset();
loggerMock.error.mockReset();
@@ -63,7 +49,7 @@ describe("answerQuestionWithRetrieval", () => {
};
const generateAnswer = vi.fn(async ({ searchSources }) => {
await searchSources({ query: "What does the document say?" });
- return "The answer is grounded.";
+ return makeHarnessRunResult("The answer is grounded.");
});
const sources = [
makeSource({ knowhereDocumentId: "doc_included" }),
@@ -96,7 +82,6 @@ describe("answerQuestionWithRetrieval", () => {
sources,
excludedSourceIds: ["source_2"],
searchSources: expect.any(Function),
- readRetrievedChunk: expect.any(Function),
});
expect(answer).toEqual({
answer: "The answer is grounded.",
@@ -143,7 +128,7 @@ describe("answerQuestionWithRetrieval", () => {
query: "冯荣洲 身份证 ID card",
targetContent: "image",
});
- return "Matched identity card image.";
+ return makeHarnessRunResult("Matched identity card image.");
});
await Effect.runPromise(
@@ -213,7 +198,9 @@ describe("answerQuestionWithRetrieval", () => {
};
const generateAnswer = vi.fn(async ({ searchSources }) => {
await searchSources({ query: "What improved?" });
- return "Revenue improved [Source 1: revenue growth]. Margins expanded [Source 2: margin expansion].";
+ return makeHarnessRunResult(
+ "Revenue improved [Source 1: revenue growth]. Margins expanded [Source 2: margin expansion].",
+ );
});
const answer = await Effect.runPromise(
@@ -255,7 +242,9 @@ describe("answerQuestionWithRetrieval", () => {
};
const generateAnswer = vi.fn(async ({ searchSources }) => {
await searchSources({ query: "Tesla xAI investment" });
- return "Tesla invested in xAI [Source 1: xAI investment].";
+ return makeHarnessRunResult(
+ "Tesla invested in xAI [Source 1: xAI investment].",
+ );
});
const sources = [
makeSource({
@@ -282,7 +271,6 @@ describe("answerQuestionWithRetrieval", () => {
sources,
excludedSourceIds: [],
searchSources: expect.any(Function),
- readRetrievedChunk: expect.any(Function),
});
const expectedResult = {
...result,
@@ -322,7 +310,9 @@ describe("answerQuestionWithRetrieval", () => {
targetContent: "image",
purpose: "Find visual rocket launch chunks.",
});
- return "Use this launch photo. https://blob.example/images/image-9-Night%20Rocket%20Launch.jpg";
+ return makeHarnessRunResult(
+ "Use this launch photo. https://blob.example/images/image-9-Night%20Rocket%20Launch.jpg",
+ );
});
const loadSourceAssetUrls = vi.fn().mockResolvedValue({
"images/image-9-Night Rocket Launch.jpg":
@@ -372,6 +362,184 @@ describe("answerQuestionWithRetrieval", () => {
]);
});
+ it("returns only harness-selected artifacts when retrieval has extra media candidates", async () => {
+ const frontAssetUrl = "https://blob.example/images/id-front.jpg";
+ const backAssetUrl = "https://blob.example/images/id-back.jpg";
+ const extraAssetUrl = "https://blob.example/images/extra.jpg";
+ const retrieval = {
+ query: vi.fn().mockResolvedValue({
+ results: [
+ makeRetrievalResult({
+ chunkType: "image",
+ assetUrl: frontAssetUrl,
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "document-generated.pdf",
+ sectionPath: "身份证正面",
+ },
+ }),
+ makeRetrievalResult({
+ chunkType: "image",
+ assetUrl: backAssetUrl,
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "document-generated.pdf",
+ sectionPath: "身份证反面",
+ },
+ }),
+ makeRetrievalResult({
+ chunkType: "image",
+ assetUrl: extraAssetUrl,
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "document-generated.pdf",
+ sectionPath: "营业执照",
+ },
+ }),
+ ],
+ evidenceText: "Identity image candidates.",
+ referencedChunks: [],
+ namespace: "notebook-workspace",
+ query: "冯荣洲 身份证 图片",
+ routerUsed: "workflow_single_step",
+ answerText: null,
+ }),
+ };
+ const generateAnswer = vi.fn(async ({ searchSources }) => {
+ await searchSources({
+ query: "冯荣洲 身份证 图片",
+ targetContent: "image",
+ });
+ const harnessResult: HarnessRunResult = {
+ manifest: {
+ text: "已找到相关身份证图片，见下方图片。",
+ citations: [],
+ artifacts: [
+ {
+ type: "image",
+ ref: "asset:r1:result:1",
+ display: true,
+ reason: "身份证正面",
+ },
+ {
+ type: "image",
+ ref: "asset:r1:result:2",
+ display: true,
+ reason: "身份证反面",
+ },
+ {
+ type: "image",
+ ref: "asset:r1:result:3",
+ display: true,
+ reason: "多余候选图片",
+ },
+ ],
+ unresolved: [],
+ },
+ trace: {
+ ledger: {
+ retrievalCount: 1,
+ evidenceText: ["Identity image candidates."],
+ stopReasons: [],
+ failureReasons: [],
+ decisionTraces: [],
+ chunks: [],
+ assets: [
+ {
+ ref: "asset:r1:result:1",
+ chunkRef: "r1:result:1",
+ type: "image",
+ assetUrl: frontAssetUrl,
+ label: "document-generated.pdf / 身份证正面 / image",
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "document-generated.pdf",
+ sectionPath: "身份证正面",
+ },
+ },
+ {
+ ref: "asset:r1:result:2",
+ chunkRef: "r1:result:2",
+ type: "image",
+ assetUrl: backAssetUrl,
+ label: "document-generated.pdf / 身份证反面 / image",
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "document-generated.pdf",
+ sectionPath: "身份证反面",
+ },
+ },
+ {
+ ref: "asset:r1:result:3",
+ chunkRef: "r1:result:3",
+ type: "image",
+ assetUrl: extraAssetUrl,
+ label: "document-generated.pdf / 营业执照 / image",
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "document-generated.pdf",
+ sectionPath: "营业执照",
+ },
+ },
+ ],
+ },
+ validationErrors: [],
+ revisionsUsed: 0,
+ intent: {
+ task: "show_media",
+ dependsOnPreviousTurn: false,
+ retrievalNeeded: "yes",
+ targetModalities: ["image"],
+ constraints: { desiredCount: 2, maxCount: 2 },
+ groundingPolicy: "must_use_sources",
+ },
+ contextPolicy: {
+ carryHistory: "none",
+ reason: "The current turn is self-contained.",
+ activePriorTurnIds: [],
+ },
+ },
+ };
+ return harnessResult;
+ });
+
+ const answer = await Effect.runPromise(
+ answerQuestionWithRetrieval({
+ question: "请只返回冯荣洲的 2 张身份证图片",
+ namespace: "notebook-workspace",
+ sources: [
+ makeSource({
+ title: "商务标文件.pdf",
+ knowhereDocumentId: "doc_identity",
+ }),
+ ],
+ excludedSourceIds: [],
+ retrieval,
+ generateAnswer,
+ messages: [],
+ }),
+ );
+
+ expect(answer.artifacts?.map((artifact) => artifact.assetUrl)).toEqual([
+ frontAssetUrl,
+ backAssetUrl,
+ ]);
+ expect(answer.artifacts?.map((artifact) => artifact.citation?.source)).toEqual(
+ [
+ {
+ documentId: "doc_identity",
+ sourceFileName: "商务标文件.pdf",
+ sectionPath: "身份证正面",
+ },
+ {
+ documentId: "doc_identity",
+ sourceFileName: "商务标文件.pdf",
+ sectionPath: "身份证反面",
+ },
+ ],
+ );
+ });
+
it("turns retrieved evidence image filenames into image citations", async () => {
const result = makeRetrievalResult({
content: "This section contains identity proof attachments.",
@@ -398,7 +566,7 @@ describe("answerQuestionWithRetrieval", () => {
query: "公民身份证明 图片",
targetContent: "image",
});
- return "这里是相关身份证明图片。";
+ return makeHarnessRunResult("这里是相关身份证明图片。");
});
const loadSourceAssetUrls = vi.fn().mockResolvedValue({
"images/image-6-中华人民共和国居民身份证.jpg":
@@ -433,7 +601,6 @@ describe("answerQuestionWithRetrieval", () => {
sources,
excludedSourceIds: [],
searchSources: expect.any(Function),
- readRetrievedChunk: expect.any(Function),
});
expect(retrieval.query).toHaveBeenCalledWith({
namespace: "notebook-workspace",
@@ -442,161 +609,19 @@ describe("answerQuestionWithRetrieval", () => {
useAgentic: true,
dataType: 3,
});
- expect(answer.citations.map((citation) => citation.assetUrl)).toEqual([
+ const imageCitations = answer.citations.filter(
+ (citation) => citation.assetUrl,
+ )
+ expect(imageCitations.map((citation) => citation.assetUrl)).toEqual([
"https://blob.example/images/image-6-id-front.jpg",
"https://blob.example/images/image-7-id-back.jpg",
]);
- expect(answer.citations.map((citation) => citation.chunkType)).toEqual([
+ expect(imageCitations.map((citation) => citation.chunkType)).toEqual([
"image",
"image",
]);
});
- it("sends requested identity-card images without exposing internal media metadata", async () => {
- const frontAssetUrl = "https://blob.example/images/feng-rongzhou-id-front.jpg";
- const backAssetUrl = "https://blob.example/images/feng-rongzhou-id-back.jpg";
- const unrelatedAssetUrl = "https://blob.example/images/company-license.jpg";
- const textResult = makeRetrievalResult({
- content: "冯荣洲的法定代表人身份证明页包含居民身份证图片。",
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "二、法定代表人身份证明",
- },
- });
- const duplicateFrontResult = {
- ...makeRetrievalResult({
- chunkType: "image",
- content: "冯荣洲居民身份证正面图片。",
- assetUrl: frontAssetUrl,
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "images/feng-rongzhou-id-front.jpg",
- },
- }),
- chunkId: "chunk_front_direct",
- } as RetrievalResult & { readonly chunkId: string };
- const richerDuplicateFrontResult = {
- ...makeRetrievalResult({
- chunkType: "image",
- content: "冯荣洲居民身份证正面图片，来源于身份证明章节。",
- assetUrl: frontAssetUrl,
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "二、法定代表人身份证明 / 身份证正面",
- },
- }),
- chunkId: "chunk_front_richer",
- } as RetrievalResult & { readonly chunkId: string };
- const backResult = {
- ...makeRetrievalResult({
- chunkType: "image",
- content: "冯荣洲居民身份证反面图片。",
- assetUrl: backAssetUrl,
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "二、法定代表人身份证明 / 身份证反面",
- },
- }),
- chunkId: "chunk_back",
- } as RetrievalResult & { readonly chunkId: string };
- const unrelatedImageResult = {
- ...makeRetrievalResult({
- chunkType: "image",
- content: "公司证照图片。",
- assetUrl: unrelatedAssetUrl,
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "images/company-license.jpg",
- },
- }),
- chunkId: "chunk_company_license",
- } as RetrievalResult & { readonly chunkId: string };
- const retrieval = {
- query: vi.fn().mockResolvedValue({
- results: [
- textResult,
- duplicateFrontResult,
- richerDuplicateFrontResult,
- backResult,
- unrelatedImageResult,
- ],
- evidenceText: "冯荣洲 身份证 图片",
- referencedChunks: [],
- namespace: "notebook-workspace",
- query: "冯荣洲 身份证 图片",
- routerUsed: "workflow_single_step",
- answerText: null,
- }),
- };
- const generateAnswer = vi.fn(async ({ searchSources }) => {
- await searchSources({
- query: "冯荣洲 身份证 图片",
- targetContent: "image",
- purpose: "查找冯荣洲的身份证图片。",
- });
- return [
- "为您找到冯荣洲的居民身份证图片，相关信息如下：",
- "- **姓名**：冯荣洲",
- "- **公民身份号码**：123456789012345678",
- "- **签发机关**：某公安局",
- "- **有效期限**：长期",
- `{"asset_id":"asset_front","assetUrl":"${frontAssetUrl}","chunkId":"chunk_front_direct"}`,
- ].join("\n");
- });
- const sources = [
- makeSource({
- id: "source_identity",
- title: "商务标文件.pdf",
- knowhereDocumentId: "doc_identity",
- }),
- ];
-
- const answer = await Effect.runPromise(
- answerQuestionWithRetrieval({
- question: "请将 冯荣洲 的身份证图片发给我",
- namespace: "notebook-workspace",
- sources,
- excludedSourceIds: [],
- retrieval,
- generateAnswer,
- messages: [],
- }),
- );
-
- expect(retrieval.query).toHaveBeenCalledWith({
- namespace: "notebook-workspace",
- query: "冯荣洲 身份证 图片",
- topK: 8,
- useAgentic: true,
- dataType: 3,
- });
- expect(answer.answer).toBe("已找到相关身份证图片，见下方图片。");
- expect(answer.answer).not.toMatch(
- /asset_id|assetUrl|asset_url|chunkId|chunk_id|https?:\/\//,
- );
- expect(answer.answer).not.toMatch(
- /姓名|公民身份号码|签发机关|有效期限|123456789012345678/,
- );
- expect(answer.citations.map((citation) => citation.assetUrl)).toEqual([
- frontAssetUrl,
- backAssetUrl,
- ]);
- expect(
- answer.citations.filter(
- (citation) => citation.assetUrl === frontAssetUrl,
- ),
- ).toHaveLength(1);
- expect(answer.citations[0]?.source).toMatchObject({
- sourceFileName: "商务标文件.pdf",
- sectionPath: "二、法定代表人身份证明 / 身份证正面",
- });
- });
-
it("returns the agent answer without citations when retrieval has no results", async () => {
const retrieval = {
query: vi.fn().mockResolvedValue({
@@ -611,7 +636,7 @@ describe("answerQuestionWithRetrieval", () => {
};
const generateAnswer = vi.fn(async ({ searchSources }) => {
await searchSources({ query: "Missing fact?" });
- return "I couldn't find that in your sources.";
+ return makeHarnessRunResult("I couldn't find that in your sources.");
});
const answer = await Effect.runPromise(
@@ -648,7 +673,7 @@ describe("answerQuestionWithRetrieval", () => {
await searchSources({
query: "Tesla Q4 2025 Update energy generation and storage deployments",
});
- return "Energy storage grew.";
+ return makeHarnessRunResult("Energy storage grew.");
});
const messages = [
{
@@ -686,7 +711,6 @@ describe("answerQuestionWithRetrieval", () => {
sources: [makeSource({ title: "TSLA-Q4-2025-Update.pdf" })],
excludedSourceIds: [],
searchSources: expect.any(Function),
- readRetrievedChunk: expect.any(Function),
});
});
@@ -704,7 +728,7 @@ describe("answerQuestionWithRetrieval", () => {
};
const generateAnswer = vi.fn(async ({ searchSources }) => {
await searchSources({ query: "Tesla energy storage deployments" });
- return "Energy storage grew.";
+ return makeHarnessRunResult("Energy storage grew.");
});
const messages = [
{
@@ -742,77 +766,6 @@ describe("answerQuestionWithRetrieval", () => {
);
});
- it("lets the agent read untruncated content from returned chunk ids", async () => {
- const longContent = `${"Earlier context. ".repeat(300)}Critical obligation: retain source receipts.`;
- const result = {
- ...makeRetrievalResult({
- content: longContent,
- source: {
- documentId: "doc_contract",
- sourceFileName: "contract.pdf",
- sectionPath: "Obligations",
- },
- }),
- chunkId: "chunk_contract_1",
- } as RetrievalResult & { readonly chunkId: string };
- const retrieval = {
- query: vi.fn().mockResolvedValue({
- results: [result],
- evidenceText: "Contract obligations were retrieved.",
- referencedChunks: [],
- namespace: "notebook-workspace",
- query: "contract obligations",
- routerUsed: "workflow_single_step",
- answerText: null,
- }),
- };
- const generateAnswer = vi.fn(
- async ({ searchSources, readRetrievedChunk }) => {
- const response = await searchSources({ query: "contract obligations" });
- expect(response.chunkReferences[0]).toMatchObject({
- id: "chunk_contract_1",
- chunkId: "chunk_contract_1",
- contentTruncated: true,
- contentLength: longContent.length,
- });
-
- const detail = await readRetrievedChunk({
- id: "chunk_contract_1",
- offset: 4_000,
- limit: 80,
- });
-
- expect(detail).toMatchObject({
- id: "chunk_contract_1",
- found: true,
- offset: 4_000,
- limit: 80,
- contentLength: longContent.length,
- });
- return detail.contentSlice;
- },
- );
-
- const answer = await Effect.runPromise(
- answerQuestionWithRetrieval({
- question: "What obligation matters?",
- namespace: "notebook-workspace",
- sources: [
- makeSource({
- title: "contract.pdf",
- knowhereDocumentId: "doc_contract",
- }),
- ],
- excludedSourceIds: [],
- retrieval,
- generateAnswer,
- messages: [],
- }),
- );
-
- expect(answer.answer).toBe(longContent.slice(4_000, 4_080).trim());
- });
-
it("uses structured referenced chunks from RetrievalQueryResponse as citations", async () => {
const retrieval = {
query: vi.fn().mockResolvedValue({
@@ -840,7 +793,7 @@ describe("answerQuestionWithRetrieval", () => {
query: "SpaceX launch image",
targetContent: "image",
});
- return "Here is the launch image.";
+ return makeHarnessRunResult("Here is the launch image.");
});
const answer = await Effect.runPromise(
@@ -876,199 +829,113 @@ describe("answerQuestionWithRetrieval", () => {
});
});
-describe("generateContextualRetrievalQuery", () => {
- it("uses the latest question directly when there is no chat history", async () => {
- const query = await generateContextualRetrievalQuery({
- question: "What does Tesla say about energy storage?",
- messages: [],
- sources: [makeSource({ title: "TSLA-Q4-2025-Update.pdf" })],
- excludedSourceIds: [],
- });
-
- expect(generateText).not.toHaveBeenCalled();
- expect(query).toBe("What does Tesla say about energy storage?");
- });
-
- it("asks the model to produce a stateless Knowhere query from chat context", async () => {
- process.env.AI_GATEWAY_API_KEY = "test_gateway_key";
- vi.mocked(generateText).mockResolvedValue({
- text: "Query: Tesla Q4 2025 Update energy storage deployments",
- } as Awaited>);
-
- const query = await generateContextualRetrievalQuery({
- question: "What about energy storage in this document?",
- messages: [
- {
- role: "user",
- content: "Tell me about Tesla's Q4 2025 update.",
- },
- ],
- sources: [makeSource({ title: "TSLA-Q4-2025-Update.pdf" })],
- excludedSourceIds: [],
- });
-
- expect(generateText).toHaveBeenCalledWith({
- model: "google/gemini-3-flash",
- prompt: expect.stringContaining("Knowhere retrieval is stateless"),
- });
- expect(query).toBe("Tesla Q4 2025 Update energy storage deployments");
- });
-});
-
-describe("generateGroundedAnswer", () => {
- it("routes grounded prompts through Vercel AI Gateway model strings", async () => {
- process.env.AI_GATEWAY_API_KEY = "test_gateway_key";
- vi.mocked(generateText).mockResolvedValue({
- text: "PR-E wires chat to retrieval.",
- } as Awaited>);
-
- const answer = await generateGroundedAnswer({
- question: "What is PR-E?",
- retrievalQuery: "PR-E retrieval",
- messages: [],
- evidenceText: "PR-E wires chat to Knowhere retrieval.",
- });
-
- expect(generateText).toHaveBeenCalledWith({
- model: "google/gemini-3-flash",
- prompt: expect.stringContaining("PR-E wires chat to Knowhere retrieval."),
- });
- expect(answer).toBe("PR-E wires chat to retrieval.");
- expect(getLoggerInfoMeta("chat-agent: llm request")).toMatchObject({
- operation: "generateGroundedAnswer",
- model: "google/gemini-3-flash",
- promptType: "text",
- prompt: expect.stringContaining("PR-E wires chat to Knowhere retrieval."),
- });
- expect(getLoggerInfoMeta("chat-agent: llm response")).toMatchObject({
- operation: "generateGroundedAnswer",
- model: "google/gemini-3-flash",
- responseText: "PR-E wires chat to retrieval.",
- responseTextCharLength: "PR-E wires chat to retrieval.".length,
- });
- });
-});
-
-describe("generateAgenticGroundedAnswer", () => {
- it("builds a Vercel AI SDK tool loop around Knowhere retrieval", async () => {
+describe("generateAgenticOutputManifest", () => {
+ it("runs the outer harness workflow around Knowhere retrieval", async () => {
process.env.AI_GATEWAY_API_KEY = "test_gateway_key";
let capturedGenerateInput:
| Parameters[0]
| undefined;
- const generateSpy = vi
- .spyOn(ToolLoopAgent.prototype, "generate")
- .mockImplementation((
+ vi.spyOn(ToolLoopAgent.prototype, "generate").mockImplementation(
+ async function mockGenerate(
+ this: ToolLoopAgent,
input: Parameters[0],
- ): ReturnType => {
+ ): ReturnType {
capturedGenerateInput = input;
- return Promise.resolve({
- text: "Here are the requested identity images.",
- } as Awaited>);
- });
- const previewWithinNewLimitMarker = "within-new-preview-limit";
- const previewAfterNewLimitMarker = "after-new-preview-limit";
- const fullToolOutputEvidenceMarker = "full-tool-output-evidence-end";
- const evidenceTreeText = [
- "Identity image evidence. https://blob.example/images/id-front.jpg",
- "[Document] document-generated.pdf",
- "▸ [L1] Assets",
- " ▸ [L2] images / id-front.jpg",
- ` ┈ ${"evidence ".repeat(500)}${fullToolOutputEvidenceMarker}`,
- ].join("\n");
- const longIdentityPreview = [
- "Identity card image front side.",
- "preview ".repeat(170),
- previewWithinNewLimitMarker,
- "preview ".repeat(70),
- previewAfterNewLimitMarker,
- ].join(" ");
+ const tools = this.tools as unknown as Record<
+ string,
+ { execute: (input: unknown) => Promise }
+ >;
+
+ await tools.declareIntent?.execute({
+ task: "show_media",
+ dependsOnPreviousTurn: false,
+ retrievalNeeded: "yes",
+ targetModalities: ["text", "image"],
+ constraints: { desiredCount: 2, maxCount: 2 },
+ groundingPolicy: "must_use_sources",
+ });
+ await tools.setContextPolicy?.execute({
+ carryHistory: "none",
+ reason: "The current request is self-contained.",
+ activePriorTurnIds: [],
+ });
+ await tools.retrieve?.execute({
+ query: "冯荣洲 身份证 图片",
+ modalities: ["text", "image"],
+ topK: 2,
+ purpose: "Find exactly the requested identity-card images.",
+ });
+ await tools.finalize?.execute({
+ text: "已找到相关身份证图片，见下方图片。",
+ citations: [
+ {
+ ref: "r1:result:1",
+ label: "商务标文件.pdf / 身份证正面",
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "商务标文件.pdf",
+ sectionPath: "身份证正面",
+ },
+ },
+ ],
+ artifacts: [
+ {
+ type: "image",
+ ref: "asset:r1:result:1",
+ display: true,
+ reason: "身份证正面",
+ },
+ ],
+ unresolved: [],
+ });
+
+ return {
+ text: "This freeform text should be ignored.",
+ } as Awaited>;
+ },
+ );
const searchSources = vi.fn().mockResolvedValue({
results: [
makeRetrievalResult({
- content: longIdentityPreview,
chunkType: "image",
assetUrl: "https://blob.example/images/id-front.jpg",
source: {
documentId: "doc_identity",
sourceFileName: "document-generated.pdf",
- sectionPath: "Assets / images / id-front.jpg",
+ sectionPath: "身份证正面",
},
}),
],
- evidenceText: evidenceTreeText,
- referencedChunks: [
- {
- chunkId: "chunk_identity_1",
- documentId: "doc_identity",
- chunkType: "image",
- sectionPath: "Assets / images / id-front.jpg",
- filePath: "images/id-front.jpg",
- jobId: "job_1",
- assetUrl: "https://blob.example/images/id-front.jpg",
- },
- ],
+ evidenceText: "Identity image evidence.",
+ referencedChunks: [],
namespace: "notebook-workspace",
- query: "公民身份证 图片",
+ query: "冯荣洲 身份证 图片",
routerUsed: "workflow_single_step",
- retrievalPlan: {
- targetContent: "image",
- purpose: "Find identity-card image evidence.",
- },
- chunkReferences: [
- {
- id: "chunk_identity_1",
- chunkId: "chunk_identity_1",
- kind: "result",
- resultIndex: 1,
- chunkType: "image",
- score: 0.9,
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "Assets / images / id-front.jpg",
- },
- hasAssetUrl: true,
- contentLength: longIdentityPreview.length,
- contentPreview: longIdentityPreview,
- contentTruncated: true,
- },
- ],
- answerText:
- "The source includes identity card images. https://blob.example/images/id-front.jpg",
+ chunkReferences: [],
+ answerText: null,
stopReason: "answer_done",
failureReason: null,
- decisionTrace: [
+ });
+
+ const result = await generateAgenticOutputManifest({
+ question: "请只返回冯荣洲的 2 张身份证图片",
+ messages: [
{
- step: "final",
- stop: "answer_done",
- assetUrl: "https://blob.example/images/id-front.jpg",
+ role: "assistant",
+ content: "上一轮是完全不同的税务问题。",
+ citations: [
+ {
+ chunkType: "text",
+ score: 0.9,
+ source: {
+ documentId: "doc_tax",
+ sourceFileName: "tax.pdf",
+ sectionPath: "deadline",
+ },
+ },
+ ],
},
],
- });
- const readRetrievedChunk = vi.fn().mockResolvedValue({
- id: "chunk_identity_1",
- chunkId: "chunk_identity_1",
- found: true,
- chunkType: "image",
- score: 0.9,
- source: {
- documentId: "doc_identity",
- sourceFileName: "document-generated.pdf",
- sectionPath: "Assets / images / id-front.jpg",
- },
- hasAssetUrl: true,
- offset: 0,
- limit: 80,
- contentLength: 96,
- contentSlice:
- "Full identity card text. https://blob.example/images/id-front.jpg",
- hasMoreContent: false,
- nextOffset: null,
- });
-
- const answer = await generateAgenticGroundedAnswer({
- question: "请发送几张关于公民身份的图片给我",
- messages: [],
sources: [
makeSource({
title: "商务标文件.pdf",
@@ -1077,559 +944,133 @@ describe("generateAgenticGroundedAnswer", () => {
],
excludedSourceIds: [],
searchSources,
- readRetrievedChunk,
});
- expect(answer).toBe("Here are the requested identity images.");
- expect(generateSpy).toHaveBeenCalledWith({
- messages: expect.any(Array),
+ expect(result.manifest.text).toBe("已找到相关身份证图片，见下方图片。");
+ expect(result.trace.intent).toMatchObject({
+ task: "show_media",
+ constraints: { desiredCount: 2, maxCount: 2 },
});
- const agent = getCapturedAgent(generateSpy.mock.contexts[0]);
- const settings = getCapturedAgentSettings(agent);
- const generateInput = getCapturedGenerateInput(capturedGenerateInput);
-
- expect(settings.instructions).toContain("markdown output gives guidance")
- expect(settings.instructions).toContain("image or text+image search")
- expect(settings.instructions).toContain("Read IDs")
- expect(settings.instructions).toContain(
- "Do not paste raw prior messages into searchSources.query",
- )
- expect(generateInput.messages.at(-1)).toEqual({
- role: "user",
- content: "请发送几张关于公民身份的图片给我",
- })
- expect(
- settings.prepareStep({
- stepNumber: 0,
- messages: [...generateInput.messages],
- }),
- ).toMatchObject({
- toolChoice: { type: "tool", toolName: "searchSources" },
- activeTools: ["searchSources"],
- })
- expect(
- settings.prepareStep({
- stepNumber: 1,
- messages: [...generateInput.messages],
- }),
- ).toMatchObject({
- toolChoice: { type: "tool", toolName: "searchSources" },
- activeTools: ["searchSources"],
- })
- expect(
- settings.prepareStep({
- stepNumber: 2,
- messages: [...generateInput.messages],
- }),
- ).toMatchObject({
- messages: expect.any(Array),
- })
-
- const searchSourcesTool = getCapturedAgentTools(agent).searchSources
- expect(
- getSearchSourcesTargetContentSchema(searchSourcesTool)._def?.innerType
- ?._def?.type,
- ).toBe("enum")
- expect(
- searchSourcesTool.inputSchema.safeParse({
- query: "公民身份证 图片",
- targetContent: "image",
- }).success,
- ).toBe(true)
- expect(
- searchSourcesTool.inputSchema.safeParse({
- query: "公民身份证 图片",
- targetContent: "video",
- }).success,
- ).toBe(false)
-
- const toolOutput = await searchSourcesTool.execute({
- query: "公民身份证 图片",
- targetContent: "image",
- purpose: "Find identity-card image evidence.",
+ expect(result.trace.contextPolicy).toMatchObject({
+ carryHistory: "none",
});
-
+ expect(result.trace.validationErrors).toEqual([]);
expect(searchSources).toHaveBeenCalledWith({
- query: "公民身份证 图片",
- targetContent: "image",
- purpose: "Find identity-card image evidence.",
- });
- expect(toolOutput).toEqual(expect.any(String));
- expect(toolOutput).toContain("## Retrieval Result");
- expect(toolOutput).toContain("Query: 公民身份证 图片");
- expect(toolOutput).toContain("Guidance: Use this evidence");
- expect(toolOutput).toContain("## Evidence");
- expect(toolOutput).toContain(
- "[Document] document-generated.pdf\n▸ [L1] Assets\n ▸ [L2] images / id-front.jpg",
- );
- expect(toolOutput).toContain(fullToolOutputEvidenceMarker);
- expect(toolOutput).not.toContain(
- "[Document] document-generated.pdf ▸ [L1] Assets",
- );
- expect(toolOutput).toContain("## Decision Trace");
- expect(toolOutput).toContain("- Step 1:");
- expect(toolOutput).toContain("- step: final");
- expect(toolOutput).toContain("- stop: answer_done");
- expect(toolOutput).toContain("### Result 1");
- expect(toolOutput).toContain("Type: image");
- expect(toolOutput).toContain(
- "Source: document-generated.pdf / Assets / images / id-front.jpg",
- );
- expect(toolOutput).toContain("Media: image available");
- expect(toolOutput).toContain("Read ID: chunk_identity_1");
- expect(toolOutput).toContain("Identity card image front side.");
- expect(toolOutput).toContain(previewWithinNewLimitMarker);
- expect(toolOutput).not.toContain(previewAfterNewLimitMarker);
- expect(toolOutput).not.toContain("https://blob.example");
- expect(toolOutput).not.toContain("assetUrl");
- expect(toolOutput).not.toContain("retrievalPlan");
- expect(toolOutput).not.toContain("decisionTrace");
- expect(getLoggerInfoMeta("chat-agent: tool output")).toMatchObject({
- toolName: "searchSources",
- output: expect.objectContaining({
- truncated: false,
- preview: expect.stringContaining(fullToolOutputEvidenceMarker),
- }),
- });
-
- const readRetrievedChunkTool = getCapturedAgentTools(agent).readRetrievedChunk;
- expect(
- readRetrievedChunkTool.inputSchema.safeParse({
- id: "chunk_identity_1",
- limit: 8_000,
- }).success,
- ).toBe(true);
- expect(
- readRetrievedChunkTool.inputSchema.safeParse({
- id: "chunk_identity_1",
- limit: 8_001,
- }).success,
- ).toBe(false);
-
- const chunkOutput = await readRetrievedChunkTool.execute({
- id: "chunk_identity_1",
- offset: 0,
- limit: 80,
- });
-
- expect(readRetrievedChunk).toHaveBeenCalledWith({
- id: "chunk_identity_1",
- offset: 0,
- limit: 80,
- });
- expect(chunkOutput).toEqual(expect.any(String));
- expect(chunkOutput).toContain("## Retrieved Content");
- expect(chunkOutput).toContain("Status: found");
- expect(chunkOutput).toContain("Read ID: chunk_identity_1");
- expect(chunkOutput).toContain(
- "Source: document-generated.pdf / Assets / images / id-front.jpg",
- );
- expect(chunkOutput).toContain(
- "Full identity card text. [media asset URL hidden]",
- );
- expect(chunkOutput).not.toContain("https://blob.example");
- expect(chunkOutput).not.toContain("chunkId");
- expect(getLoggerInfoMeta("chat-agent: tool output")).toMatchObject({
- toolName: "readRetrievedChunk",
- output: expect.objectContaining({
- truncated: false,
- preview: expect.stringContaining("## Retrieved Content\n\nStatus"),
- }),
- });
+ query: "冯荣洲 身份证 图片",
+ targetContent: "text_image",
+ purpose: "Find exactly the requested identity-card images.",
+ topK: 2,
+ signalPaths: undefined,
+ filterMode: undefined,
+ threshold: undefined,
+ });
+ expect(JSON.stringify(capturedGenerateInput)).toContain("Recent turn index");
+ expect(JSON.stringify(capturedGenerateInput)).toContain("tax.pdf / deadline");
});
- it("uses managed context for stored history and loop steps", async () => {
+ it("self-corrects an over-budget manifest via a validation-feedback revision", async () => {
process.env.AI_GATEWAY_API_KEY = "test_gateway_key";
- let capturedGenerateInput:
- | Parameters[0]
- | undefined;
- const generateSpy = vi
- .spyOn(ToolLoopAgent.prototype, "generate")
- .mockImplementation((
- input: Parameters[0],
- ): ReturnType => {
- capturedGenerateInput = input;
- return Promise.resolve({
- text: "The answer is grounded.",
- } as Awaited>);
- });
- const messages = Array.from({ length: 24 }, (_, index) => ({
- role: index % 2 === 0 ? ("user" as const) : ("assistant" as const),
- content: `history-message-${index} ${"context ".repeat(80)}`,
- }));
-
- await generateAgenticGroundedAnswer({
- question: "What should I know now?",
- messages,
- sources: [makeSource()],
- excludedSourceIds: [],
- searchSources: vi.fn(),
- readRetrievedChunk: vi.fn(),
- });
-
- const generateInput = getCapturedGenerateInput(capturedGenerateInput);
- const serializedMessages = JSON.stringify(generateInput.messages);
- expect(generateInput.messages[0]).toMatchObject({
- role: "system",
- content: expect.stringContaining("Compacted earlier conversation"),
- });
- expect(serializedMessages).not.toContain("history-message-0");
- expect(serializedMessages).toContain("history-message-23");
-
- const settings = getCapturedAgentSettings(
- getCapturedAgent(generateSpy.mock.contexts[0]),
- );
- const oversizedLoopMessages = Array.from({ length: 25 }, (_, index) => ({
- role: "user" as const,
- content: `loop-message-${index}`,
- }));
- const preparedStep = settings.prepareStep({
- stepNumber: 2,
- messages: oversizedLoopMessages,
- }) as { readonly messages: readonly ModelMessage[] };
-
- expect(preparedStep.messages.length).toBeLessThanOrEqual(12);
- expect(JSON.stringify(preparedStep.messages)).not.toContain("loop-message-0");
- expect(JSON.stringify(preparedStep.messages)).toContain("loop-message-24");
- expect(getLoggerInfoMeta("chat-agent: llm request")).toMatchObject({
- operation: "generateAgenticGroundedAnswer.step",
- model: "google/gemini-3-flash",
- promptType: "messages",
- stepNumber: 2,
- instructions: expect.stringContaining("Notebook research agent"),
- messageCount: preparedStep.messages.length,
- messages: expect.arrayContaining([
- expect.objectContaining({
- role: "user",
- content: "loop-message-24",
- }),
- ]),
- });
-
- const hugeLoopMessages = [
- {
- role: "user" as const,
- content: `huge-loop-message ${"context ".repeat(9_000)}`,
- },
- {
- role: "assistant" as const,
- content: "middle-loop-message",
+ let generateCallCount = 0;
+ vi.spyOn(ToolLoopAgent.prototype, "generate").mockImplementation(
+ async function mockGenerate(
+ this: ToolLoopAgent,
+ ): ReturnType {
+ generateCallCount += 1;
+ const tools = this.tools as unknown as Record<
+ string,
+ { execute: (input: unknown) => Promise }
+ >;
+
+ if (generateCallCount === 1) {
+ await tools.declareIntent?.execute({
+ task: "show_media",
+ dependsOnPreviousTurn: false,
+ retrievalNeeded: "yes",
+ targetModalities: ["image"],
+ constraints: { desiredCount: 2, maxCount: 2 },
+ groundingPolicy: "must_use_sources",
+ });
+ await tools.setContextPolicy?.execute({
+ carryHistory: "none",
+ reason: "Self-contained request.",
+ activePriorTurnIds: [],
+ });
+ await tools.retrieve?.execute({
+ query: "身份证 图片",
+ modalities: ["image"],
+ topK: 3,
+ purpose: "Find requested identity images.",
+ });
+ await tools.finalize?.execute({
+ text: "见下方图片。",
+ citations: [{ ref: "r1:result:1", label: "id" }],
+ artifacts: [1, 2, 3].map((index) => ({
+ type: "image",
+ ref: `asset:r1:result:${index}`,
+ display: true,
+ reason: "candidate",
+ })),
+ unresolved: [],
+ });
+ } else {
+ await tools.finalize?.execute({
+ text: "见下方图片。",
+ citations: [{ ref: "r1:result:1", label: "id" }],
+ artifacts: [1, 2].map((index) => ({
+ type: "image",
+ ref: `asset:r1:result:${index}`,
+ display: true,
+ reason: "selected",
+ })),
+ unresolved: [],
+ });
+ }
+
+ return {
+ text: "ignored",
+ response: { messages: [] },
+ } as unknown as Awaited>;
},
- {
- role: "user" as const,
- content: "latest-loop-message",
- },
- ];
- const preparedHugeStep = settings.prepareStep({
- stepNumber: 2,
- messages: hugeLoopMessages,
- }) as { readonly messages: readonly ModelMessage[] };
- const serializedHugeStepMessages = JSON.stringify(
- preparedHugeStep.messages,
);
- expect(getTestModelMessagesCharLength(preparedHugeStep.messages)).toBeLessThanOrEqual(
- 64_000,
- );
- expect(serializedHugeStepMessages).not.toContain("huge-loop-message");
- expect(serializedHugeStepMessages).toContain("latest-loop-message");
- });
-
- it("logs bounded tool calls and complete tool results for each loop step", async () => {
- process.env.AI_GATEWAY_API_KEY = "test_gateway_key";
- const fullStepToolOutputMarker = "full-step-tool-output-end";
- const fullStepToolOutput = `\n${[
- "## Retrieval Result",
- "",
- "Status: useful_evidence_found",
- "Query: 冯荣洲 身份证 ID card",
- "Guidance: Use this evidence if it directly answers the user.",
- "",
- "## Evidence",
- `Image evidence https://blob.example/id-front.jpg ${"evidence ".repeat(
- 600,
- )}`,
- "",
- "## Results",
- "### Result 1",
- "Type: image",
- "Source: 商务标文件.pdf / 二、法定代表人身份证明",
- "Media: image available",
- "Read ID: chunk_identity_1",
- "",
- "Preview:",
- `Identity image content ${"result ".repeat(80)}`,
- fullStepToolOutputMarker,
- ].join("\n")}\n `;
- const generateSpy = vi
- .spyOn(ToolLoopAgent.prototype, "generate")
- .mockResolvedValue({
- text: "The answer is grounded.",
- } as Awaited>);
-
- await generateAgenticGroundedAnswer({
- question: "请将 冯荣洲 的身份证图片发给我",
- messages: [],
- sources: [makeSource()],
- excludedSourceIds: [],
- searchSources: vi.fn(),
- readRetrievedChunk: vi.fn(),
- });
-
- const settings = getCapturedAgentSettings(
- getCapturedAgent(generateSpy.mock.contexts[0]),
- );
- loggerMock.info.mockClear();
-
- settings.onStepFinish({
- stepNumber: 1,
- finishReason: "tool-calls",
- text: `Inspecting identity image candidates. ${"reason ".repeat(200)}`,
- toolCalls: [
- {
- toolName: "searchSources",
- toolCallId: "call_1",
- input: {
- query: "冯荣洲 身份证 ID card",
- purpose: `Find the matching identity card image. ${"input ".repeat(
- 300,
- )}`,
- targetContent: "image",
+ const searchSources = vi.fn().mockResolvedValue({
+ results: [1, 2, 3].map((index) =>
+ makeRetrievalResult({
+ chunkType: "image",
+ assetUrl: `https://blob.example/images/id-${index}.jpg`,
+ source: {
+ documentId: "doc_identity",
+ sourceFileName: "ids.pdf",
+ sectionPath: `身份证 ${index}`,
},
- },
- ],
- toolResults: [
- {
- toolName: "searchSources",
- toolCallId: "call_1",
- output: fullStepToolOutput,
- },
- ],
- usage: {
- inputTokens: 11,
- outputTokens: 22,
- totalTokens: 33,
- },
- });
-
- const stepMeta = getLoggerInfoMeta("chat-agent: llm response");
- const stepLog = stepMeta as unknown as AgentLoopStepLogMeta;
- expect(stepLog).toMatchObject({
- operation: "generateAgenticGroundedAnswer.step",
- model: "google/gemini-3-flash",
- stepNumber: 1,
- finishReason: "tool-calls",
- toolCallCount: 1,
- toolResultCount: 1,
- inputTokens: 11,
- outputTokens: 22,
- totalTokens: 33,
- });
- expect(stepLog.responseText).toContain("Inspecting identity image candidates.");
- expect(stepLog.toolCalls[0]?.input.truncated).toBe(true);
- expect(stepLog.toolResults[0]?.output).toMatchObject({
- kind: "searchSources",
- output: {
- truncated: false,
- },
- });
- const searchSourcesOutput = stepLog.toolResults[0]
- ?.output as SearchSourcesToolOutputLogMeta;
- expect(searchSourcesOutput.output.preview.startsWith("\n## Retrieval Result"))
- .toBe(true);
- expect(searchSourcesOutput.output.preview.endsWith("\n ")).toBe(true);
- expect(searchSourcesOutput.output.preview).toContain("## Retrieval Result");
- expect(searchSourcesOutput.output.preview).toContain("\n\n## Evidence");
- expect(searchSourcesOutput.output.preview).toContain(
- "Query: 冯荣洲 身份证 ID card",
- );
- expect(searchSourcesOutput.output.preview).toContain(
- "[media asset URL hidden]",
- );
- expect(searchSourcesOutput.output.preview).toContain(
- fullStepToolOutputMarker,
- );
- expect(JSON.stringify(stepMeta)).not.toContain("https://blob.example");
-
- settings.onFinish({
- steps: [
- {
- stepNumber: 1,
- finishReason: "tool-calls",
- text: "Read tool result.",
- toolCalls: [
- {
- toolName: "searchSources",
- toolCallId: "call_1",
- input: { query: "冯荣洲 身份证 ID card" },
- },
- ],
- toolResults: [
- {
- toolName: "searchSources",
- toolCallId: "call_1",
- output: "## Evidence\nMatched image evidence.",
- },
- ],
- },
- ],
- finishReason: "stop",
- text: "Here is the matched identity card image.",
- totalUsage: {
- inputTokens: 40,
- outputTokens: 20,
- totalTokens: 60,
- },
- });
-
- const finishMeta = getLoggerInfoMeta("chat-agent: loop finished");
- expect(finishMeta).toMatchObject({
- stepCount: 1,
- finishReason: "stop",
- responseText: "Here is the matched identity card image.",
- toolNames: ["searchSources"],
- steps: [
- expect.objectContaining({
- stepNumber: 1,
- toolCallCount: 1,
- toolResultCount: 1,
}),
- ],
- inputTokens: 40,
- outputTokens: 20,
- totalTokens: 60,
- });
- });
-});
-
-describe("buildGroundedPrompt", () => {
- it("includes evidence text and uses evidence-based citation format", () => {
- const prompt = buildGroundedPrompt({
- question: "What is PR-E?",
- evidenceText: "PR-E wires chat to Knowhere retrieval.\n[Document] requirements.txt\n▸ [L1] N-005",
- });
-
- expect(prompt).toContain("What is PR-E?");
- expect(prompt).toContain("Retrieval query used: What is PR-E?");
- expect(prompt).toContain("PR-E wires chat to Knowhere retrieval.");
- expect(prompt).toContain("requirements.txt");
- expect(prompt).toContain(
- "Use the retrieved evidence as your primary context.",
- );
- expect(prompt).toContain("Retrieved evidence:");
- expect(prompt).not.toContain("Source excerpts:");
- });
-
- it("asks the model to answer naturally and directly", () => {
- const prompt = buildGroundedPrompt({
- question: "How about the TBD?",
- evidenceText: "Roadster location: TBD. Status: Design development.",
- });
-
- expect(prompt).toContain("Answer in a natural, friendly, and direct tone.");
- expect(prompt).toContain("Use GitHub-flavored Markdown when it improves readability");
- expect(prompt).toContain("Start with the answer first.");
- expect(prompt).toContain("Avoid meta phrases like \"Based on the sources\"");
- expect(prompt).toContain("Keep answers concise by default");
- expect(prompt).toContain(
- "If the sources are related but incomplete, answer what you can and briefly say what is not covered.",
- );
- });
-
- it("includes retrieved media asset references as internal metadata", () => {
- const prompt = buildGroundedPrompt({
- question: "Show me the launch image.",
- evidenceText: "A launch image was retrieved.",
- mediaAssetContext:
- "- spacex-s1.pdf / Assets / images / launch.jpg: https://blob.example/images/launch.jpg",
+ ),
+ evidenceText: "Identity image evidence.",
+ referencedChunks: [],
+ namespace: "notebook-workspace",
+ query: "身份证 图片",
+ routerUsed: "workflow_single_step",
+ answerText: null,
+ stopReason: "answer_done",
+ failureReason: null,
});
- expect(prompt).toContain(
- "Retrieved media asset references (internal; do not quote raw URLs):",
- );
- expect(prompt).toContain(
- "When retrieved image or table asset references are relevant to the user's request, cite the matching source label; the UI renders media from citation metadata.",
- );
- expect(prompt).toContain(
- "Do not write raw media asset URLs in the answer. They are internal metadata only.",
- );
- expect(prompt).toContain("Never output JSON metadata blocks");
- expect(prompt).toContain("Never mention asset_id, assetUrl");
- expect(prompt).toContain("do not transcribe personal details");
- expect(prompt).toContain("https://blob.example/images/launch.jpg");
- });
-});
-
-describe("buildAgenticChatSystemPrompt", () => {
- it("instructs the agent how to continue or stop from retrieval responses", () => {
- const prompt = buildAgenticChatSystemPrompt({
+ const result = await generateAgenticOutputManifest({
+ question: "只要 2 张身份证图片",
messages: [],
- sources: [makeSource({ title: "商务标文件.pdf" })],
- excludedSourceIds: [],
- });
-
- expect(prompt).toContain("Always call searchSources")
- expect(prompt).toContain("Make a second searchSources call")
- expect(prompt).toContain("readRetrievedChunk")
- expect(prompt).toContain("markdown output gives guidance")
- expect(prompt).toContain("Read IDs")
- expect(prompt).toContain("image or text+image search")
- expect(prompt).toContain("remote index")
- expect(prompt).toContain("person or section but not an image asset")
- expect(prompt).toContain("Do not paste raw prior messages")
- expect(prompt).toContain("身份证")
- expect(prompt).toContain("For image requests, search visual content directly")
- expect(prompt).toContain("Never output JSON metadata blocks")
- expect(prompt).toContain("Use GitHub-flavored Markdown when it improves readability")
- expect(prompt).toContain("do not transcribe personal details")
- expect(prompt).not.toContain("targetContent maps")
- expect(prompt).not.toContain("Read the tool output fields")
- expect(prompt).not.toContain("intent=overview")
- expect(prompt).toContain("商务标文件.pdf")
- });
-});
-
-describe("buildRetrievalQueryPrompt", () => {
- it("includes source and history context for stateless retrieval", () => {
- const prompt = buildRetrievalQueryPrompt({
- question: "What about energy storage in this document?",
- messages: [
- {
- role: "assistant",
- content: "Tesla's update mentions Q4 revenue.",
- citations: [
- {
- chunkType: "text",
- score: 0.9,
- source: {
- documentId: "doc_tesla",
- sourceFileName: "TSLA-Q4-2025-Update.pdf",
- sectionPath: "FINANCIAL SUMMARY",
- },
- },
- ],
- },
- ],
sources: [
- makeSource({
- id: "source_tesla",
- title: "TSLA-Q4-2025-Update.pdf",
- knowhereDocumentId: "doc_tesla",
- }),
- makeSource({
- id: "source_excluded",
- title: "Other.pdf",
- knowhereDocumentId: "doc_other",
- }),
+ makeSource({ title: "ids.pdf", knowhereDocumentId: "doc_identity" }),
],
- excludedSourceIds: ["source_excluded"],
+ excludedSourceIds: [],
+ searchSources,
});
- expect(prompt).toContain("Knowhere retrieval is stateless");
- expect(prompt).toContain("TSLA-Q4-2025-Update.pdf");
- expect(prompt).toContain("FINANCIAL SUMMARY");
- expect(prompt).toContain("What about energy storage in this document?");
- expect(prompt).not.toContain("Other.pdf");
+ expect(generateCallCount).toBe(2);
+ expect(result.trace.revisionsUsed).toBe(1);
+ expect(result.trace.validationErrors).toEqual([]);
+ expect(
+ result.manifest.artifacts.filter((artifact) => artifact.display).length,
+ ).toBe(2);
});
});
@@ -1699,71 +1140,28 @@ function makeSource(overrides: Partial = {}): Source {
};
}
-type CapturedAgentSettings = {
- readonly instructions: string
- readonly prepareStep: (input: {
- readonly stepNumber: number
- readonly messages: ModelMessage[]
- }) => unknown
- readonly onStepFinish: (input: unknown) => void
- readonly onFinish: (input: unknown) => void
-}
-
-type CapturedAgentTools = {
- readonly searchSources: {
- readonly inputSchema: {
- readonly _def?: {
- readonly type?: string
- readonly shape?:
- | Record
- | (() => Record)
- }
- readonly safeParse: (value: unknown) => { readonly success: boolean }
- }
- readonly execute: (input: AgenticRetrievalQuery) => Promise
- }
- readonly readRetrievedChunk: {
- readonly inputSchema: {
- readonly safeParse: (value: unknown) => { readonly success: boolean }
- }
- readonly execute: (input: ReadRetrievedChunkInput) => Promise
- }
-}
-
-type CapturedZodSchema = {
- readonly _def?: {
- readonly type?: string
- readonly innerType?: CapturedZodSchema
- }
-}
-
-type AgentLoopStepLogMeta = {
- readonly operation: string
- readonly model: string
- readonly stepNumber: number
- readonly finishReason: string
- readonly responseText: string
- readonly toolCallCount: number
- readonly toolCalls: readonly {
- readonly input: AgentLoopLogPreviewMeta
- }[]
- readonly toolResultCount: number
- readonly toolResults: readonly {
- readonly output: SearchSourcesToolOutputLogMeta | AgentLoopLogPreviewMeta
- }[]
- readonly inputTokens: number
- readonly outputTokens: number
- readonly totalTokens: number
-}
-
-type SearchSourcesToolOutputLogMeta = {
- readonly kind: "searchSources"
- readonly output: AgentLoopLogPreviewMeta
-}
-
-type AgentLoopLogPreviewMeta = {
- readonly truncated: boolean
- readonly preview: string
+function makeHarnessRunResult(text: string): HarnessRunResult {
+ return {
+ manifest: {
+ text,
+ citations: [],
+ artifacts: [],
+ unresolved: [],
+ },
+ trace: {
+ ledger: {
+ retrievalCount: 0,
+ chunks: [],
+ assets: [],
+ evidenceText: [],
+ stopReasons: [],
+ failureReasons: [],
+ decisionTraces: [],
+ },
+ validationErrors: [],
+ revisionsUsed: 0,
+ },
+ };
}
type KnowhereQueryResponseLogMeta = {
@@ -1782,72 +1180,6 @@ type KnowhereQueryResponseLogMeta = {
}[]
}
-function getCapturedAgent(agent: unknown): ToolLoopAgent {
- expect(agent).toBeInstanceOf(ToolLoopAgent)
- return agent as ToolLoopAgent
-}
-
-function getCapturedGenerateInput(
- input: Parameters[0] | undefined,
-): { readonly messages: ModelMessage[] } {
- expect(input).toBeDefined()
- return input as { readonly messages: ModelMessage[] }
-}
-
-function getCapturedAgentSettings(agent: ToolLoopAgent): CapturedAgentSettings {
- return (agent as unknown as { readonly settings: CapturedAgentSettings })
- .settings
-}
-
-function getCapturedAgentTools(agent: ToolLoopAgent): CapturedAgentTools {
- return agent.tools as unknown as CapturedAgentTools
-}
-
-function getTestModelMessagesCharLength(
- messages: readonly ModelMessage[],
-): number {
- return messages.reduce(
- (totalLength, message): number =>
- totalLength + getTestUnknownTextLength(message.content),
- 0,
- )
-}
-
-function getTestUnknownTextLength(value: unknown): number {
- if (typeof value === "string") return value.length
- if (Array.isArray(value)) {
- return value.reduce(
- (totalLength, nestedValue): number =>
- totalLength + getTestUnknownTextLength(nestedValue),
- 0,
- )
- }
- if (!value || typeof value !== "object") return 0
-
- return Object.values(value as Record).reduce(
- (totalLength, nestedValue): number =>
- totalLength + getTestUnknownTextLength(nestedValue),
- 0,
- )
-}
-
-function getSearchSourcesTargetContentSchema(
- tool: CapturedAgentTools["searchSources"],
-): CapturedZodSchema {
- const shape = tool.inputSchema._def?.shape
- const fields = typeof shape === "function" ? shape() : shape
- if (!fields) {
- throw new Error("searchSources input schema should expose fields.")
- }
-
- const targetContentSchema = fields.targetContent
- if (!targetContentSchema) {
- throw new Error("searchSources input schema should include targetContent.")
- }
-
- return targetContentSchema
-}
-
function getLoggerInfoMeta(message: string): Record {
const calls = loggerMock.info.mock.calls as unknown as readonly (readonly [
string,
diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts
index b764648..7c83b62 100644
--- a/src/domains/chat/index.ts
+++ b/src/domains/chat/index.ts
@@ -3,11 +3,20 @@ import type {
RetrievalQueryParams,
RetrievalQueryResponse,
RetrievalResult,
- RetrievalSource,
} from "@ontos-ai/knowhere-sdk"
import { logger } from "@/lib/logger"
-import type { ChatCitationView } from "@/domains/chat/types"
+import type {
+ ChatArtifactView,
+ ChatCitationView,
+} from "@/domains/chat/types"
+import type {
+ EvidenceAsset,
+ EvidenceChunk,
+ HarnessRunResult,
+ OutputArtifact,
+ OutputCitation,
+} from "@/agent-harness"
import {
toChatCitationViews,
useNotebookSourceTitles,
@@ -19,9 +28,6 @@ import type {
AgenticRetrievalResponse,
AnswerQuestionInput,
AnswerQuestionResult,
- ReadRetrievedChunkInput,
- ReadRetrievedChunkResult,
- RetrievedChunkReference,
} from "./contracts"
import {
excludeDocuments,
@@ -29,15 +35,12 @@ import {
} from "./retrieval"
import {
enrichRetrievalResultsWithAssetUrls,
- isImageAssetUrl,
removeRetrievedMediaAssetUrls,
} from "./media-assets"
const DEFAULT_TOP_K = 8
const MAX_AGENTIC_TOP_K = 12
const MAX_CITATION_RESULTS = 20
-const DEFAULT_CHUNK_READ_LIMIT = 4_000
-const MAX_CHUNK_READ_LIMIT = 8_000
const KNOWHERE_RESPONSE_TEXT_LOG_LIMIT = 200
const KNOWHERE_CHUNK_LOG_LIMIT = 100
const NO_RESULTS_ANSWER = "I couldn't find that in your sources."
@@ -56,18 +59,6 @@ const RETRIEVAL_TARGET_CONTENT_DATA_TYPES: Readonly<
type RetrievalDataType = NonNullable
-type StoredRetrievedChunk = {
- id: string
- chunkId: string | null
- kind: RetrievedChunkReference["kind"]
- resultIndex: number | null
- content: string
- chunkType: string
- score: number | null
- source: RetrievalSource
- hasAssetUrl: boolean
-}
-
type KnowhereQueryResponseLog = {
readonly namespace: string
readonly query: string
@@ -101,15 +92,8 @@ export type {
SearchSources,
} from "./contracts"
export {
- buildAgenticChatSystemPrompt,
- buildGroundedPrompt,
- buildRetrievalQueryPrompt,
- generateAgenticGroundedAnswer,
- generateAgenticGroundedAnswerEffect,
- generateContextualRetrievalQuery,
- generateContextualRetrievalQueryEffect,
- generateGroundedAnswer,
- generateGroundedAnswerEffect,
+ generateAgenticOutputManifest,
+ generateAgenticOutputManifestEffect,
} from "./prompt"
export {
parseChatRequestBody,
@@ -123,7 +107,6 @@ export const answerQuestionWithRetrieval = (
Effect.gen(function* () {
const question = input.question.trim()
const retrievalResponses: RetrievalQueryResponse[] = []
- const retrievedChunkContext = createRetrievedChunkContext()
logger.info("chat-agent: answer start", {
questionLength: question.length,
@@ -158,19 +141,11 @@ export const answerQuestionWithRetrieval = (
try {
const response = await input.retrieval.query(retrievalQueryParams)
retrievalResponses.push(response)
- const chunkReferences = retrievedChunkContext.registerResponse({
- response,
- responseIndex: retrievalResponses.length,
- })
logger.info("chat-agent: searchSources ok", {
query: response.query,
durationMs: Date.now() - startedAt,
resultCount: response.results.length,
referencedChunkCount: response.referencedChunks.length,
- readableChunkCount: chunkReferences.length,
- truncatedChunkCount: chunkReferences.filter(
- (reference): boolean => reference.contentTruncated,
- ).length,
stopReason: response.stopReason ?? null,
failureReason: response.failureReason ?? null,
targetContent: retrievalPlan.targetContent,
@@ -179,7 +154,7 @@ export const answerQuestionWithRetrieval = (
durationMs: Date.now() - startedAt,
response: formatKnowhereQueryResponseForLog(response),
})
- return { ...response, chunkReferences, retrievalPlan }
+ return { ...response, retrievalPlan }
} catch (error) {
logger.error("chat-agent: searchSources failed", {
query: retrievalQueryParams.query,
@@ -191,23 +166,6 @@ export const answerQuestionWithRetrieval = (
}
}
- const readRetrievedChunk = async (
- readInput: ReadRetrievedChunkInput,
- ): Promise => {
- const result = retrievedChunkContext.read(readInput)
- logger.info("chat-agent: readRetrievedChunk", {
- id: result.id,
- found: result.found,
- offset: result.offset,
- limit: result.limit,
- contentLength: result.contentLength,
- returnedLength: result.contentSlice.length,
- hasMoreContent: result.hasMoreContent,
- nextOffset: result.nextOffset,
- })
- return result
- }
-
const generatedAnswer = yield* Effect.tryPromise(() =>
input.generateAnswer({
question,
@@ -215,18 +173,23 @@ export const answerQuestionWithRetrieval = (
sources: input.sources,
excludedSourceIds: input.excludedSourceIds,
searchSources,
- readRetrievedChunk,
}),
)
logger.info("chat-agent: answer generated", {
- answerLength: generatedAnswer.length,
+ answerLength: generatedAnswer.manifest.text.length,
retrievalCallCount: retrievalResponses.length,
- registeredChunkCount: retrievedChunkContext.size(),
+ citationCount: generatedAnswer.manifest.citations.length,
+ harnessValidationErrorCount: generatedAnswer.trace.validationErrors.length,
+ revisionsUsed: generatedAnswer.trace.revisionsUsed,
})
- const rawResults = collectRetrievalResults(retrievalResponses, input.sources)
- if (rawResults.length === 0 && generatedAnswer.trim().length === 0) {
+ const rawResults = selectCitationRawResults({
+ generatedAnswer,
+ retrievalResponses,
+ sources: input.sources,
+ })
+ if (rawResults.length === 0 && generatedAnswer.manifest.text.trim().length === 0) {
return { answer: NO_RESULTS_ANSWER, citations: [] as ChatCitationView[] }
}
@@ -239,190 +202,153 @@ export const answerQuestionWithRetrieval = (
}),
)
const answer = sanitizeGeneratedAnswer({
- answer: generatedAnswer,
- question,
- results,
- })
- const citationResults = selectCitationResultsForAnswer({
- question,
+ answer: generatedAnswer.manifest.text,
results,
})
+ const citationResults = results
+ const artifacts = toChatArtifactViewsFromHarness(generatedAnswer, input.sources)
logger.info("chat-agent: answer complete", {
answerLength: answer.length,
citationCount: citationResults.length,
+ artifactCount: artifacts?.length ?? 0,
})
return {
answer,
citations: toChatCitationViews(citationResults, answer),
+ ...(artifacts && artifacts.length > 0 ? { artifacts } : {}),
}
})
-type GeneratedAnswerSanitizerInput = {
- readonly answer: string
- readonly question: string
- readonly results: readonly RetrievalResult[]
-}
-
-function sanitizeGeneratedAnswer({
- answer,
- question,
- results,
-}: GeneratedAnswerSanitizerInput): string {
- const sanitizedAnswer = removeRetrievedMediaAssetUrls(answer, results)
-
- if (
- shouldUseConciseImageRequestAnswer({
- answer: sanitizedAnswer,
- question,
- results,
- })
- ) {
- return buildConciseImageRequestAnswer(question)
- }
-
- return sanitizedAnswer
-}
-
-function shouldUseConciseImageRequestAnswer({
- answer,
- question,
- results,
-}: GeneratedAnswerSanitizerInput): boolean {
- return (
- isShowOrSendImageRequest(question) &&
- !isExplicitPersonalDetailRequest(question) &&
- hasImageCitationResult(results) &&
- shouldSimplifyImageRequestAnswer(answer)
- )
-}
-
-function selectCitationResultsForAnswer(input: {
- readonly question: string
- readonly results: readonly RetrievalResult[]
-}): readonly RetrievalResult[] {
- if (!isShowOrSendImageRequest(input.question)) return input.results
-
- const imageResults = input.results.filter(isImageCitationResult)
- if (imageResults.length === 0) return input.results
-
- const focusedImageResults = filterFocusedImageCitationResults(
- input.question,
- imageResults,
+function toChatArtifactViewsFromHarness(
+ result: HarnessRunResult,
+ sources: readonly AnswerQuestionInput["sources"][number][],
+): ChatArtifactView[] | undefined {
+ const assetsByRef = new Map(
+ result.trace.ledger.assets.map((asset): readonly [string, EvidenceAsset] => [
+ asset.ref,
+ asset,
+ ]),
)
- return focusedImageResults.length > 0 ? focusedImageResults : imageResults
-}
-
-function hasImageCitationResult(results: readonly RetrievalResult[]): boolean {
- return results.some(isImageCitationResult)
-}
-
-function isImageCitationResult(result: RetrievalResult): boolean {
- const assetUrl = result.assetUrl?.trim()
- if (!assetUrl) return false
-
- return result.chunkType.toLowerCase() === "image" || isImageAssetUrl(assetUrl)
-}
-
-function filterFocusedImageCitationResults(
- question: string,
- results: readonly RetrievalResult[],
-): readonly RetrievalResult[] {
- const labelPattern = getFocusedImageCitationLabelPattern(question)
- if (!labelPattern) return results
-
- return results.filter((result): boolean =>
- labelPattern.test(getImageCitationLabel(result)),
+ const chunksByRef = new Map(
+ result.trace.ledger.chunks.map((chunk): readonly [string, EvidenceChunk] => [
+ chunk.ref,
+ chunk,
+ ]),
)
-}
-function getFocusedImageCitationLabelPattern(question: string): RegExp | null {
- if (/身份证|公民身份|居民身份证|\bid card\b|\bidentity card\b/iu.test(question)) {
- return /身份证|居民身份证|\bid card\b|\bidentity card\b/iu
- }
+ const displayLimit = getHarnessArtifactDisplayLimit(result)
+ const artifacts: ChatArtifactView[] = []
+ let displayedArtifactCount = 0
- return null
-}
-
-function getImageCitationLabel(result: RetrievalResult): string {
- return [
- result.source.sourceFileName,
- result.source.sectionPath,
- getAssetPathFromCitationUrl(result.assetUrl),
- ]
- .filter((value): value is string => Boolean(value?.trim()))
- .join(" ")
-}
-
-function getAssetPathFromCitationUrl(assetUrl: string | undefined): string | null {
- if (!assetUrl) return null
+ for (const artifact of result.manifest.artifacts) {
+ const artifactView = resolveHarnessArtifactView({
+ artifact,
+ assetsByRef,
+ chunksByRef,
+ sources,
+ })
+ if (!artifactView) continue
+
+ const isDisplayed = artifactView.display !== false
+ if (
+ isDisplayed &&
+ typeof displayLimit === "number" &&
+ displayedArtifactCount >= displayLimit
+ ) {
+ continue
+ }
- try {
- return decodeURIComponent(new URL(assetUrl).pathname)
- } catch {
- return assetUrl
+ artifacts.push(artifactView)
+ if (isDisplayed) displayedArtifactCount += 1
}
-}
-function isShowOrSendImageRequest(question: string): boolean {
- const normalizedQuestion = question.toLowerCase()
- const hasImageTerm =
- /图片|照片|图像|截图|身份证|\bimage\b|\bimages\b|\bphoto\b|\bphotos\b|\bpicture\b|\bpictures\b|\bscreenshot\b|\bid card\b|\bidentity card\b/u.test(
- normalizedQuestion,
- )
- const hasActionTerm =
- /请将|请把|发送|发给我|发来|给我看|展示|显示|看一下|\bshow\b|\bsend\b|\bdisplay\b|\battach\b|\bgive me\b/u.test(
- normalizedQuestion,
- )
-
- return hasImageTerm && hasActionTerm
+ return artifacts.length > 0 ? artifacts : undefined
}
-function isExplicitPersonalDetailRequest(question: string): boolean {
- return /号码|身份证号|身份号码|住址|地址|出生|有效期限|签发机关|姓名|是什么|多少|\bid number\b|\bidentity number\b|\baddress\b|\bbirth\b|\bissuer\b|\bvalid/u.test(
- question.toLowerCase(),
+function getHarnessArtifactDisplayLimit(result: HarnessRunResult): number | null {
+ const constraints = result.trace.intent?.constraints
+ const limits = [constraints?.desiredCount, constraints?.maxCount].filter(
+ (value): value is number =>
+ typeof value === "number" && Number.isSafeInteger(value) && value > 0,
)
-}
-
-function containsPersonalDetailField(answer: string): boolean {
- return /公民身份号码|身份号码|身份证号|身份证号码|住址|地址|出生日期|出生|有效期限|签发机关|性别|民族|姓名|\bid number\b|\bidentity number\b|\baddress\b|\bdate of birth\b|\bbirth date\b|\bissuer\b|\bissuing authority\b|\bvalid until\b|\bvalid through\b/i.test(
- answer,
- )
-}
-
-function shouldSimplifyImageRequestAnswer(answer: string): boolean {
- const trimmedAnswer = answer.trim()
- return (
- containsPersonalDetailField(trimmedAnswer) ||
- containsMarkdownList(trimmedAnswer) ||
- containsSourceIndexReference(trimmedAnswer) ||
- trimmedAnswer.length > getConciseImageAnswerLengthLimit(trimmedAnswer)
- )
-}
+ return limits.length > 0 ? Math.min(...limits) : null
+}
+
+function resolveHarnessArtifactView(input: {
+ readonly artifact: OutputArtifact
+ readonly assetsByRef: ReadonlyMap
+ readonly chunksByRef: ReadonlyMap
+ readonly sources: readonly AnswerQuestionInput["sources"][number][]
+}): ChatArtifactView | null {
+ const asset = input.assetsByRef.get(input.artifact.ref)
+ if (asset) {
+ return toChatArtifactView({
+ artifact: input.artifact,
+ asset,
+ sources: input.sources,
+ })
+ }
-function containsMarkdownList(value: string): boolean {
- return /\n\s*[-*]\s+/u.test(value)
+ const chunk = input.chunksByRef.get(input.artifact.ref)
+ const chunkAssetRef = chunk?.assetRef
+ const chunkAsset = chunkAssetRef ? input.assetsByRef.get(chunkAssetRef) : null
+ return chunkAsset
+ ? toChatArtifactView({
+ artifact: input.artifact,
+ asset: chunkAsset,
+ sources: input.sources,
+ })
+ : null
}
-function containsSourceIndexReference(value: string): boolean {
- return /\bSource\s+\d+\b/iu.test(value)
+function toChatArtifactView(input: {
+ readonly artifact: OutputArtifact
+ readonly asset: EvidenceAsset
+ readonly sources: readonly AnswerQuestionInput["sources"][number][]
+}): ChatArtifactView {
+ const source = normalizeHarnessSource(input.asset.source, input.sources)
+ return {
+ type: input.artifact.type,
+ ref: input.artifact.ref,
+ display: input.artifact.display,
+ reason: input.artifact.reason,
+ assetUrl: input.asset.assetUrl,
+ label: input.asset.label,
+ citation: {
+ chunkType: input.asset.type,
+ score: null,
+ assetUrl: input.asset.assetUrl,
+ source,
+ },
+ }
}
-function getConciseImageAnswerLengthLimit(answer: string): number {
- return containsCjkText(answer) ? 120 : 220
-}
+function normalizeHarnessSource(
+ source: OutputCitation["source"],
+ sources: readonly AnswerQuestionInput["sources"][number][],
+): ChatCitationView["source"] {
+ const sourceTitle = source.documentId
+ ? sources.find((candidate) => candidate.knowhereDocumentId === source.documentId)
+ ?.title
+ : undefined
-function buildConciseImageRequestAnswer(question: string): string {
- if (containsCjkText(question)) {
- return question.includes("身份证")
- ? "已找到相关身份证图片，见下方图片。"
- : "已找到相关图片，见下方图片。"
+ return {
+ documentId: source.documentId,
+ sourceFileName: sourceTitle ?? source.sourceFileName,
+ sectionPath: source.sectionPath,
}
+}
- return "I found the relevant image. See the image below."
+type GeneratedAnswerSanitizerInput = {
+ readonly answer: string
+ readonly results: readonly RetrievalResult[]
}
-function containsCjkText(value: string): boolean {
- return /[\u3400-\u9fff]/u.test(value)
+function sanitizeGeneratedAnswer({
+ answer,
+ results,
+}: GeneratedAnswerSanitizerInput): string {
+ return removeRetrievedMediaAssetUrls(answer, results)
}
function formatKnowhereQueryResponseForLog(
@@ -540,163 +466,84 @@ function normalizeRetrievalTargetContent(
return value ?? "all"
}
-function createRetrievedChunkContext(): {
- registerResponse(input: {
- readonly response: RetrievalQueryResponse
- readonly responseIndex: number
- }): readonly RetrievedChunkReference[]
- read(input: ReadRetrievedChunkInput): ReadRetrievedChunkResult
- size(): number
-} {
- const chunksById = new Map()
-
- function storeChunk(chunk: StoredRetrievedChunk): void {
- chunksById.set(chunk.id, chunk)
- if (chunk.chunkId && chunk.chunkId !== chunk.id) {
- chunksById.set(chunk.chunkId, chunk)
- }
- }
-
- return {
- registerResponse(input): readonly RetrievedChunkReference[] {
- const references: RetrievedChunkReference[] = []
- input.response.results.forEach((result, index): void => {
- const resultIndex = index + 1
- const chunkId = getRetrievalResultChunkId(result)
- const id = chunkId ?? `search_${input.responseIndex}_result_${resultIndex}`
- const storedChunk: StoredRetrievedChunk = {
- id,
- chunkId,
- kind: "result",
- resultIndex,
- content: result.content,
- chunkType: result.chunkType,
- score: result.score,
- source: result.source,
- hasAssetUrl: Boolean(result.assetUrl),
- }
- storeChunk(storedChunk)
- references.push(toRetrievedChunkReference(storedChunk))
- })
-
- input.response.referencedChunks.forEach((chunk, index): void => {
- const id = chunk.chunkId || `search_${input.responseIndex}_reference_${index + 1}`
- const existingChunk = chunksById.get(id)
- if (existingChunk) {
- references.push(toRetrievedChunkReference(existingChunk))
- return
- }
-
- const storedChunk: StoredRetrievedChunk = {
- id,
- chunkId: chunk.chunkId || null,
- kind: "referencedChunk",
- resultIndex: null,
- content: "",
- chunkType: chunk.chunkType,
- score: null,
- source: {
- documentId: chunk.documentId,
- sourceFileName: null,
- sectionPath: chunk.sectionPath,
- },
- hasAssetUrl: Boolean(chunk.assetUrl),
- }
- storeChunk(storedChunk)
- references.push(toRetrievedChunkReference(storedChunk))
- })
-
- return references
- },
- read(input): ReadRetrievedChunkResult {
- const offset = normalizeChunkReadOffset(input.offset)
- const limit = normalizeChunkReadLimit(input.limit)
- const chunk = chunksById.get(input.id)
- if (!chunk) {
- return {
- id: input.id,
- chunkId: null,
- found: false,
- chunkType: null,
- score: null,
- source: null,
- hasAssetUrl: false,
- offset,
- limit,
- contentLength: 0,
- contentSlice: "",
- hasMoreContent: false,
- nextOffset: null,
- }
- }
-
- const boundedOffset = Math.min(offset, chunk.content.length)
- const endOffset = Math.min(boundedOffset + limit, chunk.content.length)
- return {
- id: chunk.id,
- chunkId: chunk.chunkId,
- found: true,
- chunkType: chunk.chunkType,
- score: chunk.score,
- source: chunk.source,
- hasAssetUrl: chunk.hasAssetUrl,
- offset: boundedOffset,
- limit,
- contentLength: chunk.content.length,
- contentSlice: chunk.content.slice(boundedOffset, endOffset),
- hasMoreContent: endOffset < chunk.content.length,
- nextOffset: endOffset < chunk.content.length ? endOffset : null,
- }
- },
- size(): number {
- return chunksById.size
- },
+function normalizeTopK(value: number | undefined): number {
+ if (typeof value !== "number" || !Number.isSafeInteger(value)) {
+ return DEFAULT_TOP_K
}
+ return Math.min(Math.max(value, 1), MAX_AGENTIC_TOP_K)
}
-function toRetrievedChunkReference(
- chunk: StoredRetrievedChunk,
-): RetrievedChunkReference {
- const contentPreview = chunk.content.slice(0, DEFAULT_CHUNK_READ_LIMIT)
- return {
- id: chunk.id,
- chunkId: chunk.chunkId,
- kind: chunk.kind,
- resultIndex: chunk.resultIndex,
- chunkType: chunk.chunkType,
- score: chunk.score,
- source: chunk.source,
- hasAssetUrl: chunk.hasAssetUrl,
- contentLength: chunk.content.length,
- contentPreview,
- contentTruncated: contentPreview.length < chunk.content.length,
- }
-}
+/**
+ * Display citations come from the agent-curated manifest (the refs it chose to
+ * cite), resolved against the evidence ledger. Only when the agent cited
+ * nothing do we fall back to the full set of retrieved results, so a grounded
+ * answer still shows its sources instead of appearing unsupported.
+ */
+function selectCitationRawResults(input: {
+ readonly generatedAnswer: HarnessRunResult
+ readonly retrievalResponses: readonly RetrievalQueryResponse[]
+ readonly sources: readonly AnswerQuestionInput["sources"][number][]
+}): RetrievalResult[] {
+ const curated = mapManifestCitationsToResults(input.generatedAnswer)
+ if (curated.length > 0) return curated
+ return collectRetrievalResults(input.retrievalResponses, input.sources)
+}
+
+function mapManifestCitationsToResults(
+ result: HarnessRunResult,
+): RetrievalResult[] {
+ const chunksByRef = new Map(
+ result.trace.ledger.chunks.map((chunk): readonly [string, EvidenceChunk] => [
+ chunk.ref,
+ chunk,
+ ]),
+ )
+ const assetsByRef = new Map(
+ result.trace.ledger.assets.map((asset): readonly [string, EvidenceAsset] => [
+ asset.ref,
+ asset,
+ ]),
+ )
-function getRetrievalResultChunkId(result: RetrievalResult): string | null {
- const resultWithChunkId = result as RetrievalResult & {
- readonly chunkId?: string | null
- }
- return resultWithChunkId.chunkId?.trim() || null
-}
+ const results: RetrievalResult[] = []
+ const seenKeys = new Set()
-function normalizeChunkReadOffset(value: number | undefined): number {
- if (typeof value !== "number" || !Number.isSafeInteger(value)) return 0
- return Math.max(value, 0)
-}
+ for (const citation of result.manifest.citations) {
+ const chunk =
+ chunksByRef.get(citation.ref) ??
+ resolveChunkForAssetRef(citation.ref, assetsByRef, chunksByRef)
+ if (!chunk) continue
+
+ const retrievalResult: RetrievalResult = {
+ content: chunk.content,
+ chunkType: chunk.chunkType,
+ score: chunk.score,
+ ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}),
+ source: {
+ documentId: chunk.source.documentId ?? undefined,
+ sourceFileName: chunk.source.sourceFileName ?? undefined,
+ sectionPath: chunk.source.sectionPath ?? undefined,
+ },
+ }
+ const key = getRetrievalResultKey(retrievalResult)
+ if (seenKeys.has(key)) continue
-function normalizeChunkReadLimit(value: number | undefined): number {
- if (typeof value !== "number" || !Number.isSafeInteger(value)) {
- return DEFAULT_CHUNK_READ_LIMIT
+ seenKeys.add(key)
+ results.push(retrievalResult)
+ if (results.length >= MAX_CITATION_RESULTS) break
}
- return Math.min(Math.max(value, 1), MAX_CHUNK_READ_LIMIT)
+
+ return results
}
-function normalizeTopK(value: number | undefined): number {
- if (typeof value !== "number" || !Number.isSafeInteger(value)) {
- return DEFAULT_TOP_K
- }
- return Math.min(Math.max(value, 1), MAX_AGENTIC_TOP_K)
+function resolveChunkForAssetRef(
+ ref: string,
+ assetsByRef: ReadonlyMap,
+ chunksByRef: ReadonlyMap,
+): EvidenceChunk | undefined {
+ const asset = assetsByRef.get(ref)
+ if (!asset) return undefined
+ return chunksByRef.get(asset.chunkRef)
}
function collectRetrievalResults(
diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts
index eba12ad..b13831e 100644
--- a/src/domains/chat/prompt.ts
+++ b/src/domains/chat/prompt.ts
@@ -1,194 +1,39 @@
-import {
- generateText,
- pruneMessages,
- stepCountIs,
- ToolLoopAgent,
- tool,
- type ModelMessage,
- type PrepareStepFunction,
-} from "ai"
import { Effect } from "effect"
-import type {
- RetrievalQueryResponse,
- RetrievalSource,
-} from "@ontos-ai/knowhere-sdk"
-import { z } from "zod"
import { CHAT_MODEL } from "@/lib/ai"
import { logger } from "@/lib/logger"
import type { Source } from "@/infrastructure/db/schema"
import type { ChatCitationView } from "@/domains/chat/types"
+import {
+ runAgentHarness,
+ type AgentTurn,
+ type AgentTurnInput,
+ type HarnessRetrievalRequest,
+ type HarnessRunResult,
+ type TargetModality,
+} from "@/agent-harness"
import type {
AgenticRetrievalQuery,
- AgenticRetrievalResponse,
+ AgenticRetrievalTargetContent,
ChatHistoryMessage,
- ReadRetrievedChunk,
- ReadRetrievedChunkInput,
- ReadRetrievedChunkResult,
- RetrievedChunkReference,
SearchSources,
} from "./contracts"
-import { normalizeRetrievalQuery } from "./retrieval"
const RECENT_CONTEXT_MESSAGE_LIMIT = 8
const CONTEXT_CONTENT_CHAR_LIMIT = 900
-const COMPACTED_HISTORY_MESSAGE_LIMIT = 12
-const COMPACTED_HISTORY_CONTENT_CHAR_LIMIT = 500
-const STORED_HISTORY_MESSAGE_LIMIT = 20
-const STORED_HISTORY_CHAR_BUDGET = 32_000
-const AGENT_STEP_MESSAGE_LIMIT = 20
-const AGENT_STEP_RECENT_MESSAGE_LIMIT = 12
-const AGENT_STEP_CONTEXT_CHAR_BUDGET = 64_000
const SOURCE_CONTEXT_LIMIT = 12
-const AGENTIC_SEARCH_STEP_LIMIT = 5
-const TOOL_EVIDENCE_CHAR_LIMIT = 12_000
-const TOOL_RESULT_CONTENT_CHAR_LIMIT = 1_500
-const TOOL_CHUNK_READ_LIMIT_DEFAULT = 4_000
-const TOOL_CHUNK_READ_LIMIT_MAX = 8_000
-const AGENT_LOOP_TOOL_INPUT_LOG_LIMIT = 1_200
-const AGENT_LOOP_TOOL_LOG_ENTRY_LIMIT = 4
-const AGENT_REQUIRED_SEARCH_STEP_COUNT = 2
-const RAW_URL_PATTERN = /https?:\/\/[^\s)\]}>"']+/g
-const REDACTED_MEDIA_URL = "[media asset URL hidden]"
-type GenerateContextualRetrievalQueryInput = {
- question: string
- messages: readonly ChatHistoryMessage[]
- sources: readonly Source[]
- excludedSourceIds: readonly string[]
-}
-
-type GenerateGroundedAnswerInput = {
- question: string
- retrievalQuery: string
- messages: readonly ChatHistoryMessage[]
- evidenceText: string
- mediaAssetContext?: string
-}
-
-type BuildGroundedPromptInput = {
- question: string
- retrievalQuery?: string
- messages?: readonly ChatHistoryMessage[]
- evidenceText: string
- mediaAssetContext?: string
-}
-
-type GenerateAgenticGroundedAnswerInput = {
+type GenerateAgenticOutputManifestInput = {
question: string
messages: readonly ChatHistoryMessage[]
sources: readonly Source[]
excludedSourceIds: readonly string[]
searchSources: SearchSources
- readRetrievedChunk: ReadRetrievedChunk
-}
-
-type AgenticChatTools = ReturnType
-
-type AgentLoopLogPreview = {
- readonly charLength: number
- readonly truncated: boolean
- readonly preview: string
-}
-
-type AgentLoopToolCallLog = {
- readonly toolName: string
- readonly toolCallId: string | null
- readonly input: AgentLoopLogPreview
-}
-
-type AgentLoopToolOutputLog =
- | AgentLoopLogPreview
- | AgentLoopSearchSourcesOutputLog
- | AgentLoopReadChunkOutputLog
-
-type AgentLoopToolResultLog = {
- readonly toolName: string
- readonly toolCallId: string | null
- readonly output: AgentLoopToolOutputLog
-}
-
-type AgentLoopStepLog = {
- readonly stepNumber: number
- readonly finishReason: string | null
- readonly responseText: string
- readonly responseTextCharLength: number
- readonly toolCallCount: number
- readonly toolCalls: readonly AgentLoopToolCallLog[]
- readonly toolCallsOmitted: number
- readonly toolResultCount: number
- readonly toolResults: readonly AgentLoopToolResultLog[]
- readonly toolResultsOmitted: number
-}
-
-type AgentLoopSearchSourcesOutputLog = {
- readonly kind: "searchSources"
- readonly output: AgentLoopLogPreview
-}
-
-type AgentLoopReadChunkOutputLog = {
- readonly kind: "readRetrievedChunk"
- readonly output: AgentLoopLogPreview
-}
-
-type LlmModelMessageLog = {
- readonly role: string
- readonly contentCharLength: number
- readonly content: unknown
-}
-
-type RetrievalResponseWithDecisionData = AgenticRetrievalResponse & {
- readonly decision_trace?: unknown
- readonly decisionTree?: unknown
- readonly decision_tree?: unknown
-}
-
-type GenerateLoggedTextInput = {
- readonly operation: string
- readonly prompt: string
-}
-
-export const generateContextualRetrievalQueryEffect = (
- input: GenerateContextualRetrievalQueryInput,
-): Effect.Effect =>
- Effect.gen(function* () {
- const question = input.question.trim()
- if (input.messages.length === 0) return question
-
- if (!process.env.AI_GATEWAY_API_KEY) {
- return yield* Effect.die(
- new Error(
- "AI_GATEWAY_API_KEY environment variable is required. " +
- "Set it in your .env.local file.",
- ),
- )
- }
-
- const prompt = buildRetrievalQueryPrompt({
- question,
- messages: input.messages,
- sources: input.sources,
- excludedSourceIds: input.excludedSourceIds,
- })
- const response = yield* Effect.tryPromise(() =>
- generateLoggedText({
- operation: "generateContextualRetrievalQuery",
- prompt,
- }),
- )
- return normalizeRetrievalQuery(response.text, question)
- })
-
-/** Async wrapper for the legacy single-query retrieval flow. */
-export async function generateContextualRetrievalQuery(
- input: GenerateContextualRetrievalQueryInput,
-): Promise {
- return Effect.runPromise(generateContextualRetrievalQueryEffect(input))
}
-export const generateGroundedAnswerEffect = (
- input: GenerateGroundedAnswerInput,
-): Effect.Effect =>
+export const generateAgenticOutputManifestEffect = (
+ input: GenerateAgenticOutputManifestInput,
+): Effect.Effect =>
Effect.gen(function* () {
if (!process.env.AI_GATEWAY_API_KEY) {
return yield* Effect.die(
@@ -199,1074 +44,111 @@ export const generateGroundedAnswerEffect = (
)
}
- const response = yield* Effect.tryPromise(() =>
- generateLoggedText({
- operation: "generateGroundedAnswer",
- prompt: buildGroundedPrompt(input),
- }),
- )
- return response.text.trim()
- })
-
-/** Async wrapper for the legacy single-response answer flow. */
-export async function generateGroundedAnswer(
- input: GenerateGroundedAnswerInput,
-): Promise {
- return Effect.runPromise(generateGroundedAnswerEffect(input))
-}
-
-export const generateAgenticGroundedAnswerEffect = (
- input: GenerateAgenticGroundedAnswerInput,
-): Effect.Effect =>
- Effect.gen(function* () {
- if (!process.env.AI_GATEWAY_API_KEY) {
- return yield* Effect.die(
- new Error(
- "AI_GATEWAY_API_KEY environment variable is required. " +
- "Set it in your .env.local file.",
- ),
- )
- }
-
- const agent = buildAgenticChatAgent(input)
- const messages = buildAgenticChatMessages(input)
- logger.info("chat-agent: llm request", {
- operation: "generateAgenticGroundedAnswer.initial",
+ const turn = buildNotebookHarnessTurn(input)
+ logger.info("chat-agent: harness request", {
+ operation: "generateAgenticOutputManifest.initial",
model: CHAT_MODEL,
- promptType: "messages",
- messageCount: messages.length,
- messages: formatModelMessagesForLlmLog(messages),
+ surface: turn.surface,
+ recentTurnCount: turn.recentTurns.length,
+ messageCharLength: turn.userText.length,
})
- const response = yield* Effect.tryPromise(async () => {
- const generationResponse = await agent.generate({ messages })
- logger.info("chat-agent: llm response", {
- operation: "generateAgenticGroundedAnswer.final",
- model: CHAT_MODEL,
- responseTextCharLength: generationResponse.text.length,
- responseText: redactRawUrls(generationResponse.text),
- })
- return generationResponse
- })
- return response.text.trim()
- })
-export async function generateAgenticGroundedAnswer(
- input: GenerateAgenticGroundedAnswerInput,
-): Promise {
- return Effect.runPromise(generateAgenticGroundedAnswerEffect(input))
-}
-
-async function generateLoggedText(
- input: GenerateLoggedTextInput,
-): Promise>> {
- logger.info("chat-agent: llm request", {
- operation: input.operation,
- model: CHAT_MODEL,
- promptType: "text",
- promptCharLength: input.prompt.length,
- prompt: redactRawUrls(input.prompt),
- })
- const response = await generateText({
- model: CHAT_MODEL,
- prompt: input.prompt,
- })
- logger.info("chat-agent: llm response", {
- operation: input.operation,
- model: CHAT_MODEL,
- responseTextCharLength: response.text.length,
- responseText: redactRawUrls(response.text),
- })
- return response
-}
-
-export function buildRetrievalQueryPrompt(
- input: GenerateContextualRetrievalQueryInput,
-): string {
- const sourceContext = formatSourceContext(input.sources, input.excludedSourceIds)
- const conversationContext = formatConversationContext(input.messages)
-
- return [
- "You prepare one search query for the Knowhere SDK retrieval API.",
- "Knowhere retrieval is stateless: it only sees the query string you return and does not know the chat history.",
- "Rewrite the user's latest question into a self-contained retrieval query by adding missing document, company, topic, date, or section context from the recent conversation.",
- "If the latest question already has enough context, keep it concise and close to the user's wording.",
- "Do not answer the question. Return only the retrieval query text.",
- "",
- "Searchable sources:",
- sourceContext,
- "",
- "Recent conversation:",
- conversationContext,
- "",
- `Latest user question: ${input.question}`,
- "",
- "Retrieval query:",
- ].join("\n")
-}
-
-export function buildGroundedPrompt(input: BuildGroundedPromptInput): string {
- const retrievalQuery = input.retrievalQuery?.trim() || input.question
- const conversationContext = formatConversationContext(input.messages ?? [])
- const mediaAssetContext = input.mediaAssetContext?.trim()
-
- const promptLines = [
- "You answer user questions.",
- "Use the retrieved evidence as your primary context.",
- "Cite document sections (e.g. [文档名 / 章节名]) when they support a claim.",
- "When retrieved image or table asset references are relevant to the user's request, cite the matching source label; the UI renders media from citation metadata.",
- "Do not write raw media asset URLs in the answer. They are internal metadata only.",
- "Never output JSON metadata blocks for citations, images, tables, or media.",
- "Never mention asset_id, assetUrl, raw URLs, chunk ids, request-local ids, or retrieval internals.",
- "For image requests, answer briefly and let the UI render images from citation metadata.",
- "For send/show image requests, do not transcribe personal details from the image; do not list identity numbers, addresses, birth dates, or document fields unless the user explicitly asks for those details.",
- "Do not invent asset URLs; use only the retrieved media asset references listed below.",
- "If the sources are related but incomplete, answer what you can and briefly say what is not covered.",
- "Do not invent document-specific facts that are not in the sources.",
- "Use the recent conversation only to resolve references like \"this document\"; do not use it as factual evidence.",
- "Answer in a natural, friendly, and direct tone.",
- "Use GitHub-flavored Markdown when it improves readability, such as short lists, tables, or code blocks. Keep simple answers as plain sentences.",
- "Start with the answer first. Avoid meta phrases like \"Based on the sources\" or \"Based on the source excerpts\" unless the user asks how you know.",
- "Use plain language.",
- "Keep answers concise by default: 1-3 short paragraphs unless the user asks for detail.",
- "CITATION FORMAT: Cite evidence by document and section path, e.g. [文档名 / 章节名].",
- "",
- `Question: ${input.question}`,
- `Retrieval query used: ${retrievalQuery}`,
- "",
- "Recent conversation:",
- conversationContext,
- "",
- "Retrieved evidence:",
- input.evidenceText,
- ]
-
- if (mediaAssetContext) {
- promptLines.push(
- "",
- "Retrieved media asset references (internal; do not quote raw URLs):",
- mediaAssetContext,
- )
- }
-
- return promptLines.join("\n")
-}
-
-export function buildAgenticChatSystemPrompt(
- input: Pick<
- GenerateAgenticGroundedAnswerInput,
- "messages" | "sources" | "excludedSourceIds"
- >,
-): string {
- const sourceContext = formatSourceContext(input.sources, input.excludedSourceIds)
-
- return [
- "Role",
- "You are a Notebook research agent that answers user questions from their uploaded sources.",
- "Use retrieved source evidence as the factual source of truth. Do not invent document-specific facts.",
- "",
- "Retrieval strategy",
- "You have two tools: searchSources and readRetrievedChunk.",
- "Use searchSources for source discovery. Its markdown output gives guidance, evidence, result previews, and Read IDs.",
- "Use readRetrievedChunk only when a relevant search result preview is too short and the markdown output shows a Read ID.",
- "Treat tool output like source notes from a remote index: inspect it, reason over it, then decide whether to answer, search again, or read more.",
- "",
- "Tool use rules",
- "1. Always call searchSources before writing a final answer.",
- "2. Make a second searchSources call before answering to double-check the retrieved data. Reuse the same core query or refine it with entities, document names, section paths, file paths, content types, or failure hints from the first output.",
- "3. Choose the content target from the user's request: broad questions use broad or text-only search, image requests use image or text+image search, and table requests use table or text+table search.",
- "4. Do not paste raw prior messages into searchSources.query. The query must be concise and contain only distilled search terms such as document title, person, topic, date, section path, or asset kind.",
- "5. Use one response to guide the next query: carry forward discovered people, organizations, document names, section paths, file paths, content types, and failure hints.",
- "6. After the verification search, if the markdown guidance says the evidence is useful and the evidence/results directly support the answer, stop searching and answer.",
- "7. If results are missing, weak, or do not cover the requested entity/topic/media/table, search again with a broader or more specific query.",
- "8. Use readRetrievedChunk selectively; do not read every result when the previews already answer the question.",
- "9. Stop after enough evidence or when further searches are unlikely to help; then clearly say what was not found and what retrieval context was missing.",
- "",
- "Media/table handling",
- "For image requests, search visual content directly or combine text and image evidence. If an initial text result identifies a relevant person or section but not an image asset, query again with that person/section plus the requested image concept, e.g. identity card / 身份证 / 公民身份证明.",
- "For table requests, search table content directly or combine text and table evidence.",
- "When retrieved image or table assets are relevant, cite the matching source label; the UI renders media from citation metadata.",
- "Do not invent asset URLs or describe hidden asset metadata.",
- "",
- "Final answer contract",
- "Conversation context is supplied as managed model messages. Use it only to resolve references like \"this document\" or \"those images\".",
- "Cite document sections in the answer, e.g. [文档名 / 章节名].",
- "Use existing [Source N: label] labels only when they are the clearest available citation form.",
- "Never output JSON metadata blocks for citations, images, tables, or media.",
- "Never mention asset_id, assetUrl, raw URLs, chunk ids, Read IDs, tool parameters, or retrieval internals.",
- "For image requests, answer briefly and let the UI render images from citation metadata.",
- "For send/show image requests, do not transcribe personal details from the image; do not list identity numbers, addresses, birth dates, or document fields unless the user explicitly asks for those details.",
- "Do not add unrelated personal details for send/show image requests unless the user asks.",
- "Use GitHub-flavored Markdown when it improves readability, such as short lists, tables, or code blocks. Keep simple answers as plain sentences.",
- "Start with the answer first. Keep answers concise unless the user asks for detail.",
- "",
- "Searchable sources",
- sourceContext,
- ].join("\n")
-}
-
-function buildAgenticChatAgent(
- input: GenerateAgenticGroundedAnswerInput,
-): ToolLoopAgent {
- const instructions = buildAgenticChatSystemPrompt(input)
- return new ToolLoopAgent({
- model: CHAT_MODEL,
- instructions,
- tools: buildAgenticChatTools(input),
- stopWhen: stepCountIs(AGENTIC_SEARCH_STEP_LIMIT),
- prepareStep: buildAgenticPrepareStep(instructions),
- onStepFinish: (event) => {
- logger.info("chat-agent: llm response", {
- operation: "generateAgenticGroundedAnswer.step",
+ const result = yield* Effect.tryPromise(() =>
+ runAgentHarness({
model: CHAT_MODEL,
- stepNumber: event.stepNumber,
- finishReason: event.finishReason,
- responseTextCharLength: event.text.length,
- responseText: redactRawUrls(event.text),
- toolCallCount: event.toolCalls.length,
- toolCalls: formatAgentLoopToolCalls(event.toolCalls),
- toolCallsOmitted: getOmittedAgentLoopEntryCount(event.toolCalls),
- toolResultCount: event.toolResults.length,
- toolResults: formatAgentLoopToolResults(event.toolResults),
- toolResultsOmitted: getOmittedAgentLoopEntryCount(event.toolResults),
- inputTokens: event.usage.inputTokens,
- outputTokens: event.usage.outputTokens,
- totalTokens: event.usage.totalTokens,
- })
- },
- onFinish: (event) => {
- logger.info("chat-agent: loop finished", {
- stepCount: event.steps.length,
- finishReason: event.finishReason,
- responseTextCharLength: event.text.length,
- responseText: redactRawUrls(event.text),
- steps: event.steps.map(formatAgentLoopStep),
- toolNames: Array.from(
- new Set(
- event.steps.flatMap((step) =>
- step.toolCalls.map((toolCall) => toolCall.toolName),
- ),
- ),
- ),
- inputTokens: event.totalUsage.inputTokens,
- outputTokens: event.totalUsage.outputTokens,
- totalTokens: event.totalUsage.totalTokens,
- })
- },
- })
-}
-
-function buildAgenticChatTools(
- input: Pick<
- GenerateAgenticGroundedAnswerInput,
- "searchSources" | "readRetrievedChunk"
- >,
-) {
- return {
- searchSources: tool({
- description:
- "Search the user's Notebook sources through Knowhere retrieval. " +
- "It returns markdown source notes with guidance, evidence, previews, " +
- "and Read IDs for follow-up reads. Use it before answering and call it " +
- "again with refined text, media, or section-path queries when evidence is missing or weak.",
- inputSchema: z.object({
- query: z
- .string()
- .min(1)
- .describe(
- "A concise, self-contained retrieval query. Do not paste raw chat history or previous messages. Use only distilled terms such as document title, person, topic, date, section path, or asset kind when needed.",
- ),
- targetContent: z
- .enum([
- "all",
- "text",
- "image",
- "table",
- "text_image",
- "text_table",
- ])
- .optional()
- .describe(
- "The content type to retrieve: all, text, image, table, text_image, or text_table. Omit only when all content types are useful.",
- ),
- purpose: z
- .string()
- .min(1)
- .max(240)
- .optional()
- .describe(
- "Short reason this query is needed, such as finding an entity, locating an image asset, or verifying a citation.",
- ),
- topK: z
- .number()
- .int()
- .min(1)
- .max(12)
- .optional()
- .describe("Number of chunks to return. Defaults to 8."),
- signalPaths: z
- .array(z.string().min(1))
- .max(8)
- .optional()
- .describe(
- "Optional section/path keywords when a previous result points to a useful section.",
- ),
- filterMode: z
- .enum(["keep", "delete"])
- .optional()
- .describe(
- "How to apply signalPaths. Use keep to focus on matching paths, delete to exclude them.",
- ),
- threshold: z
- .number()
- .min(0)
- .max(1)
- .optional()
- .describe("Optional minimum retrieval score threshold."),
- }),
- execute: async (queryInput: AgenticRetrievalQuery) => {
- const output = buildRetrievalToolOutput(
- await input.searchSources(queryInput),
- )
- logToolMarkdownOutput("searchSources", output)
- return output
- },
- }),
- readRetrievedChunk: tool({
- description:
- "Read an offset/limit content slice from a Read ID shown in searchSources markdown. " +
- "Use this when a returned result preview is relevant and you want more data before answering.",
- inputSchema: z.object({
- id: z
- .string()
- .min(1)
- .describe(
- "The Read ID shown in searchSources markdown for a relevant result.",
- ),
- offset: z
- .number()
- .int()
- .min(0)
- .optional()
- .describe("Character offset to start reading from. Defaults to 0."),
- limit: z
- .number()
- .int()
- .min(1)
- .max(TOOL_CHUNK_READ_LIMIT_MAX)
- .optional()
- .describe(
- `Maximum characters to return. Defaults to ${TOOL_CHUNK_READ_LIMIT_DEFAULT}; max ${TOOL_CHUNK_READ_LIMIT_MAX}.`,
- ),
- }),
- execute: async (readInput: ReadRetrievedChunkInput) => {
- const output = buildRetrievedChunkToolOutput(
- await input.readRetrievedChunk(readInput),
- )
- logToolMarkdownOutput("readRetrievedChunk", output)
- return output
- },
- }),
- } as const
-}
-
-function buildAgenticPrepareStep(
- instructions: string,
-): PrepareStepFunction {
- return ({ stepNumber, messages }) => {
- const managedMessages = buildAgentStepMessages(messages)
- if (stepNumber < AGENT_REQUIRED_SEARCH_STEP_COUNT) {
- const stepInput = {
- messages: managedMessages,
- toolChoice: {
- type: "tool" as const,
- toolName: "searchSources" as const,
+ turn,
+ retrieval: {
+ query: (request) =>
+ input.searchSources(toAgenticRetrievalQuery(request)),
},
- activeTools: ["searchSources" as const],
- }
- logAgentStepLlmRequest({
- stepNumber,
- instructions,
- messages: managedMessages,
- toolChoice: stepInput.toolChoice,
- activeTools: stepInput.activeTools,
- })
- return stepInput
- }
+ }),
+ )
- logAgentStepLlmRequest({
- stepNumber,
- instructions,
- messages: managedMessages,
- toolChoice: null,
- activeTools: null,
+ logger.info("chat-agent: harness response", {
+ operation: "generateAgenticOutputManifest.final",
+ model: CHAT_MODEL,
+ answerLength: result.manifest.text.length,
+ citationCount: result.manifest.citations.length,
+ artifactCount: result.manifest.artifacts.length,
+ unresolvedCount: result.manifest.unresolved.length,
+ validationErrorCount: result.trace.validationErrors.length,
+ intentTask: result.trace.intent?.task ?? null,
+ carryHistory: result.trace.contextPolicy?.carryHistory ?? null,
})
- return { messages: managedMessages }
- }
-}
-
-function formatAgentLoopStep(step: unknown, index: number): AgentLoopStepLog {
- const record = getRecordFromUnknown(step)
- const toolCalls = getRecordArray(record, "toolCalls")
- const toolResults = getRecordArray(record, "toolResults")
- const responseText = getRecordString(record, "text") ?? ""
- return {
- stepNumber:
- getRecordNumber(record, "stepNumber") ??
- getRecordNumber(record, "stepIndex") ??
- index + 1,
- finishReason: getRecordString(record, "finishReason"),
- responseText: redactRawUrls(responseText),
- responseTextCharLength: responseText.length,
- toolCallCount: toolCalls.length,
- toolCalls: formatAgentLoopToolCalls(toolCalls),
- toolCallsOmitted: getOmittedAgentLoopEntryCount(toolCalls),
- toolResultCount: toolResults.length,
- toolResults: formatAgentLoopToolResults(toolResults),
- toolResultsOmitted: getOmittedAgentLoopEntryCount(toolResults),
- }
-}
-
-function formatAgentLoopToolCalls(
- toolCalls: readonly unknown[],
-): readonly AgentLoopToolCallLog[] {
- return toolCalls
- .slice(0, AGENT_LOOP_TOOL_LOG_ENTRY_LIMIT)
- .map(formatAgentLoopToolCall)
-}
-
-function formatAgentLoopToolCall(toolCall: unknown): AgentLoopToolCallLog {
- const record = getRecordFromUnknown(toolCall)
- return {
- toolName: getRecordString(record, "toolName") ?? "unknown",
- toolCallId: getRecordString(record, "toolCallId"),
- input: buildAgentLoopPreview(
- getFirstRecordValue(record, ["input", "args", "arguments"]),
- AGENT_LOOP_TOOL_INPUT_LOG_LIMIT,
- ),
- }
-}
-
-function formatAgentLoopToolResults(
- toolResults: readonly unknown[],
-): readonly AgentLoopToolResultLog[] {
- return toolResults
- .slice(0, AGENT_LOOP_TOOL_LOG_ENTRY_LIMIT)
- .map(formatAgentLoopToolResult)
-}
-
-function formatAgentLoopToolResult(toolResult: unknown): AgentLoopToolResultLog {
- const record = getRecordFromUnknown(toolResult)
- const toolName = getRecordString(record, "toolName") ?? "unknown"
- return {
- toolName,
- toolCallId: getRecordString(record, "toolCallId"),
- output: formatAgentLoopToolOutput(
- toolName,
- getFirstRecordValue(record, ["output", "result", "content"]),
- ),
- }
-}
-
-function formatAgentLoopToolOutput(
- toolName: string,
- output: unknown,
-): AgentLoopToolOutputLog {
- if (toolName === "searchSources") {
- return {
- kind: "searchSources",
- output: buildAgentLoopFullMarkdownPreview(output),
- }
- }
- if (toolName === "readRetrievedChunk") {
- return {
- kind: "readRetrievedChunk",
- output: buildAgentLoopFullMarkdownPreview(output),
- }
- }
- return buildAgentLoopFullPreview(output)
-}
-
-function getOmittedAgentLoopEntryCount(entries: readonly unknown[]): number {
- return Math.max(0, entries.length - AGENT_LOOP_TOOL_LOG_ENTRY_LIMIT)
-}
-
-function logAgentStepLlmRequest(input: {
- readonly stepNumber: number
- readonly instructions: string
- readonly messages: readonly ModelMessage[]
- readonly toolChoice: unknown
- readonly activeTools: readonly string[] | null
-}): void {
- logger.info("chat-agent: llm request", {
- operation: "generateAgenticGroundedAnswer.step",
- model: CHAT_MODEL,
- promptType: "messages",
- stepNumber: input.stepNumber,
- instructionsCharLength: input.instructions.length,
- instructions: redactRawUrls(input.instructions),
- messageCount: input.messages.length,
- messages: formatModelMessagesForLlmLog(input.messages),
- toolChoice: input.toolChoice,
- activeTools: input.activeTools,
+ return result
})
-}
-function formatModelMessagesForLlmLog(
- messages: readonly ModelMessage[],
-): readonly LlmModelMessageLog[] {
- return messages.map(formatModelMessageForLlmLog)
-}
-
-function formatModelMessageForLlmLog(message: ModelMessage): LlmModelMessageLog {
- return {
- role: message.role,
- contentCharLength: getUnknownTextLength(message.content),
- content: redactRawUrlsFromUnknown(message.content),
- }
-}
-
-function buildAgentLoopPreview(
- value: unknown,
- limit: number,
-): AgentLoopLogPreview {
- const normalized = redactRawUrls(stringifyAgentLoopLogValue(value))
- .replace(/\s+/g, " ")
- .trim()
- const truncated = normalized.length > limit
- return {
- charLength: normalized.length,
- truncated,
- preview: truncated ? `${normalized.slice(0, limit)}...` : normalized,
- }
+export async function generateAgenticOutputManifest(
+ input: GenerateAgenticOutputManifestInput,
+): Promise {
+ return Effect.runPromise(generateAgenticOutputManifestEffect(input))
}
-function buildAgentLoopFullPreview(value: unknown): AgentLoopLogPreview {
- const normalized = redactRawUrls(stringifyAgentLoopLogValue(value))
+function buildNotebookHarnessTurn(
+ input: GenerateAgenticOutputManifestInput,
+): AgentTurnInput {
return {
- charLength: normalized.length,
- truncated: false,
- preview: normalized,
- }
-}
-
-function buildAgentLoopFullMarkdownPreview(value: unknown): AgentLoopLogPreview {
- const normalized = redactRawUrls(stringifyAgentLoopLogValue(value))
- return {
- charLength: normalized.length,
- truncated: false,
- preview: normalized,
- }
-}
-
-function stringifyAgentLoopLogValue(value: unknown): string {
- if (typeof value === "string") return value
- if (value === undefined) return "undefined"
- if (typeof value === "function") {
- return `[Function ${value.name || "anonymous"}]`
- }
- if (typeof value === "symbol") return value.toString()
-
- const json = JSON.stringify(value, createAgentLoopLogJsonReplacer())
- return json ?? String(value)
-}
-
-function createAgentLoopLogJsonReplacer(): (
- key: string,
- value: unknown,
-) => unknown {
- const seenObjects = new WeakSet