Fix/image render

src/mcp/runtime/PluginToToolBridge.ts

@@ -10,17 +10,19 @@
  *          reflected onto the PilotDeck tool flags so the permission
  *          engine can decide whether to ask.
  *
- * Result transformation (M14): we currently emit a single `json` result
- * block. The existing `ToolRuntime` already truncates oversized payloads
- * via `maxResultBytes`; deferring the persisted-large-blob path for now
- * (recorded as `intentional_difference` in the parity table).
+ * Result transformation (M14): MCP ContentBlock types `text` and `image`
+ * are mapped to their PilotDeck equivalents so that images (e.g. Playwright
+ * screenshots) render inline in the chat UI. Remaining block types
+ * (`audio`, `resource`, `resource_link`) fall through as a single `json`
+ * block until the downstream pipeline supports them.
  */
 
 import { PilotDeckToolRuntimeError } from "../../tool/protocol/errors.js";
 import type {
   PilotDeckToolDefinition,
   PilotDeckToolExecutionOutput,
   PilotDeckToolInputSchema,
+  PilotDeckToolResultContent,
 } from "../../tool/index.js";
 import type { McpClient } from "../client/McpClient.js";
 import type { McpRuntime } from "./McpRuntime.js";
@@ -85,7 +87,7 @@ function buildToolDefinition(
           );
         }
         return {
-          content: [{ type: "json", value: content }],
+          content: marshalMcpContent(content),
           data: content,
           metadata: {
             mcp: { serverId: spec.serverId, toolName: spec.toolName, wireName: spec.wireName },
@@ -118,6 +120,48 @@ function buildToolDefinition(
   };
 }
 
+type McpContentBlock = { type: string; [key: string]: unknown };
+
+/**
+ * Map MCP `ContentBlock[]` → `PilotDeckToolResultContent[]`.
+ *
+ * `TextContent`  → `{ type: "text" }`
+ * `ImageContent` → `{ type: "image" }` (renders inline in chat)
+ * Everything else falls through as a single `json` block.
+ */
+function marshalMcpContent(raw: unknown): PilotDeckToolResultContent[] {
+  if (!Array.isArray(raw)) return [{ type: "json", value: raw }];
+
+  const result: PilotDeckToolResultContent[] = [];
+  const remainder: unknown[] = [];
+
+  for (const block of raw as McpContentBlock[]) {
+    if (!block || typeof block !== "object" || typeof block.type !== "string") {
+      remainder.push(block);
+      continue;
+    }
+    if (block.type === "text" && typeof block.text === "string") {
+      result.push({ type: "text", text: block.text });
+    } else if (
+      block.type === "image" &&
+      typeof block.data === "string" &&
+      typeof block.mimeType === "string"
+    ) {
+      result.push({ type: "image", mimeType: block.mimeType as string, data: block.data as string });
+    } else {
+      remainder.push(block);
+    }
+  }
+
+  if (remainder.length > 0) {
+    result.push({ type: "json", value: remainder });
+  }
+  if (result.length === 0) {
+    result.push({ type: "json", value: raw });
+  }
+  return result;
+}
+
 function extractMcpErrorText(
   content: unknown,
   serverId: string,

src/model/index.ts

@@ -67,6 +67,7 @@ export {
   type StructuredOutputExtractionError,
 } from "./structuredOutput/extractStructuredOutput.js";
 export type { ModelCapabilities } from "./protocol/capabilities.js";
+export { downgradeUnsupportedContent } from "./protocol/multimodal.js";
 export type { InputModality, MultimodalConstraints } from "./protocol/multimodal.js";
 export {
   ModelConfigError,

src/model/protocol/multimodal.ts

@@ -1,5 +1,9 @@
 import { ModelRequestError } from "./errors.js";
-import type { CanonicalContentBlock } from "./canonical.js";
+import type {
+  CanonicalContentBlock,
+  CanonicalMessage,
+  CanonicalToolResultContentBlock,
+} from "./canonical.js";
 
 export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio"] as const;
 
@@ -24,6 +28,69 @@ export function isInputModality(value: unknown): value is InputModality {
   return typeof value === "string" && SUPPORTED_INPUT_MODALITIES.includes(value as InputModality);
 }
 
+/**
+ * Pre-flight downgrade: replace media blocks the target model cannot accept
+ * with descriptive text placeholders.  Mutates `messages` in-place so the
+ * caller's cloned request is updated without copying the entire array.
+ *
+ * Only tool_result sub-blocks and top-level content blocks are converted;
+ * tool_call / thinking blocks are left untouched.
+ */
+export function downgradeUnsupportedContent(
+  messages: CanonicalMessage[],
+  constraints: MultimodalConstraints,
+): void {
+  const allowed = new Set<InputModality>(constraints.input);
+  if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio")) return;
+
+  for (const msg of messages) {
+    for (let i = 0; i < msg.content.length; i++) {
+      const block = msg.content[i];
+
+      if (block.type === "tool_result") {
+        let changed = false;
+        const newContent: CanonicalToolResultContentBlock[] = [];
+        for (const sub of block.content) {
+          const placeholder = mediaBlockToPlaceholder(sub, allowed);
+          if (placeholder) {
+            newContent.push({ type: "text", text: placeholder });
+            changed = true;
+          } else {
+            newContent.push(sub);
+          }
+        }
+        if (changed) {
+          (block as { content: CanonicalToolResultContentBlock[] }).content = newContent;
+        }
+        continue;
+      }
+
+      const placeholder = mediaBlockToPlaceholder(block, allowed);
+      if (placeholder) {
+        (msg.content as CanonicalContentBlock[])[i] = { type: "text", text: placeholder };
+      }
+    }
+  }
+}
+
+function mediaBlockToPlaceholder(
+  block: CanonicalContentBlock | CanonicalToolResultContentBlock,
+  allowed: Set<InputModality>,
+): string | undefined {
+  if (block.type === "image" && !allowed.has("image")) {
+    const sizeHint = block.bytes ? `, ${Math.round(block.bytes / 1024)}KB` : "";
+    return `[Image: ${block.mimeType}${sizeHint} — omitted, model does not support image input]`;
+  }
+  if (block.type === "pdf" && !allowed.has("pdf")) {
+    const pagesHint = block.pages ? `, ${block.pages} pages` : "";
+    return `[PDF: ${block.mimeType}, ${Math.round(block.bytes / 1024)}KB${pagesHint} — omitted, model does not support PDF input]`;
+  }
+  if (block.type === "audio" && !allowed.has("audio")) {
+    return `[Audio: ${block.mimeType} — omitted, model does not support audio input]`;
+  }
+  return undefined;
+}
+
 export function contentBlockToInputModality(block: CanonicalContentBlock): InputModality | undefined {
   switch (block.type) {
     case "text":

src/model/request/validateModelRequest.ts

@@ -1,6 +1,6 @@
 import type { CanonicalModelRequest, ModelConfig, ModelDefinition, ProviderConfig } from "../protocol/canonical.js";
 import { ModelRequestError } from "../protocol/errors.js";
-import { assertContentSupported } from "../protocol/multimodal.js";
+import { assertContentSupported, downgradeUnsupportedContent } from "../protocol/multimodal.js";
 
 export type ResolvedModelRequest = {
   provider: ProviderConfig;
@@ -39,6 +39,8 @@ export function validateModelRequest(
     throw new ModelRequestError("unsupported_tool_use", `Model ${request.model} does not support tools.`);
   }
 
+  downgradeUnsupportedContent(request.messages, model.multimodal);
+
   for (const message of request.messages) {
     assertContentSupported(message.content, model.multimodal);
   }

src/web/server/readSessionMessages.ts

@@ -319,6 +319,12 @@ function flushBlock(
       const resultText = flattenToolResultBlockText(block);
       const errorCode = readToolResultErrorCode(block.raw);
       const planData = readPlanData(block.raw);
+      const resultImages: NonNullable<WebMessage["images"]> = [];
+      for (const sub of block.content) {
+        if (sub.type === "image") {
+          resultImages.push(toWebMessageImage(sub));
+        }
+      }
       out.push({
         id: `${context.sessionKey}-tool-${block.toolCallId}-result`,
         sessionKey: context.sessionKey,
@@ -332,6 +338,7 @@ function flushBlock(
         text: resultText,
         ...(errorCode ? { errorCode } : {}),
         ...(planData ? { payload: planData } : {}),
+        ...(resultImages.length > 0 ? { images: resultImages } : {}),
         source: "history",
       });
       return;

ui/src/components/chat-v2/MessageRowV2.tsx

@@ -272,7 +272,7 @@ function MessageRowV2({
                 </div>
               ) : null}
               {formattedContent ? (
-                <Markdown className="min-w-0 break-words [overflow-wrap:anywhere]">{formattedContent}</Markdown>
+                <Markdown className="min-w-0 break-words [overflow-wrap:anywhere]" projectName={selectedProject?.name}>{formattedContent}</Markdown>
               ) : null}
             </>
           )}
@@ -296,7 +296,7 @@ function MessageRowV2({
           <AlertTriangle className="h-3.5 w-3.5" strokeWidth={2} />
         </div>
         <div className="min-w-0 flex-1 pt-0.5 text-[14px] leading-relaxed text-red-500">
-          <Markdown>{formattedContent}</Markdown>
+          <Markdown projectName={selectedProject?.name}>{formattedContent}</Markdown>
         </div>
       </div>,
     );
@@ -312,7 +312,7 @@ function MessageRowV2({
             <span>{t('thinking.title', { defaultValue: 'Thinking...' })}</span>
           </summary>
           <div className="mt-1.5 border-l-2 border-neutral-300 pl-3 text-[13px] text-neutral-500 dark:border-neutral-700 dark:text-neutral-400">
-            <Markdown>{formattedContent}</Markdown>
+            <Markdown projectName={selectedProject?.name}>{formattedContent}</Markdown>
           </div>
         </details>
       </div>,
@@ -326,7 +326,7 @@ function MessageRowV2({
         <span className="inline-block h-4 w-2 animate-pulse bg-neutral-400 dark:bg-neutral-500" />
       ) : (
         <>
-          <Markdown className="prose prose-sm prose-neutral max-w-none dark:prose-invert prose-headings:mb-2 prose-headings:mt-4 prose-h2:text-lg prose-h3:text-base prose-p:my-2 prose-pre:my-3 prose-ol:my-2 prose-ul:my-2 prose-table:my-0 prose-hr:my-4">{formattedContent}</Markdown>
+          <Markdown className="prose prose-sm prose-neutral max-w-none dark:prose-invert prose-headings:mb-2 prose-headings:mt-4 prose-h2:text-lg prose-h3:text-base prose-p:my-2 prose-pre:my-3 prose-ol:my-2 prose-ul:my-2 prose-table:my-0 prose-hr:my-4" projectName={selectedProject?.name}>{formattedContent}</Markdown>
           {formattedContent.trim() &&
            (!nextMessage || nextMessage.type === 'user' || nextMessage.type === 'error') ? (
             <div className="mt-1.5 flex justify-end">

ui/src/components/chat/view/subcomponents/Markdown.tsx

@@ -12,6 +12,8 @@ import { copyTextToClipboard } from '../../../../utils/clipboard';
 type MarkdownProps = {
   children: React.ReactNode;
   className?: string;
+  /** When set, relative image paths are resolved via the project files API. */
+  projectName?: string;
 };
 
 type CodeBlockProps = {
@@ -116,7 +118,14 @@ const CodeBlock = ({ node, inline, className, children, ...props }: CodeBlockPro
   );
 };
 
-const markdownComponents = {
+function resolveImageSrc(src: string | undefined, projectName: string | undefined): string | undefined {
+  if (!src || !projectName) return src;
+  if (src.startsWith('data:') || src.startsWith('http://') || src.startsWith('https://')) return src;
+  const cleaned = src.replace(/^\.\//, '');
+  return `/api/projects/${encodeURIComponent(projectName)}/files/content?path=${encodeURIComponent(cleaned)}`;
+}
+
+const baseMarkdownComponents = {
   code: CodeBlock,
   blockquote: ({ children }: { children?: React.ReactNode }) => (
     <blockquote className="my-2 border-l-4 border-gray-300 pl-4 italic text-gray-600 dark:border-gray-600 dark:text-gray-400">
@@ -143,14 +152,30 @@ const markdownComponents = {
   ),
 };
 
-export function Markdown({ children, className }: MarkdownProps) {
+export function Markdown({ children, className, projectName }: MarkdownProps) {
   const content = normalizeInlineCodeFences(String(children ?? ''));
   const remarkPlugins = useMemo(() => [remarkGfm, remarkMath], []);
   const rehypePlugins = useMemo(() => [rehypeKatex], []);
 
+  const components = useMemo(() => {
+    if (!projectName) return baseMarkdownComponents;
+    return {
+      ...baseMarkdownComponents,
+      img: ({ src, alt, ...rest }: React.ImgHTMLAttributes<HTMLImageElement>) => (
+        <img
+          src={resolveImageSrc(src, projectName)}
+          alt={alt || ''}
+          className="my-2 max-w-full rounded-lg"
+          loading="lazy"
+          {...rest}
+        />
+      ),
+    };
+  }, [projectName]);
+
   return (
     <div className={className}>
-      <ReactMarkdown remarkPlugins={remarkPlugins} rehypePlugins={rehypePlugins} components={markdownComponents as any}>
+      <ReactMarkdown remarkPlugins={remarkPlugins} rehypePlugins={rehypePlugins} components={components as any}>
         {content}
       </ReactMarkdown>
     </div>

ui/src/components/chat/view/subcomponents/MessageComponent.tsx

@@ -323,7 +323,7 @@ const MessageComponent = memo(({ message, prevMessage, createDiff, onFileOpen, o
               <>
                 <div className="flex flex-col">
                   <div className="flex flex-col">
-                    <Markdown className="prose prose-sm max-w-none dark:prose-invert">
+                    <Markdown className="prose prose-sm max-w-none dark:prose-invert" projectName={selectedProject?.name}>
                       {String(message.displayText || '')}
                     </Markdown>
                   </div>
@@ -392,7 +392,7 @@ const MessageComponent = memo(({ message, prevMessage, createDiff, onFileOpen, o
                               toolCategory="default"
                               autoExpandable={false}
                             >
-                              <Markdown className="prose prose-sm max-w-none dark:prose-invert">
+                              <Markdown className="prose prose-sm max-w-none dark:prose-invert" projectName={selectedProject?.name}>
                                 {renderedErrorContent}
                               </Markdown>
                             </CollapsibleDisplay>
@@ -427,7 +427,7 @@ const MessageComponent = memo(({ message, prevMessage, createDiff, onFileOpen, o
                                 ) : null}
                               </summary>
                               <div className="mt-1.5 pl-[18px] text-xs leading-5 text-gray-700 dark:text-gray-300">
-                                <Markdown className="prose prose-sm prose-red max-w-none dark:prose-invert">
+                                <Markdown className="prose prose-sm prose-red max-w-none dark:prose-invert" projectName={selectedProject?.name}>
                                   {renderedErrorContent}
                                 </Markdown>
                                 <div className="mt-3 border-t border-red-200/60 pt-3 dark:border-red-800/60">
@@ -607,7 +607,7 @@ const MessageComponent = memo(({ message, prevMessage, createDiff, onFileOpen, o
                     <span>{t('thinking.emoji')}</span>
                   </summary>
                   <div className="mt-2 border-l-2 border-gray-300 pl-4 text-sm text-gray-600 dark:border-gray-600 dark:text-gray-400">
-                    <Markdown className="prose prose-sm prose-gray max-w-none dark:prose-invert">
+                    <Markdown className="prose prose-sm prose-gray max-w-none dark:prose-invert" projectName={selectedProject?.name}>
                       {messageContent}
                     </Markdown>
                   </div>
@@ -664,7 +664,7 @@ const MessageComponent = memo(({ message, prevMessage, createDiff, onFileOpen, o
 
                   // Normal rendering for non-JSON content
                   return message.type === 'assistant' ? (
-                    <Markdown className="prose prose-sm prose-gray max-w-none dark:prose-invert">
+                    <Markdown className="prose prose-sm prose-gray max-w-none dark:prose-invert" projectName={selectedProject?.name}>
                       {content}
                     </Markdown>
                   ) : (