From 6af5d31ab33c0ad54e38063c485153e760c49693 Mon Sep 17 00:00:00 2001
From: Stephen Toub <stoub@microsoft.com>
Date: Thu, 4 Jun 2026 10:58:16 -0400
Subject: [PATCH] Normalize shell completion markers in replay proxy

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 test/harness/replayingCapiProxy.test.ts | 82 +++++++++++++++++++++++++
 test/harness/replayingCapiProxy.ts      |  8 +++
 2 files changed, 90 insertions(+)

diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts
index 2ac1f76c2..803226548 100644
--- a/test/harness/replayingCapiProxy.test.ts
+++ b/test/harness/replayingCapiProxy.test.ts
@@ -745,6 +745,88 @@ Always include PINEAPPLE_COCONUT_42.
       }
     });
 
+    test("matches shell tool results with shell ID completion markers", async () => {
+      const originalShellConfig =
+        process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash;
+      const cachePath = path.join(tempDir, "cache.yaml");
+      const cacheContent = yaml.stringify({
+        models: ["test-model"],
+        conversations: [
+          {
+            messages: [
+              { role: "system", content: "${system}" },
+              { role: "user", content: "Run command" },
+              {
+                role: "assistant",
+                tool_calls: [
+                  {
+                    id: "toolcall_0",
+                    type: "function",
+                    function: {
+                      name: "${shell}",
+                      arguments: '{"command":"echo ok"}',
+                    },
+                  },
+                ],
+              },
+              {
+                role: "tool",
+                tool_call_id: "toolcall_0",
+                content: "ok\n<exited with exit code 0>",
+              },
+              { role: "assistant", content: "Done" },
+            ],
+          },
+        ],
+      } satisfies NormalizedData);
+      await writeFile(cachePath, cacheContent);
+
+      const proxy = new ReplayingCapiProxy(
+        "http://localhost:9999",
+        cachePath,
+        workDir,
+      );
+      const proxyUrl = await proxy.start();
+
+      try {
+        const response = await makeRequest(proxyUrl, "/chat/completions", {
+          body: {
+            model: "test-model",
+            messages: [
+              { role: "system", content: "System prompt" },
+              { role: "user", content: "Run command" },
+              {
+                role: "assistant",
+                tool_calls: [
+                  {
+                    id: "runtime-call-id",
+                    type: "function",
+                    function: {
+                      name: originalShellConfig.shellToolName,
+                      arguments: '{"command":"echo ok"}',
+                    },
+                  },
+                ],
+              },
+              {
+                role: "tool",
+                tool_call_id: "runtime-call-id",
+                content: "ok\n<shellId: 42 completed with exit code 0>",
+              },
+            ],
+          },
+        });
+
+        expect(response.status).toBe(200);
+        expect(
+          (JSON.parse(response.body) as ChatCompletion).choices[0].message
+            .content,
+        ).toBe("Done");
+      } finally {
+        await proxy.stop();
+      }
+    });
+
     test("expands workdir placeholder in cached response", async () => {
       const cachePath = path.join(tempDir, "cache.yaml");
       const cacheContent = yaml.stringify({
diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts
index bee00ffcc..9c2467b2a 100644
--- a/test/harness/replayingCapiProxy.ts
+++ b/test/harness/replayingCapiProxy.ts
@@ -54,6 +54,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy {
   private startPromise: Promise<string> | null = null;
   private defaultToolResultNormalizers: ToolResultNormalizer[] = [
     { toolName: "*", normalizer: normalizeLargeOutputFilepaths },
+    { toolName: "${shell}", normalizer: normalizeShellExitMarkers },
     { toolName: "*", normalizer: normalizeGhAuthMessages },
     { toolName: "read_agent", normalizer: normalizeReadAgentTimings },
   ];
@@ -1087,6 +1088,13 @@ function normalizeLargeOutputFilepaths(result: string): string {
     );
 }
 
+function normalizeShellExitMarkers(result: string): string {
+  return result.replace(
+    /<shellId:\s*[^>\r\n]+?\s+completed with exit code (-?\d+)>/g,
+    "<exited with exit code $1>",
+  );
+}
+
 // The `gh` CLI emits different "not authenticated" help text depending on the
 // environment (local dev vs. inside GitHub Actions). Normalize both forms to a
 // stable placeholder so snapshots don't drift between environments.