From 71897706a984a94e6e51c0f2f300e77e83c0494a Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Wed, 8 Apr 2026 11:36:09 -0700 Subject: [PATCH 1/4] Add Codex support for explain --generate Entire-Checkpoint: 9b668a38f4bf --- cmd/entire/cli/agent/codex/transcript.go | 28 +++++++ cmd/entire/cli/explain.go | 7 ++ cmd/entire/cli/explain_test.go | 32 ++++++++ cmd/entire/cli/summarize/summarize.go | 96 ++++++++++++++++++++++ cmd/entire/cli/summarize/summarize_test.go | 37 +++++++++ cmd/entire/cli/transcript/compact/codex.go | 7 +- 6 files changed, 206 insertions(+), 1 deletion(-) diff --git a/cmd/entire/cli/agent/codex/transcript.go b/cmd/entire/cli/agent/codex/transcript.go index 6445fba81..d71a7e22b 100644 --- a/cmd/entire/cli/agent/codex/transcript.go +++ b/cmd/entire/cli/agent/codex/transcript.go @@ -303,6 +303,34 @@ func (c *CodexAgent) ExtractPrompts(sessionRef string, fromOffset int) ([]string return prompts, nil } +// SliceFromResponseItem returns the transcript content after skipping n response_item +// entries. Non-response_item lines are preserved in the returned suffix. +func SliceFromResponseItem(data []byte, n int) ([]byte, error) { + if len(data) == 0 || n <= 0 { + return data, nil + } + + lines := splitJSONL(data) + seen := 0 + for i, lineData := range lines { + var line rolloutLine + if err := json.Unmarshal(lineData, &line); err != nil { + continue + } + if line.Type == "response_item" { + seen++ + } + if seen >= n { + if i+1 >= len(lines) { + return nil, nil + } + return bytes.Join(lines[i+1:], []byte("\n")), nil + } + } + + return nil, nil +} + // splitJSONL splits JSONL bytes into individual lines, skipping empty lines. func splitJSONL(data []byte) [][]byte { var lines [][]byte diff --git a/cmd/entire/cli/explain.go b/cmd/entire/cli/explain.go index b9de61383..c8b77f70d 100644 --- a/cmd/entire/cli/explain.go +++ b/cmd/entire/cli/explain.go @@ -13,6 +13,7 @@ import ( "time" "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/agent/codex" "github.com/entireio/cli/cmd/entire/cli/agent/geminicli" "github.com/entireio/cli/cmd/entire/cli/agent/opencode" "github.com/entireio/cli/cmd/entire/cli/agent/types" @@ -572,6 +573,12 @@ func scopeTranscriptForCheckpoint(fullTranscript []byte, startOffset int, agentT return nil } return scoped + case agent.AgentTypeCodex: + scoped, err := codex.SliceFromResponseItem(fullTranscript, startOffset) + if err != nil { + return nil + } + return scoped case agent.AgentTypeClaudeCode, agent.AgentTypeCursor, agent.AgentTypeFactoryAIDroid, agent.AgentTypeUnknown: return transcript.SliceFromLine(fullTranscript, startOffset) } diff --git a/cmd/entire/cli/explain_test.go b/cmd/entire/cli/explain_test.go index 14259e7ec..addba03e1 100644 --- a/cmd/entire/cli/explain_test.go +++ b/cmd/entire/cli/explain_test.go @@ -15,6 +15,7 @@ import ( "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/strategy" + "github.com/entireio/cli/cmd/entire/cli/summarize" "github.com/entireio/cli/cmd/entire/cli/testutil" "github.com/entireio/cli/cmd/entire/cli/trailers" "github.com/entireio/cli/cmd/entire/cli/transcript" @@ -2174,6 +2175,37 @@ func TestScopeTranscriptForCheckpoint_ZeroLinesReturnsAll(t *testing.T) { } } +func TestScopeTranscriptForCheckpoint_CodexUsesResponseItemOffsets(t *testing.T) { + t.Parallel() + + fullTranscript := []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions"}]}} +{"timestamp":"t3","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"# AGENTS.md\ninstructions"}]}} +{"timestamp":"t4","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"first prompt"}]}} +{"timestamp":"t5","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"response to first"}]}} +{"timestamp":"t6","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"second prompt"}]}} +{"timestamp":"t7","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"response to second"}]}} +`) + + scoped := scopeTranscriptForCheckpoint(fullTranscript, 4, agent.AgentTypeCodex) + entries, err := summarize.BuildCondensedTranscriptFromBytes(scoped, agent.AgentTypeCodex) + if err != nil { + t.Fatalf("failed to build condensed transcript: %v", err) + } + + if len(entries) != 2 { + t.Fatalf("expected 2 scoped entries, got %d", len(entries)) + } + + if entries[0].Type != summarize.EntryTypeUser || entries[0].Content != "second prompt" { + t.Fatalf("expected first entry to be second prompt, got %#v", entries[0]) + } + + if entries[1].Type != summarize.EntryTypeAssistant || entries[1].Content != "response to second" { + t.Fatalf("expected second entry to be second response, got %#v", entries[1]) + } +} + func TestExtractPromptsFromScopedTranscript(t *testing.T) { // Transcript with 4 lines - 2 user prompts, 2 assistant responses transcript := []byte(`{"type":"user","uuid":"u1","message":{"content":"First prompt"}} diff --git a/cmd/entire/cli/summarize/summarize.go b/cmd/entire/cli/summarize/summarize.go index f6d1ae357..8b56580fa 100644 --- a/cmd/entire/cli/summarize/summarize.go +++ b/cmd/entire/cli/summarize/summarize.go @@ -2,6 +2,7 @@ package summarize import ( + "bytes" "context" "encoding/json" "errors" @@ -15,6 +16,7 @@ import ( "github.com/entireio/cli/cmd/entire/cli/agent/types" "github.com/entireio/cli/cmd/entire/cli/checkpoint" "github.com/entireio/cli/cmd/entire/cli/transcript" + "github.com/entireio/cli/cmd/entire/cli/transcript/compact" ) // GenerateFromTranscript generates a summary from raw transcript bytes. @@ -123,6 +125,8 @@ func BuildCondensedTranscriptFromBytes(content []byte, agentType types.AgentType return buildCondensedTranscriptFromDroid(content) case agent.AgentTypeOpenCode: return buildCondensedTranscriptFromOpenCode(content) + case agent.AgentTypeCodex: + return buildCondensedTranscriptFromCodex(content) case agent.AgentTypeClaudeCode, agent.AgentTypeCursor, agent.AgentTypeUnknown: // Claude/cursor format - fall through to shared logic below } @@ -217,6 +221,84 @@ func buildCondensedTranscriptFromOpenCode(content []byte) ([]Entry, error) { return entries, nil } +// buildCondensedTranscriptFromCodex converts Codex rollout JSONL into the compact +// transcript format, then reuses the shared transcript condensation logic. +func buildCondensedTranscriptFromCodex(content []byte) ([]Entry, error) { + compacted, err := compact.Compact(content, compact.MetadataFields{ + Agent: "codex", + CLIVersion: "summarize", + }) + if err != nil { + return nil, fmt.Errorf("failed to compact Codex transcript: %w", err) + } + + type compactUserTextBlock struct { + Text string `json:"text"` + } + type compactAssistantBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + Name string `json:"name,omitempty"` + Input json.RawMessage `json:"input,omitempty"` + } + type compactLine struct { + Type string `json:"type"` + Content json.RawMessage `json:"content"` + } + + var entries []Entry + for _, lineBytes := range splitCompactJSONL(compacted) { + var line compactLine + if err := json.Unmarshal(lineBytes, &line); err != nil { + continue + } + + switch line.Type { + case transcript.TypeUser: + var blocks []compactUserTextBlock + if err := json.Unmarshal(line.Content, &blocks); err != nil { + continue + } + for _, block := range blocks { + if block.Text != "" { + entries = append(entries, Entry{ + Type: EntryTypeUser, + Content: block.Text, + }) + } + } + case transcript.TypeAssistant: + var blocks []compactAssistantBlock + if err := json.Unmarshal(line.Content, &blocks); err != nil { + continue + } + for _, block := range blocks { + switch block.Type { + case transcript.ContentTypeText: + if block.Text != "" { + entries = append(entries, Entry{ + Type: EntryTypeAssistant, + Content: block.Text, + }) + } + case transcript.ContentTypeToolUse: + var input map[string]interface{} + if err := json.Unmarshal(block.Input, &input); err != nil { + input = nil + } + entries = append(entries, Entry{ + Type: EntryTypeTool, + ToolName: block.Name, + ToolDetail: extractGenericToolDetail(input), + }) + } + } + } + } + + return entries, nil +} + // buildCondensedTranscriptFromDroid parses Droid transcript and extracts a condensed view. func buildCondensedTranscriptFromDroid(content []byte) ([]Entry, error) { droidLines, _, err := factoryaidroid.ParseDroidTranscriptFromBytes(content, 0) @@ -240,6 +322,9 @@ func extractOpenCodeToolDetail(input map[string]interface{}) string { // extractGenericToolDetail extracts an appropriate detail string from a tool's input/args map. // Checks common fields in order of preference. Used by Gemini condensation. func extractGenericToolDetail(input map[string]interface{}) string { + if input == nil { + return "" + } for _, key := range []string{"description", "command", "file_path", "path", "pattern"} { if v, ok := input[key].(string); ok && v != "" { return v @@ -248,6 +333,17 @@ func extractGenericToolDetail(input map[string]interface{}) string { return "" } +func splitCompactJSONL(data []byte) [][]byte { + var lines [][]byte + for _, line := range bytes.Split(data, []byte("\n")) { + line = bytes.TrimSpace(line) + if len(line) > 0 { + lines = append(lines, line) + } + } + return lines +} + // BuildCondensedTranscript extracts a condensed view of the transcript. // It processes user prompts, assistant responses, and tool calls into // a simplified format suitable for LLM summarization. diff --git a/cmd/entire/cli/summarize/summarize_test.go b/cmd/entire/cli/summarize/summarize_test.go index dd20f0d2a..6e64ff34c 100644 --- a/cmd/entire/cli/summarize/summarize_test.go +++ b/cmd/entire/cli/summarize/summarize_test.go @@ -696,6 +696,43 @@ func TestGenerateFromTranscript_NilGenerator(t *testing.T) { } } +func TestBuildCondensedTranscriptFromBytes_Codex(t *testing.T) { + t.Parallel() + + codexTranscript := []byte(`{"timestamp":"2026-04-01T23:31:27.000Z","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"2026-04-01T23:31:28.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"create hello.txt"}]}} +{"timestamp":"2026-04-01T23:31:29.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Creating the file now."}]}} +{"timestamp":"2026-04-01T23:31:30.000Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call_1","arguments":"{\"cmd\":\"touch hello.txt\",\"workdir\":\"/repo\"}"}} +{"timestamp":"2026-04-01T23:31:31.000Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call_1","output":"ok"}} +{"timestamp":"2026-04-01T23:31:32.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Done."}]}} +`) + + entries, err := BuildCondensedTranscriptFromBytes(codexTranscript, agent.AgentTypeCodex) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(entries) != 4 { + t.Fatalf("expected 4 entries, got %d", len(entries)) + } + + if entries[0].Type != EntryTypeUser || entries[0].Content != "create hello.txt" { + t.Fatalf("unexpected first entry: %#v", entries[0]) + } + + if entries[1].Type != EntryTypeAssistant || entries[1].Content != "Creating the file now." { + t.Fatalf("unexpected second entry: %#v", entries[1]) + } + + if entries[2].Type != EntryTypeTool || entries[2].ToolName != "exec_command" { + t.Fatalf("unexpected tool entry: %#v", entries[2]) + } + + if entries[3].Type != EntryTypeAssistant || entries[3].Content != "Done." { + t.Fatalf("unexpected final entry: %#v", entries[3]) + } +} + func TestBuildCondensedTranscriptFromBytes_OpenCodeUserAndAssistant(t *testing.T) { // OpenCode export JSON format ocExportJSON := `{ diff --git a/cmd/entire/cli/transcript/compact/codex.go b/cmd/entire/cli/transcript/compact/codex.go index 315abbc2c..deec3f4ef 100644 --- a/cmd/entire/cli/transcript/compact/codex.go +++ b/cmd/entire/cli/transcript/compact/codex.go @@ -33,7 +33,12 @@ func isCodexFormat(content []byte) bool { if json.Unmarshal(line, &probe) != nil { continue } - return probe.Type == "session_meta" + switch probe.Type { + case "session_meta", "response_item", "event_msg", "turn_context": + return true + default: + return false + } } if scanner.Err() != nil { return false From 81e7c45b6ed493917db4618a0e27317eb0645db3 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Wed, 8 Apr 2026 11:52:49 -0700 Subject: [PATCH 2/4] Fix Codex transcript offset handling Entire-Checkpoint: a1c5ef641832 --- cmd/entire/cli/agent/codex/transcript.go | 28 ---------------------- cmd/entire/cli/explain.go | 7 +----- cmd/entire/cli/explain_test.go | 9 +++---- cmd/entire/cli/transcript/compact/codex.go | 5 ++-- 4 files changed, 9 insertions(+), 40 deletions(-) diff --git a/cmd/entire/cli/agent/codex/transcript.go b/cmd/entire/cli/agent/codex/transcript.go index d71a7e22b..6445fba81 100644 --- a/cmd/entire/cli/agent/codex/transcript.go +++ b/cmd/entire/cli/agent/codex/transcript.go @@ -303,34 +303,6 @@ func (c *CodexAgent) ExtractPrompts(sessionRef string, fromOffset int) ([]string return prompts, nil } -// SliceFromResponseItem returns the transcript content after skipping n response_item -// entries. Non-response_item lines are preserved in the returned suffix. -func SliceFromResponseItem(data []byte, n int) ([]byte, error) { - if len(data) == 0 || n <= 0 { - return data, nil - } - - lines := splitJSONL(data) - seen := 0 - for i, lineData := range lines { - var line rolloutLine - if err := json.Unmarshal(lineData, &line); err != nil { - continue - } - if line.Type == "response_item" { - seen++ - } - if seen >= n { - if i+1 >= len(lines) { - return nil, nil - } - return bytes.Join(lines[i+1:], []byte("\n")), nil - } - } - - return nil, nil -} - // splitJSONL splits JSONL bytes into individual lines, skipping empty lines. func splitJSONL(data []byte) [][]byte { var lines [][]byte diff --git a/cmd/entire/cli/explain.go b/cmd/entire/cli/explain.go index c8b77f70d..4bbb5f034 100644 --- a/cmd/entire/cli/explain.go +++ b/cmd/entire/cli/explain.go @@ -13,7 +13,6 @@ import ( "time" "github.com/entireio/cli/cmd/entire/cli/agent" - "github.com/entireio/cli/cmd/entire/cli/agent/codex" "github.com/entireio/cli/cmd/entire/cli/agent/geminicli" "github.com/entireio/cli/cmd/entire/cli/agent/opencode" "github.com/entireio/cli/cmd/entire/cli/agent/types" @@ -574,11 +573,7 @@ func scopeTranscriptForCheckpoint(fullTranscript []byte, startOffset int, agentT } return scoped case agent.AgentTypeCodex: - scoped, err := codex.SliceFromResponseItem(fullTranscript, startOffset) - if err != nil { - return nil - } - return scoped + return transcript.SliceFromLine(fullTranscript, startOffset) case agent.AgentTypeClaudeCode, agent.AgentTypeCursor, agent.AgentTypeFactoryAIDroid, agent.AgentTypeUnknown: return transcript.SliceFromLine(fullTranscript, startOffset) } diff --git a/cmd/entire/cli/explain_test.go b/cmd/entire/cli/explain_test.go index addba03e1..85d2c00db 100644 --- a/cmd/entire/cli/explain_test.go +++ b/cmd/entire/cli/explain_test.go @@ -2175,7 +2175,7 @@ func TestScopeTranscriptForCheckpoint_ZeroLinesReturnsAll(t *testing.T) { } } -func TestScopeTranscriptForCheckpoint_CodexUsesResponseItemOffsets(t *testing.T) { +func TestScopeTranscriptForCheckpoint_CodexUsesStoredLineOffsets(t *testing.T) { t.Parallel() fullTranscript := []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} @@ -2183,11 +2183,12 @@ func TestScopeTranscriptForCheckpoint_CodexUsesResponseItemOffsets(t *testing.T) {"timestamp":"t3","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"# AGENTS.md\ninstructions"}]}} {"timestamp":"t4","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"first prompt"}]}} {"timestamp":"t5","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"response to first"}]}} -{"timestamp":"t6","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"second prompt"}]}} -{"timestamp":"t7","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"response to second"}]}} +{"timestamp":"t6","type":"event_msg","payload":{"type":"token_count","input_tokens":10,"output_tokens":1}} +{"timestamp":"t7","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"second prompt"}]}} +{"timestamp":"t8","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"response to second"}]}} `) - scoped := scopeTranscriptForCheckpoint(fullTranscript, 4, agent.AgentTypeCodex) + scoped := scopeTranscriptForCheckpoint(fullTranscript, 6, agent.AgentTypeCodex) entries, err := summarize.BuildCondensedTranscriptFromBytes(scoped, agent.AgentTypeCodex) if err != nil { t.Fatalf("failed to build condensed transcript: %v", err) diff --git a/cmd/entire/cli/transcript/compact/codex.go b/cmd/entire/cli/transcript/compact/codex.go index deec3f4ef..71b7c1405 100644 --- a/cmd/entire/cli/transcript/compact/codex.go +++ b/cmd/entire/cli/transcript/compact/codex.go @@ -14,6 +14,7 @@ import ( const ( transcriptTypeMessage = "message" + codexTypeResponseItem = "response_item" codexTypeFunctionCall = "function_call" codexTypeFunctionCallOutput = "function_call_output" ) @@ -34,7 +35,7 @@ func isCodexFormat(content []byte) bool { continue } switch probe.Type { - case "session_meta", "response_item", "event_msg", "turn_context": + case "session_meta", codexTypeResponseItem, "event_msg", "turn_context": return true default: return false @@ -341,7 +342,7 @@ func codexSliceFromResponseItem(lines []codexLine, n int) []codexLine { seen := 0 for i, line := range lines { - if line.Type == "response_item" { + if line.Type == codexTypeResponseItem { seen++ } if seen >= n { From b5170e7ddb76aaa3102dfc5d3b1f833aab6133f1 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 9 Apr 2026 11:36:59 -0700 Subject: [PATCH 3/4] Fix Codex summary gaps: custom_tool_call, tool detail, and auto-summarize The Codex compactor only handled function_call/function_call_output, silently dropping custom_tool_call/custom_tool_call_output entries used for apply_patch file edits. This caused explain --generate to produce summaries that omitted the actual file modifications. - Handle custom_tool_call and custom_tool_call_output in compactCodex alongside function_call variants via shared codexConsumeToolCall helper - Add "cmd" to extractGenericToolDetail so Codex exec_command tool details are extracted (Codex uses "cmd", not "command") - Add AgentTypeCodex to generateSummary in condensation so auto-summarize works for Codex sessions (was silently skipped) - Add adversarial tests: missing output, mixed tool types, mismatched call IDs, empty input, plain string output, consecutive standalone calls, and real Codex rollout shape end-to-end Co-Authored-By: Claude Opus 4.6 (1M context) Entire-Checkpoint: 31d7794d8618 --- .../strategy/manual_commit_condensation.go | 2 +- cmd/entire/cli/summarize/summarize.go | 2 +- cmd/entire/cli/summarize/summarize_test.go | 70 ++++++++ cmd/entire/cli/transcript/compact/codex.go | 161 +++++++++++++----- .../cli/transcript/compact/codex_test.go | 34 ++++ 5 files changed, 229 insertions(+), 40 deletions(-) diff --git a/cmd/entire/cli/strategy/manual_commit_condensation.go b/cmd/entire/cli/strategy/manual_commit_condensation.go index 20dcf55c8..2cd0ef790 100644 --- a/cmd/entire/cli/strategy/manual_commit_condensation.go +++ b/cmd/entire/cli/strategy/manual_commit_condensation.go @@ -290,7 +290,7 @@ func generateSummary(ctx context.Context, sessionData *ExtractedSessionData, sta slog.String("error", sliceErr.Error())) } scopedTranscript = scoped - case agent.AgentTypeClaudeCode, agent.AgentTypeCursor, agent.AgentTypeFactoryAIDroid, agent.AgentTypeUnknown: + case agent.AgentTypeCodex, agent.AgentTypeClaudeCode, agent.AgentTypeCursor, agent.AgentTypeFactoryAIDroid, agent.AgentTypeUnknown: scopedTranscript = transcript.SliceFromLine(sessionData.Transcript, state.CheckpointTranscriptStart) } diff --git a/cmd/entire/cli/summarize/summarize.go b/cmd/entire/cli/summarize/summarize.go index 8b56580fa..7d5ae525d 100644 --- a/cmd/entire/cli/summarize/summarize.go +++ b/cmd/entire/cli/summarize/summarize.go @@ -325,7 +325,7 @@ func extractGenericToolDetail(input map[string]interface{}) string { if input == nil { return "" } - for _, key := range []string{"description", "command", "file_path", "path", "pattern"} { + for _, key := range []string{"description", "command", "cmd", "file_path", "path", "pattern"} { if v, ok := input[key].(string); ok && v != "" { return v } diff --git a/cmd/entire/cli/summarize/summarize_test.go b/cmd/entire/cli/summarize/summarize_test.go index 6e64ff34c..7f9314e1c 100644 --- a/cmd/entire/cli/summarize/summarize_test.go +++ b/cmd/entire/cli/summarize/summarize_test.go @@ -733,6 +733,76 @@ func TestBuildCondensedTranscriptFromBytes_Codex(t *testing.T) { } } +func TestBuildCondensedTranscriptFromBytes_Codex_CustomToolCall(t *testing.T) { + t.Parallel() + + codexTranscript := []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"create hello.txt"}]}} +{"timestamp":"t3","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Creating the file now."}]}} +{"timestamp":"t4","type":"response_item","payload":{"type":"custom_tool_call","status":"completed","call_id":"call_1","name":"apply_patch","input":"*** Begin Patch\n*** Add File: hello.txt\n+Hello World\n*** End Patch\n"}} +{"timestamp":"t5","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_1","output":{"type":"text","text":"Success. Updated: A hello.txt"}}} +{"timestamp":"t6","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Done."}]}} +`) + + entries, err := BuildCondensedTranscriptFromBytes(codexTranscript, agent.AgentTypeCodex) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(entries) != 4 { + t.Fatalf("expected 4 entries, got %d: %#v", len(entries), entries) + } + + if entries[0].Type != EntryTypeUser || entries[0].Content != "create hello.txt" { + t.Fatalf("unexpected first entry: %#v", entries[0]) + } + + if entries[1].Type != EntryTypeAssistant || entries[1].Content != "Creating the file now." { + t.Fatalf("unexpected second entry: %#v", entries[1]) + } + + if entries[2].Type != EntryTypeTool || entries[2].ToolName != "apply_patch" { + t.Fatalf("expected apply_patch tool entry, got: %#v", entries[2]) + } + + if entries[3].Type != EntryTypeAssistant || entries[3].Content != "Done." { + t.Fatalf("unexpected final entry: %#v", entries[3]) + } +} + +func TestBuildCondensedTranscriptFromBytes_Codex_ExecCommandDetail(t *testing.T) { + t.Parallel() + + codexTranscript := []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Running command."}]}} +{"timestamp":"t3","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call_1","arguments":"{\"cmd\":\"ls -la\",\"workdir\":\"/repo\"}"}} +{"timestamp":"t4","type":"response_item","payload":{"type":"function_call_output","call_id":"call_1","output":"total 0"}} +`) + + entries, err := BuildCondensedTranscriptFromBytes(codexTranscript, agent.AgentTypeCodex) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Find the tool entry + var toolEntry *Entry + for i := range entries { + if entries[i].Type == EntryTypeTool { + toolEntry = &entries[i] + break + } + } + if toolEntry == nil { + t.Fatalf("no tool entry found in entries: %#v", entries) + } + if toolEntry.ToolName != "exec_command" { + t.Fatalf("expected exec_command, got %q", toolEntry.ToolName) + } + if toolEntry.ToolDetail != "ls -la" { + t.Fatalf("expected tool detail 'ls -la', got %q", toolEntry.ToolDetail) + } +} + func TestBuildCondensedTranscriptFromBytes_OpenCodeUserAndAssistant(t *testing.T) { // OpenCode export JSON format ocExportJSON := `{ diff --git a/cmd/entire/cli/transcript/compact/codex.go b/cmd/entire/cli/transcript/compact/codex.go index 71b7c1405..977115be5 100644 --- a/cmd/entire/cli/transcript/compact/codex.go +++ b/cmd/entire/cli/transcript/compact/codex.go @@ -13,10 +13,12 @@ import ( ) const ( - transcriptTypeMessage = "message" - codexTypeResponseItem = "response_item" - codexTypeFunctionCall = "function_call" - codexTypeFunctionCallOutput = "function_call_output" + transcriptTypeMessage = "message" + codexTypeResponseItem = "response_item" + codexTypeFunctionCall = "function_call" + codexTypeFunctionCallOutput = "function_call_output" + codexTypeCustomToolCall = "custom_tool_call" + codexTypeCustomToolCallOutput = "custom_tool_call_output" ) // isCodexFormat checks whether JSONL content uses the Codex format. @@ -62,6 +64,7 @@ type codexPayload struct { Phase string `json:"phase"` Name string `json:"name"` Arguments string `json:"arguments"` + Input string `json:"input"` CallID string `json:"call_id"` Output string `json:"output"` } @@ -125,7 +128,7 @@ func compactCodex(content []byte, opts MetadataFields) ([]byte, error) { continue } - // Collect any function_calls that follow this assistant message. + // Collect any tool calls that follow this assistant message. var toolBlocks []map[string]json.RawMessage inTok, outTok := pendingInTok, pendingOutTok pendingInTok, pendingOutTok = 0, 0 @@ -140,27 +143,14 @@ func compactCodex(content []byte, opts MetadataFields) ([]byte, error) { if json.Unmarshal(next.Payload, &np) != nil { break } - if np.Type == codexTypeFunctionCall { - tb := codexToolUseBlock(np) - i++ // consume the function_call line - // Skip token_count lines between function_call and output. - for i+1 < len(lines) && isCodexTokenCountLine(lines[i+1]) { - inTok, outTok = codexTokenCount(lines[i+1].Payload) - i++ - } - // Look ahead for the matching output. - if i+1 < len(lines) { - var outp codexPayload - if json.Unmarshal(lines[i+1].Payload, &outp) == nil && outp.Type == codexTypeFunctionCallOutput && outp.CallID == np.CallID { - tb["result"] = buildToolResult(toolResultEntry{output: outp.Output}) - i++ // consume the output line - } - } + if np.Type == codexTypeFunctionCall || np.Type == codexTypeCustomToolCall { + i++ // consume the tool call line + tb := codexConsumeToolCall(np, lines, &i, &inTok, &outTok) toolBlocks = append(toolBlocks, tb) continue } - // function_call_output without a preceding function_call — skip. - if np.Type == codexTypeFunctionCallOutput { + // Orphan output without a preceding call — skip. + if np.Type == codexTypeFunctionCallOutput || np.Type == codexTypeCustomToolCallOutput { i++ continue } @@ -176,23 +166,11 @@ func compactCodex(content []byte, opts MetadataFields) ([]byte, error) { line.Content = contentArr appendLine(&result, line) - case p.Type == codexTypeFunctionCall: - // Standalone function_call not preceded by assistant text. - tb := codexToolUseBlock(p) + case p.Type == codexTypeFunctionCall || p.Type == codexTypeCustomToolCall: + // Standalone tool call not preceded by assistant text. inTok, outTok := pendingInTok, pendingOutTok pendingInTok, pendingOutTok = 0, 0 - // Skip token_count lines between function_call and output. - for i+1 < len(lines) && isCodexTokenCountLine(lines[i+1]) { - inTok, outTok = codexTokenCount(lines[i+1].Payload) - i++ - } - if i+1 < len(lines) { - var np codexPayload - if json.Unmarshal(lines[i+1].Payload, &np) == nil && np.Type == codexTypeFunctionCallOutput && np.CallID == p.CallID { - tb["result"] = buildToolResult(toolResultEntry{output: np.Output}) - i++ - } - } + tb := codexConsumeToolCall(p, lines, &i, &inTok, &outTok) // Also consume any trailing token_count. for i+1 < len(lines) && isCodexTokenCountLine(lines[i+1]) { inTok, outTok = codexTokenCount(lines[i+1].Payload) @@ -372,6 +350,113 @@ func codexToolUseBlock(p codexPayload) map[string]json.RawMessage { return block } +// codexConsumeToolCall builds a tool_use block from a function_call or custom_tool_call, +// consuming any trailing token_count lines and the matching output line. +// The caller must advance i past the tool call line itself before calling this function, +// and must verify p.Type is a tool call type. +func codexConsumeToolCall(p codexPayload, lines []codexLine, i *int, inTok, outTok *int) map[string]json.RawMessage { + var tb map[string]json.RawMessage + var outputType string + + switch p.Type { + case codexTypeFunctionCall: + tb = codexToolUseBlock(p) + outputType = codexTypeFunctionCallOutput + case codexTypeCustomToolCall: + tb = codexCustomToolUseBlock(p) + outputType = codexTypeCustomToolCallOutput + default: + return nil + } + + for *i+1 < len(lines) && isCodexTokenCountLine(lines[*i+1]) { + *inTok, *outTok = codexTokenCount(lines[*i+1].Payload) + *i++ + } + + if *i+1 < len(lines) { + callID, typ := codexCallIDAndType(lines[*i+1].Payload) + if typ == outputType && callID == p.CallID { + output := codexToolOutputText(lines[*i+1].Payload, outputType) + tb["result"] = buildToolResult(toolResultEntry{output: output}) + *i++ + } + } + + return tb +} + +// codexToolOutputText extracts the output text from a tool call output payload. +// For function_call_output, the output field is a plain string. +// For custom_tool_call_output, it is an object {"type":"text","text":"..."}. +func codexToolOutputText(payload json.RawMessage, outputType string) string { + if outputType == codexTypeCustomToolCallOutput { + return codexCustomOutputText(payload) + } + var p codexPayload + if json.Unmarshal(payload, &p) == nil { + return p.Output + } + return "" +} + +// codexCallIDAndType extracts just the type and call_id from a payload without +// failing on fields with unexpected types (e.g., custom_tool_call_output has an +// object "output" field that can't unmarshal into codexPayload.Output string). +func codexCallIDAndType(payload json.RawMessage) (callID, typ string) { + var p struct { + Type string `json:"type"` + CallID string `json:"call_id"` + } + if json.Unmarshal(payload, &p) == nil { + return p.CallID, p.Type + } + return "", "" +} + +// codexCustomToolUseBlock builds a tool_use block from a custom_tool_call payload. +// Unlike function_call, the input is plain text (e.g., apply_patch content) rather +// than a JSON arguments string. +func codexCustomToolUseBlock(p codexPayload) map[string]json.RawMessage { + block := map[string]json.RawMessage{ + "type": mustJSON(transcript.ContentTypeToolUse), + "name": mustJSON(p.Name), + } + if p.CallID != "" { + block["id"] = mustJSON(p.CallID) + } + if p.Input != "" { + if inputJSON, err := json.Marshal(map[string]string{"input": p.Input}); err == nil { + block["input"] = inputJSON + } + } + return block +} + +// codexCustomOutputText extracts the output text from a custom_tool_call_output payload. +// The output field is an object {"type":"text","text":"..."} rather than a plain string. +func codexCustomOutputText(payload json.RawMessage) string { + var p struct { + Output json.RawMessage `json:"output"` + } + if json.Unmarshal(payload, &p) != nil || p.Output == nil { + return "" + } + // Try as plain string first (for forward compatibility). + var s string + if json.Unmarshal(p.Output, &s) == nil { + return s + } + // Try as object with "text" field. + var obj struct { + Text string `json:"text"` + } + if json.Unmarshal(p.Output, &obj) == nil { + return obj.Text + } + return "" +} + // mustJSON marshals v to JSON, panicking on error (only used for simple types). func mustJSON(v interface{}) json.RawMessage { b, err := json.Marshal(v) diff --git a/cmd/entire/cli/transcript/compact/codex_test.go b/cmd/entire/cli/transcript/compact/codex_test.go index a6a1bc839..360f0ca3b 100644 --- a/cmd/entire/cli/transcript/compact/codex_test.go +++ b/cmd/entire/cli/transcript/compact/codex_test.go @@ -52,6 +52,40 @@ func TestCompact_CodexInlineCases(t *testing.T) { `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"assistant","ts":"t2","content":[{"type":"text","text":"first line\n\nsecond line"}]}`, }, }, + { + name: "assistant with custom_tool_call apply_patch and output", + input: []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Creating the file now."}]}} +{"timestamp":"t3","type":"response_item","payload":{"type":"custom_tool_call","status":"completed","call_id":"call_1","name":"apply_patch","input":"*** Begin Patch\n*** Add File: hello.txt\n+Hello World\n*** End Patch\n"}} +{"timestamp":"t4","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_1","output":{"type":"text","text":"Success. Updated: A hello.txt"}}} +`), + expected: []string{ + `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"assistant","ts":"t2","content":[{"type":"text","text":"Creating the file now."},{"type":"tool_use","id":"call_1","name":"apply_patch","input":{"input":"*** Begin Patch\n*** Add File: hello.txt\n+Hello World\n*** End Patch\n"},"result":{"output":"Success. Updated: A hello.txt","status":"success"}}]}`, + }, + }, + { + name: "standalone custom_tool_call without preceding assistant text", + input: []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"t2","type":"response_item","payload":{"type":"custom_tool_call","status":"completed","call_id":"call_2","name":"apply_patch","input":"*** Begin Patch\n*** Update File: readme.md\n-old\n+new\n*** End Patch\n"}} +{"timestamp":"t3","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_2","output":{"type":"text","text":"Success. Updated: M readme.md"}}} +`), + expected: []string{ + `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"assistant","ts":"t2","content":[{"type":"tool_use","id":"call_2","name":"apply_patch","input":{"input":"*** Begin Patch\n*** Update File: readme.md\n-old\n+new\n*** End Patch\n"},"result":{"output":"Success. Updated: M readme.md","status":"success"}}]}`, + }, + }, + { + name: "mixed function_call and custom_tool_call after same assistant", + input: []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} +{"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Running then patching."}]}} +{"timestamp":"t3","type":"response_item","payload":{"type":"function_call","name":"exec_command","arguments":"{\"cmd\":\"ls\"}","call_id":"call_1"}} +{"timestamp":"t4","type":"response_item","payload":{"type":"function_call_output","call_id":"call_1","output":"file.txt"}} +{"timestamp":"t5","type":"response_item","payload":{"type":"custom_tool_call","status":"completed","call_id":"call_2","name":"apply_patch","input":"patch data"}} +{"timestamp":"t6","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_2","output":{"type":"text","text":"ok"}}} +`), + expected: []string{ + `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"assistant","ts":"t2","content":[{"type":"text","text":"Running then patching."},{"type":"tool_use","id":"call_1","name":"exec_command","input":{"cmd":"ls"},"result":{"output":"file.txt","status":"success"}},{"type":"tool_use","id":"call_2","name":"apply_patch","input":{"input":"patch data"},"result":{"output":"ok","status":"success"}}]}`, + }, + }, { name: "drops reasoning and event_msg", input: []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} From 37d742107642b77714ac688fe91bc7b94e3b078f Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 10 Apr 2026 10:38:25 +0200 Subject: [PATCH 4/4] Fix Codex compact transcript offset semantics Entire-Checkpoint: 5afb961f5b9b --- cmd/entire/cli/transcript/compact/codex.go | 34 ++++--------------- .../cli/transcript/compact/codex_test.go | 13 +++---- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/cmd/entire/cli/transcript/compact/codex.go b/cmd/entire/cli/transcript/compact/codex.go index 977115be5..2f6872758 100644 --- a/cmd/entire/cli/transcript/compact/codex.go +++ b/cmd/entire/cli/transcript/compact/codex.go @@ -71,18 +71,18 @@ type codexPayload struct { // compactCodex converts a Codex JSONL transcript into the compact format. func compactCodex(content []byte, opts MetadataFields) ([]byte, error) { - lines, err := parseCodexLines(content) - if err != nil { - return nil, err - } - if opts.StartLine > 0 { - lines = codexSliceFromResponseItem(lines, opts.StartLine) - if len(lines) == 0 { + content = transcript.SliceFromLine(content, opts.StartLine) + if content == nil { return []byte{}, nil } } + lines, err := parseCodexLines(content) + if err != nil { + return nil, err + } + base := newTranscriptLine(opts) var result []byte var pendingInTok, pendingOutTok int @@ -311,26 +311,6 @@ func codexAssistantText(raw json.RawMessage) string { return strings.Join(texts, "\n\n") } -// codexSliceFromResponseItem returns a suffix of lines starting after skipping -// n response_item entries. token_count lines do not count toward the offset. -func codexSliceFromResponseItem(lines []codexLine, n int) []codexLine { - if n <= 0 { - return lines - } - - seen := 0 - for i, line := range lines { - if line.Type == codexTypeResponseItem { - seen++ - } - if seen >= n { - return lines[i+1:] - } - } - - return nil -} - // codexToolUseBlock builds a compact tool_use content block from a function_call. func codexToolUseBlock(p codexPayload) map[string]json.RawMessage { block := map[string]json.RawMessage{ diff --git a/cmd/entire/cli/transcript/compact/codex_test.go b/cmd/entire/cli/transcript/compact/codex_test.go index 360f0ca3b..c2075ee83 100644 --- a/cmd/entire/cli/transcript/compact/codex_test.go +++ b/cmd/entire/cli/transcript/compact/codex_test.go @@ -162,9 +162,9 @@ func TestIsCodexFormat_LargeFirstLine(t *testing.T) { func TestCompact_CodexStartLine(t *testing.T) { t.Parallel() - // StartLine skips the first N response_item entries (not raw JSONL lines). - // There are 6 response_items here; StartLine=4 skips developer, AGENTS.md - // user, first-prompt user, and first assistant — leaving second user + assistant. + // StartLine skips raw JSONL lines to match Codex CheckpointTranscriptStart. + // StartLine=4 skips session_meta, developer, AGENTS.md user, and first prompt, + // leaving the first assistant response and everything after it. opts := MetadataFields{Agent: "codex", CLIVersion: "0.5.1", StartLine: 4} input := []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} @@ -177,6 +177,7 @@ func TestCompact_CodexStartLine(t *testing.T) { `) expected := []string{ + `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"assistant","ts":"t5","content":[{"type":"text","text":"response to first"}]}`, `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"user","ts":"t6","content":[{"text":"second prompt"}]}`, `{"v":1,"agent":"codex","cli_version":"0.5.1","type":"assistant","ts":"t7","content":[{"type":"text","text":"response to second"}]}`, } @@ -191,9 +192,9 @@ func TestCompact_CodexStartLine(t *testing.T) { func TestCompact_CodexStartLine_IgnoresTokenCountEvents(t *testing.T) { t.Parallel() - // StartLine=1 should skip exactly one response_item (the first user), - // not the token_count event line. - opts := MetadataFields{Agent: "codex", CLIVersion: "0.5.1", StartLine: 1} + // StartLine counts raw JSONL lines, so StartLine=2 skips session_meta and the + // first user line, leaving the token_count event to attach to the assistant. + opts := MetadataFields{Agent: "codex", CLIVersion: "0.5.1", StartLine: 2} input := []byte(`{"timestamp":"t1","type":"session_meta","payload":{"id":"s1"}} {"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"first prompt"}]}}