From 27ad7165d243c628aee94d96f74cbb8a2195d326 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 18 Dec 2025 00:29:34 +0000 Subject: [PATCH 1/7] feat(go/plugins/googlegenai): add multi-part tools support --- go/ai/generate.go | 13 +++++- go/go.mod | 2 +- go/go.sum | 4 +- go/plugins/googlegenai/gemini.go | 57 +++++++++++++++++++++++-- go/plugins/googlegenai/gemini_test.go | 61 +++++++++++++++++++++++++++ go/samples/basic-gemini/main.go | 41 +++++++++++++----- 6 files changed, 160 insertions(+), 18 deletions(-) diff --git a/go/ai/generate.go b/go/ai/generate.go index 2bea88240f..75239ccff0 100644 --- a/go/ai/generate.go +++ b/go/ai/generate.go @@ -389,7 +389,18 @@ func GenerateWithRequest(ctx context.Context, r api.Registry, opts *GenerateActi return resp, nil } - return generate(ctx, newReq, currentTurn+1, currentIndex+1) + finalResp, err := generate(ctx, newReq, currentTurn+1, currentIndex+1) + if err != nil { + return nil, err + } + + if finalResp.Message != nil && resp.Message != nil { + var reasoningParts []*Part + reasoningParts = append(reasoningParts, resp.Message.Content...) + finalResp.Message.Content = append(reasoningParts, finalResp.Message.Content...) + } + return finalResp, nil + // return generate(ctx, newReq, currentTurn+1, currentIndex+1) }) } diff --git a/go/go.mod b/go/go.mod index 3aa1cd948b..3472c0f4cb 100644 --- a/go/go.mod +++ b/go/go.mod @@ -41,7 +41,7 @@ require ( golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 golang.org/x/tools v0.34.0 google.golang.org/api v0.236.0 - google.golang.org/genai v1.36.0 + google.golang.org/genai v1.40.0 ) require ( diff --git a/go/go.sum b/go/go.sum index 43f5ac29cd..e7abcc1495 100644 --- a/go/go.sum +++ b/go/go.sum @@ -537,8 +537,8 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine/v2 v2.0.6 h1:LvPZLGuchSBslPBp+LAhihBeGSiRh1myRoYK4NtuBIw= google.golang.org/appengine/v2 v2.0.6/go.mod h1:WoEXGoXNfa0mLvaH5sV3ZSGXwVmy8yf7Z1JKf3J3wLI= -google.golang.org/genai v1.36.0 h1:sJCIjqTAmwrtAIaemtTiKkg2TO1RxnYEusTmEQ3nGxM= -google.golang.org/genai v1.36.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= +google.golang.org/genai v1.40.0 h1:kYxyQSH+vsib8dvsgyLJzsVEIv5k3ZmHJyVqdvGncmc= +google.golang.org/genai v1.40.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= diff --git a/go/plugins/googlegenai/gemini.go b/go/plugins/googlegenai/gemini.go index 047c3785eb..3f37712ed5 100644 --- a/go/plugins/googlegenai/gemini.go +++ b/go/plugins/googlegenai/gemini.go @@ -478,6 +478,34 @@ func toGeminiTools(inTools []*ai.ToolDefinition) ([]*genai.Tool, error) { return outTools, nil } +// toGeminiFunctionResponsePart translates a slice of [ai.Part] to a slice of [genai.FunctionResponsePart] +func toGeminiFunctionResponsePart(parts []*ai.Part) ([]*genai.FunctionResponsePart, error) { + frp := []*genai.FunctionResponsePart{} + for _, p := range parts { + switch { + case p.IsData(): + contentType, data, err := uri.Data(p) + if err != nil { + return nil, err + } + frp = append(frp, genai.NewFunctionResponsePartFromBytes(data, contentType)) + case p.IsMedia(): + if strings.HasPrefix(p.Text, "data:") { + contentType, data, err := uri.Data(p) + if err != nil { + return nil, err + } + frp = append(frp, genai.NewFunctionResponsePartFromBytes(data, contentType)) + continue + } + frp = append(frp, genai.NewFunctionResponsePartFromURI(p.Text, p.ContentType)) + default: + return nil, fmt.Errorf("unsupported function response part type: %d", p.Kind) + } + } + return frp, nil +} + // mergeTools consolidates all FunctionDeclarations into a single Tool // while preserving non-function tools (Retrieval, GoogleSearch, CodeExecution, etc.) func mergeTools(ts []*genai.Tool) []*genai.Tool { @@ -808,6 +836,14 @@ func translateCandidate(cand *genai.Candidate) (*ai.ModelResponse, error) { Name: part.FunctionCall.Name, Input: part.FunctionCall.Args, }) + // FunctionCall parts may contain a ThoughtSignature that must be preserved + // and returned in subsequent requests for the tool call to be valid. + if len(part.ThoughtSignature) > 0 { + if p.Metadata == nil { + p.Metadata = make(map[string]any) + } + p.Metadata["signature"] = part.ThoughtSignature + } } if part.CodeExecutionResult != nil { partFound++ @@ -888,7 +924,7 @@ func toGeminiParts(parts []*ai.Part) ([]*genai.Part, error) { func toGeminiPart(p *ai.Part) (*genai.Part, error) { switch { case p.IsReasoning(): - // TODO: go-genai does not support genai.NewPartFromThought() + // NOTE: go-genai does not support genai.NewPartFromThought() signature := []byte{} if p.Metadata != nil { if sig, ok := p.Metadata["signature"].([]byte); ok { @@ -928,8 +964,17 @@ func toGeminiPart(p *ai.Part) (*genai.Part, error) { "content": toolResp.Output, } } - fr := genai.NewPartFromFunctionResponse(toolResp.Name, output) - return fr, nil + if multiPart, ok := p.Metadata["multipart"].(bool); ok { + if multiPart { + toolRespParts, err := toGeminiFunctionResponsePart(toolResp.Content) + if err != nil { + return nil, err + } + return genai.NewPartFromFunctionResponseWithParts(toolResp.Name, output, toolRespParts), nil + } + } + fmt.Printf("tool response: %#v\n", toolResp.Content) + return genai.NewPartFromFunctionResponse(toolResp.Name, output), nil case p.IsToolRequest(): toolReq := p.ToolRequest var input map[string]any @@ -941,6 +986,12 @@ func toGeminiPart(p *ai.Part) (*genai.Part, error) { } } fc := genai.NewPartFromFunctionCall(toolReq.Name, input) + // Restore ThoughtSignature if present in metadata + if p.Metadata != nil { + if sig, ok := p.Metadata["signature"].([]byte); ok { + fc.ThoughtSignature = sig + } + } return fc, nil default: panic("unknown part type in a request") diff --git a/go/plugins/googlegenai/gemini_test.go b/go/plugins/googlegenai/gemini_test.go index d9332e0cd4..8ab332848b 100644 --- a/go/plugins/googlegenai/gemini_test.go +++ b/go/plugins/googlegenai/gemini_test.go @@ -717,3 +717,64 @@ func genToolName(length int, chars string) string { } return string(r) } + +func TestToGeminiParts_MultipartToolResponse(t *testing.T) { + // Create a tool response with both output and additional content (text) + toolResp := &ai.ToolResponse{ + Name: "generateImage", + Output: map[string]any{"status": "success"}, + Content: []*ai.Part{ + ai.NewTextPart("Generated image description"), + }, + } + + // Create an ai.Part wrapping the tool response + part := ai.NewToolResponsePart(toolResp) + + // Convert to Gemini parts + geminiParts, err := toGeminiParts([]*ai.Part{part}) + if err != nil { + t.Fatalf("toGeminiParts failed: %v", err) + } + + // Expecting 2 parts: 1 for function response, 1 for text content + if len(geminiParts) != 2 { + t.Fatalf("expected 2 Gemini parts, got %d", len(geminiParts)) + } + + // Check first part (FunctionResponse) + if geminiParts[0].FunctionResponse == nil { + t.Error("expected first part to be FunctionResponse") + } + if geminiParts[0].FunctionResponse.Name != "generateImage" { + t.Errorf("expected function name 'generateImage', got %q", geminiParts[0].FunctionResponse.Name) + } + + // Check second part (Text) + if geminiParts[1].Text != "Generated image description" { + t.Errorf("expected second part text 'Generated image description', got %q", geminiParts[1].Text) + } +} + +func TestToGeminiParts_SimpleToolResponse(t *testing.T) { + // Create a simple tool response (no content) + toolResp := &ai.ToolResponse{ + Name: "search", + Output: map[string]any{"result": "foo"}, + } + + part := ai.NewToolResponsePart(toolResp) + + geminiParts, err := toGeminiParts([]*ai.Part{part}) + if err != nil { + t.Fatalf("toGeminiParts failed: %v", err) + } + + if len(geminiParts) != 1 { + t.Fatalf("expected 1 Gemini part, got %d", len(geminiParts)) + } + + if geminiParts[0].FunctionResponse == nil { + t.Error("expected part to be FunctionResponse") + } +} diff --git a/go/samples/basic-gemini/main.go b/go/samples/basic-gemini/main.go index 6be8c112ac..c6b264be92 100644 --- a/go/samples/basic-gemini/main.go +++ b/go/samples/basic-gemini/main.go @@ -16,6 +16,7 @@ package main import ( "context" + "fmt" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -32,26 +33,44 @@ func main() { // practice. g := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{})) - // Define a simple flow that generates jokes about a given topic - genkit.DefineStreamingFlow(g, "jokesFlow", func(ctx context.Context, input string, cb ai.ModelStreamCallback) (string, error) { - type Joke struct { - Joke string `json:"joke"` - Category string `json:"jokeCategory" description:"What is the joke about"` - } + // Define a multipart tool. + // This simulates a tool that "generates" an invitation card and returns content (description) + invitationTool := genkit.DefineMultipartTool(g, "createInvitationCard", "Creates a greeting card", + func(ctx *ai.ToolContext, input struct { + Name string `json:"name"` + Occasion string `json:"occasion"` + }, + ) (*ai.MultipartToolResponse, error) { + rectangle := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAHIAAABUAQMAAABk5vEVAAAABlBMVEX///8AAABVwtN+" + + "AAAAI0lEQVR4nGNgGHaA/z8UHIDwOWASDqP8Uf7w56On/1FAQwAAVM0exw1hqwkAAAAASUVORK5CYII=" + return &ai.MultipartToolResponse{ + Output: map[string]any{"success": true}, + Content: []*ai.Part{ + // ai.NewTextPart(fmt.Sprintf("I created an invitation card for %s for their %s. It features a beautiful rectangle.", input.Name, input.Occasion)), + ai.NewMediaPart("image/png", rectangle), + }, + }, nil + }, + ) - genkit.DefineSchemaFor[Joke](g) + type InvitationCard struct { + Ocassion string `json:"occasion"` + } + // Define a simple flow that uses the multipart tool + genkit.DefineStreamingFlow(g, "cardFlow", func(ctx context.Context, input InvitationCard, cb ai.ModelStreamCallback) (string, error) { resp, err := genkit.Generate(ctx, g, - ai.WithModelName("googleai/gemini-2.5-flash"), + ai.WithModelName("googleai/gemini-3-pro-preview"), ai.WithConfig(&genai.GenerateContentConfig{ Temperature: genai.Ptr[float32](1.0), ThinkingConfig: &genai.ThinkingConfig{ - ThinkingBudget: genai.Ptr[int32](0), + ThinkingLevel: genai.ThinkingLevelHigh, }, }), + ai.WithTools(invitationTool), ai.WithStreaming(cb), - ai.WithOutputSchemaName("joke"), - ai.WithPrompt(`Tell short jokes about %s`, input)) + ai.WithPrompt(fmt.Sprintf("Create an invitation card for the following ocassion: %s. Create one for Alex and another one for Pavel. Describe what you made.", input.Ocassion)), + ) if err != nil { return "", err } From 57b678bfb5103789cfc8fad78125d81fa9714514 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 18 Dec 2025 01:10:27 +0000 Subject: [PATCH 2/7] fix multi-part issues --- go/plugins/googlegenai/gemini.go | 19 +++-- go/plugins/googlegenai/gemini_test.go | 88 ++++++++++++-------- go/plugins/googlegenai/googleai_live_test.go | 37 +++++++- 3 files changed, 99 insertions(+), 45 deletions(-) diff --git a/go/plugins/googlegenai/gemini.go b/go/plugins/googlegenai/gemini.go index 3f37712ed5..ea59b1d586 100644 --- a/go/plugins/googlegenai/gemini.go +++ b/go/plugins/googlegenai/gemini.go @@ -964,16 +964,21 @@ func toGeminiPart(p *ai.Part) (*genai.Part, error) { "content": toolResp.Output, } } + + var isMultipart bool if multiPart, ok := p.Metadata["multipart"].(bool); ok { - if multiPart { - toolRespParts, err := toGeminiFunctionResponsePart(toolResp.Content) - if err != nil { - return nil, err - } - return genai.NewPartFromFunctionResponseWithParts(toolResp.Name, output, toolRespParts), nil + isMultipart = multiPart + } + if len(toolResp.Content) > 0 { + isMultipart = true + } + if isMultipart { + toolRespParts, err := toGeminiFunctionResponsePart(toolResp.Content) + if err != nil { + return nil, err } + return genai.NewPartFromFunctionResponseWithParts(toolResp.Name, output, toolRespParts), nil } - fmt.Printf("tool response: %#v\n", toolResp.Content) return genai.NewPartFromFunctionResponse(toolResp.Name, output), nil case p.IsToolRequest(): toolReq := p.ToolRequest diff --git a/go/plugins/googlegenai/gemini_test.go b/go/plugins/googlegenai/gemini_test.go index 8ab332848b..f7e9a39e1c 100644 --- a/go/plugins/googlegenai/gemini_test.go +++ b/go/plugins/googlegenai/gemini_test.go @@ -719,41 +719,59 @@ func genToolName(length int, chars string) string { } func TestToGeminiParts_MultipartToolResponse(t *testing.T) { - // Create a tool response with both output and additional content (text) - toolResp := &ai.ToolResponse{ - Name: "generateImage", - Output: map[string]any{"status": "success"}, - Content: []*ai.Part{ - ai.NewTextPart("Generated image description"), - }, - } - - // Create an ai.Part wrapping the tool response - part := ai.NewToolResponsePart(toolResp) - - // Convert to Gemini parts - geminiParts, err := toGeminiParts([]*ai.Part{part}) - if err != nil { - t.Fatalf("toGeminiParts failed: %v", err) - } - - // Expecting 2 parts: 1 for function response, 1 for text content - if len(geminiParts) != 2 { - t.Fatalf("expected 2 Gemini parts, got %d", len(geminiParts)) - } - - // Check first part (FunctionResponse) - if geminiParts[0].FunctionResponse == nil { - t.Error("expected first part to be FunctionResponse") - } - if geminiParts[0].FunctionResponse.Name != "generateImage" { - t.Errorf("expected function name 'generateImage', got %q", geminiParts[0].FunctionResponse.Name) - } - - // Check second part (Text) - if geminiParts[1].Text != "Generated image description" { - t.Errorf("expected second part text 'Generated image description', got %q", geminiParts[1].Text) - } + t.Run("Success", func(t *testing.T) { + // Create a tool response with both output and additional content (media) + toolResp := &ai.ToolResponse{ + Name: "generateImage", + Output: map[string]any{"status": "success"}, + Content: []*ai.Part{ + ai.NewMediaPart("image/png", "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="), + }, + } + + // Create an ai.Part wrapping the tool response + part := ai.NewToolResponsePart(toolResp) + // CRITICAL: Set the multipart metadata + part.Metadata = map[string]any{"multipart": true} + + // Convert to Gemini parts + geminiParts, err := toGeminiParts([]*ai.Part{part}) + if err != nil { + t.Fatalf("toGeminiParts failed: %v", err) + } + + // Expecting 1 part which contains the function response with internal parts + if len(geminiParts) != 1 { + t.Fatalf("expected 1 Gemini part, got %d", len(geminiParts)) + } + + // Check first part (FunctionResponse) + if geminiParts[0].FunctionResponse == nil { + t.Error("expected first part to be FunctionResponse") + } + if geminiParts[0].FunctionResponse.Name != "generateImage" { + t.Errorf("expected function name 'generateImage', got %q", geminiParts[0].FunctionResponse.Name) + } + }) + + t.Run("Fail_UnsupportedPartType", func(t *testing.T) { + // Create a tool response with text content (unsupported for multipart) + toolResp := &ai.ToolResponse{ + Name: "generateText", + Output: map[string]any{"status": "success"}, + Content: []*ai.Part{ + ai.NewTextPart("Generated text"), + }, + } + + part := ai.NewToolResponsePart(toolResp) + part.Metadata = map[string]any{"multipart": true} + + _, err := toGeminiParts([]*ai.Part{part}) + if err == nil { + t.Fatal("expected error for unsupported text part in multipart response, got nil") + } + }) } func TestToGeminiParts_SimpleToolResponse(t *testing.T) { diff --git a/go/plugins/googlegenai/googleai_live_test.go b/go/plugins/googlegenai/googleai_live_test.go index 4e78ab4f4c..783eccd239 100644 --- a/go/plugins/googlegenai/googleai_live_test.go +++ b/go/plugins/googlegenai/googleai_live_test.go @@ -170,7 +170,7 @@ func TestGoogleAILive(t *testing.T) { t.Fatal(err) } - out := resp.Message.Content[0].Text + out := resp.Text() const want = "11.31" if !strings.Contains(out, want) { t.Errorf("got %q, expecting it to contain %q", out, want) @@ -219,7 +219,7 @@ func TestGoogleAILive(t *testing.T) { t.Fatal(err) } - out := resp.Message.Content[0].Text + out := resp.Text() const want = "11.31" if !strings.Contains(out, want) { t.Errorf("got %q, expecting it to contain %q", out, want) @@ -307,7 +307,7 @@ func TestGoogleAILive(t *testing.T) { t.Fatal(err) } - out := resp.Message.Content[0].Text + out := resp.Text() const doNotWant = "11.31" if strings.Contains(out, doNotWant) { t.Errorf("got %q, expecting it NOT to contain %q", out, doNotWant) @@ -582,6 +582,37 @@ func TestGoogleAILive(t *testing.T) { t.Fatal("thoughts tokens should be zero") } }) + t.Run("multipart tool", func(t *testing.T) { + m := googlegenai.GoogleAIModel(g, "gemini-3-pro-preview") + img64, err := fetchImgAsBase64() + if err != nil { + t.Fatal(err) + } + + tool := genkit.DefineMultipartTool(g, "getImage", "returns a misterious image", + func(ctx *ai.ToolContext, input any) (*ai.MultipartToolResponse, error) { + return &ai.MultipartToolResponse{ + Output: map[string]any{"status": "success"}, + Content: []*ai.Part{ + ai.NewMediaPart("image/jpeg", "data:image/jpeg;base64,"+img64), + }, + }, nil + }, + ) + + resp, err := genkit.Generate(ctx, g, + ai.WithModel(m), + ai.WithTools(tool), + ai.WithPrompt("get an image and tell me what is in it"), + ) + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(strings.ToLower(resp.Text()), "cat") { + t.Errorf("expected response to contain 'cat', got: %s", resp.Text()) + } + }) } func TestCacheHelper(t *testing.T) { From a1c141ef176c40ae50a980e3d21b3109757e19c7 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 18 Dec 2025 01:34:12 +0000 Subject: [PATCH 3/7] fmt --- go/plugins/googlegenai/gemini_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/plugins/googlegenai/gemini_test.go b/go/plugins/googlegenai/gemini_test.go index f7e9a39e1c..442ea41dfe 100644 --- a/go/plugins/googlegenai/gemini_test.go +++ b/go/plugins/googlegenai/gemini_test.go @@ -780,18 +780,18 @@ func TestToGeminiParts_SimpleToolResponse(t *testing.T) { Name: "search", Output: map[string]any{"result": "foo"}, } - + part := ai.NewToolResponsePart(toolResp) - + geminiParts, err := toGeminiParts([]*ai.Part{part}) if err != nil { t.Fatalf("toGeminiParts failed: %v", err) } - + if len(geminiParts) != 1 { t.Fatalf("expected 1 Gemini part, got %d", len(geminiParts)) } - + if geminiParts[0].FunctionResponse == nil { t.Error("expected part to be FunctionResponse") } From d59c37017730908676662c05840fd26668fbb0a3 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 18 Dec 2025 01:37:15 +0000 Subject: [PATCH 4/7] remove code from ai/generate --- go/ai/generate.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/go/ai/generate.go b/go/ai/generate.go index 75239ccff0..2bea88240f 100644 --- a/go/ai/generate.go +++ b/go/ai/generate.go @@ -389,18 +389,7 @@ func GenerateWithRequest(ctx context.Context, r api.Registry, opts *GenerateActi return resp, nil } - finalResp, err := generate(ctx, newReq, currentTurn+1, currentIndex+1) - if err != nil { - return nil, err - } - - if finalResp.Message != nil && resp.Message != nil { - var reasoningParts []*Part - reasoningParts = append(reasoningParts, resp.Message.Content...) - finalResp.Message.Content = append(reasoningParts, finalResp.Message.Content...) - } - return finalResp, nil - // return generate(ctx, newReq, currentTurn+1, currentIndex+1) + return generate(ctx, newReq, currentTurn+1, currentIndex+1) }) } From 9910d0a51b548af087b14fb35f1f2307f9cb37bb Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 18 Dec 2025 18:57:18 +0000 Subject: [PATCH 5/7] update sample and GoogleAI live tests --- go/plugins/googlegenai/gemini_test.go | 31 ++++++++++++--------------- go/samples/basic-gemini/main.go | 27 +++++++++++++++++------ 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/go/plugins/googlegenai/gemini_test.go b/go/plugins/googlegenai/gemini_test.go index 442ea41dfe..a7c36b9761 100644 --- a/go/plugins/googlegenai/gemini_test.go +++ b/go/plugins/googlegenai/gemini_test.go @@ -707,19 +707,8 @@ func TestValidToolName(t *testing.T) { } } -// genToolName generates a string of a specified length using only -// the valid characters for a Gemini Tool name -func genToolName(length int, chars string) string { - r := make([]byte, length) - - for i := range length { - r[i] = chars[i%len(chars)] - } - return string(r) -} - func TestToGeminiParts_MultipartToolResponse(t *testing.T) { - t.Run("Success", func(t *testing.T) { + t.Run("ValidPartType", func(t *testing.T) { // Create a tool response with both output and additional content (media) toolResp := &ai.ToolResponse{ Name: "generateImage", @@ -729,12 +718,10 @@ func TestToGeminiParts_MultipartToolResponse(t *testing.T) { }, } - // Create an ai.Part wrapping the tool response + // create a mock ToolResponsePart, setting "multipart" to true is required part := ai.NewToolResponsePart(toolResp) - // CRITICAL: Set the multipart metadata part.Metadata = map[string]any{"multipart": true} - // Convert to Gemini parts geminiParts, err := toGeminiParts([]*ai.Part{part}) if err != nil { t.Fatalf("toGeminiParts failed: %v", err) @@ -745,7 +732,6 @@ func TestToGeminiParts_MultipartToolResponse(t *testing.T) { t.Fatalf("expected 1 Gemini part, got %d", len(geminiParts)) } - // Check first part (FunctionResponse) if geminiParts[0].FunctionResponse == nil { t.Error("expected first part to be FunctionResponse") } @@ -754,7 +740,7 @@ func TestToGeminiParts_MultipartToolResponse(t *testing.T) { } }) - t.Run("Fail_UnsupportedPartType", func(t *testing.T) { + t.Run("UnsupportedPartType", func(t *testing.T) { // Create a tool response with text content (unsupported for multipart) toolResp := &ai.ToolResponse{ Name: "generateText", @@ -796,3 +782,14 @@ func TestToGeminiParts_SimpleToolResponse(t *testing.T) { t.Error("expected part to be FunctionResponse") } } + +// genToolName generates a string of a specified length using only +// the valid characters for a Gemini Tool name +func genToolName(length int, chars string) string { + r := make([]byte, length) + + for i := range length { + r[i] = chars[i%len(chars)] + } + return string(r) +} diff --git a/go/samples/basic-gemini/main.go b/go/samples/basic-gemini/main.go index c6b264be92..4a8503b7a9 100644 --- a/go/samples/basic-gemini/main.go +++ b/go/samples/basic-gemini/main.go @@ -34,7 +34,7 @@ func main() { g := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{})) // Define a multipart tool. - // This simulates a tool that "generates" an invitation card and returns content (description) + // This simulates a tool that "generates" an invitation card invitationTool := genkit.DefineMultipartTool(g, "createInvitationCard", "Creates a greeting card", func(ctx *ai.ToolContext, input struct { Name string `json:"name"` @@ -46,7 +46,6 @@ func main() { return &ai.MultipartToolResponse{ Output: map[string]any{"success": true}, Content: []*ai.Part{ - // ai.NewTextPart(fmt.Sprintf("I created an invitation card for %s for their %s. It features a beautiful rectangle.", input.Name, input.Occasion)), ai.NewMediaPart("image/png", rectangle), }, }, nil @@ -54,11 +53,11 @@ func main() { ) type InvitationCard struct { - Ocassion string `json:"occasion"` + Occasion string `json:"occasion"` } // Define a simple flow that uses the multipart tool - genkit.DefineStreamingFlow(g, "cardFlow", func(ctx context.Context, input InvitationCard, cb ai.ModelStreamCallback) (string, error) { + genkit.DefineStreamingFlow(g, "cardFlow", func(ctx context.Context, input InvitationCard, cb ai.ModelStreamCallback) ([]string, error) { resp, err := genkit.Generate(ctx, g, ai.WithModelName("googleai/gemini-3-pro-preview"), ai.WithConfig(&genai.GenerateContentConfig{ @@ -69,13 +68,27 @@ func main() { }), ai.WithTools(invitationTool), ai.WithStreaming(cb), - ai.WithPrompt(fmt.Sprintf("Create an invitation card for the following ocassion: %s. Create one for Alex and another one for Pavel. Describe what you made.", input.Ocassion)), + ai.WithPrompt(fmt.Sprintf("Create an invitation card for the following ocassion: %s. Create one for Alex and another one for Pavel", input.Occasion)), ) if err != nil { - return "", err + return nil, err } - return resp.Text(), nil + invitations := []string{} + for _, m := range resp.History() { + if m.Role == ai.RoleTool { + for _, p := range m.Content { + if p.IsToolResponse() { + for _, contentPart := range p.ToolResponse.Content { + if contentPart.IsMedia() { + invitations = append(invitations, contentPart.Text) + } + } + } + } + } + } + return invitations, nil }) <-ctx.Done() From af663bbd32d80437c5a3b3129bfeb1a290706a94 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Fri, 19 Dec 2025 19:15:48 +0000 Subject: [PATCH 6/7] create a new sample --- go/samples/basic-gemini/main.go | 58 ++++++------------------- go/samples/multipart-tools/main.go | 68 ++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 45 deletions(-) create mode 100644 go/samples/multipart-tools/main.go diff --git a/go/samples/basic-gemini/main.go b/go/samples/basic-gemini/main.go index 4a8503b7a9..e61ec9df42 100644 --- a/go/samples/basic-gemini/main.go +++ b/go/samples/basic-gemini/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "fmt" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -33,62 +32,31 @@ func main() { // practice. g := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{})) - // Define a multipart tool. - // This simulates a tool that "generates" an invitation card - invitationTool := genkit.DefineMultipartTool(g, "createInvitationCard", "Creates a greeting card", - func(ctx *ai.ToolContext, input struct { - Name string `json:"name"` - Occasion string `json:"occasion"` - }, - ) (*ai.MultipartToolResponse, error) { - rectangle := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAHIAAABUAQMAAABk5vEVAAAABlBMVEX///8AAABVwtN+" + - "AAAAI0lEQVR4nGNgGHaA/z8UHIDwOWASDqP8Uf7w56On/1FAQwAAVM0exw1hqwkAAAAASUVORK5CYII=" - return &ai.MultipartToolResponse{ - Output: map[string]any{"success": true}, - Content: []*ai.Part{ - ai.NewMediaPart("image/png", rectangle), - }, - }, nil - }, - ) + // Define a simple flow that generates jokes about a given topic + genkit.DefineStreamingFlow(g, "jokesFlow", func(ctx context.Context, input string, cb ai.ModelStreamCallback) (string, error) { + type Joke struct { + Joke string `json:"joke"` + Category string `json:"jokeCategory" description:"What is the joke about"` + } - type InvitationCard struct { - Occasion string `json:"occasion"` - } + genkit.DefineSchemaFor[Joke](g) - // Define a simple flow that uses the multipart tool - genkit.DefineStreamingFlow(g, "cardFlow", func(ctx context.Context, input InvitationCard, cb ai.ModelStreamCallback) ([]string, error) { resp, err := genkit.Generate(ctx, g, - ai.WithModelName("googleai/gemini-3-pro-preview"), + ai.WithModelName("googleai/gemini-2.5-flash"), ai.WithConfig(&genai.GenerateContentConfig{ Temperature: genai.Ptr[float32](1.0), ThinkingConfig: &genai.ThinkingConfig{ - ThinkingLevel: genai.ThinkingLevelHigh, + ThinkingBudget: genai.Ptr[int32](0), }, }), - ai.WithTools(invitationTool), ai.WithStreaming(cb), - ai.WithPrompt(fmt.Sprintf("Create an invitation card for the following ocassion: %s. Create one for Alex and another one for Pavel", input.Occasion)), - ) + ai.WithOutputSchemaName("Joke"), + ai.WithPrompt(`Tell short jokes about %s`, input)) if err != nil { - return nil, err + return "", err } - invitations := []string{} - for _, m := range resp.History() { - if m.Role == ai.RoleTool { - for _, p := range m.Content { - if p.IsToolResponse() { - for _, contentPart := range p.ToolResponse.Content { - if contentPart.IsMedia() { - invitations = append(invitations, contentPart.Text) - } - } - } - } - } - } - return invitations, nil + return resp.Text(), nil }) <-ctx.Done() diff --git a/go/samples/multipart-tools/main.go b/go/samples/multipart-tools/main.go new file mode 100644 index 0000000000..c9cb04bdc3 --- /dev/null +++ b/go/samples/multipart-tools/main.go @@ -0,0 +1,68 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + + "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/genkit" + "github.com/firebase/genkit/go/plugins/googlegenai" + "google.golang.org/genai" +) + +func main() { + ctx := context.Background() + + g := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{})) + + // Define a multipart tool. + // This simulates a tool that takes a screenshot + screenshot := genkit.DefineMultipartTool(g, "screenshot", "Takes a screenshot", + func(ctx *ai.ToolContext, input any) (*ai.MultipartToolResponse, error) { + rectangle := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAHIAAABUAQMAAABk5vEVAAAABlBMVEX///8AAABVwtN+" + + "AAAAI0lEQVR4nGNgGHaA/z8UHIDwOWASDqP8Uf7w56On/1FAQwAAVM0exw1hqwkAAAAASUVORK5CYII=" + return &ai.MultipartToolResponse{ + Output: map[string]any{"success": true}, + Content: []*ai.Part{ + ai.NewMediaPart("image/png", rectangle), + }, + }, nil + }, + ) + + // Define a simple flow that uses the multipart tool + genkit.DefineStreamingFlow(g, "cardFlow", func(ctx context.Context, input any, cb ai.ModelStreamCallback) (string, error) { + resp, err := genkit.Generate(ctx, g, + ai.WithModelName("googleai/gemini-3-pro-preview"), + ai.WithConfig(&genai.GenerateContentConfig{ + Temperature: genai.Ptr[float32](1.0), + ThinkingConfig: &genai.ThinkingConfig{ + ThinkingLevel: genai.ThinkingLevelHigh, + }, + }), + ai.WithTools(screenshot), + ai.WithStreaming(cb), + ai.WithPrompt("Tell me what I'm seeing in the screen"), + ) + if err != nil { + return "", err + } + + return resp.Text(), nil + }) + + <-ctx.Done() +} From f0eaa03fac717bd2a8e4aaccae28844b22cacfc2 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Fri, 19 Dec 2025 19:50:28 +0000 Subject: [PATCH 7/7] fix formatter test case --- go/ai/formatter_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/go/ai/formatter_test.go b/go/ai/formatter_test.go index 40afd0008d..792247e665 100644 --- a/go/ai/formatter_test.go +++ b/go/ai/formatter_test.go @@ -661,17 +661,14 @@ func TestResolveFormat(t *testing.T) { } }) - t.Run("defaults to text even when schema present but no format", func(t *testing.T) { + t.Run("defaults to json even when schema present but no format", func(t *testing.T) { schema := map[string]any{"type": "object"} formatter, err := resolveFormat(r, schema, "") if err != nil { t.Fatalf("resolveFormat() error = %v", err) } - // Note: The current implementation defaults to text when format is empty, - // even if schema is present. The schema/format combination is typically - // handled at a higher level (e.g., in Generate options). - if formatter.Name() != OutputFormatText { - t.Errorf("resolveFormat() = %q, want %q", formatter.Name(), OutputFormatText) + if formatter.Name() != OutputFormatJSON { + t.Errorf("resolveFormat() = %q, want %q", formatter.Name(), OutputFormatJSON) } })