Feat(ai-gemini): Gemini Realtime Adapter

.changeset/huge-lizards-admire.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai-gemini': minor
+---
+
+Added Gemini Realtime Adapter

examples/ts-react-chat/src/lib/use-realtime.ts

@@ -6,12 +6,15 @@ import {
   elevenlabsRealtime,
   elevenlabsRealtimeToken,
 } from '@tanstack/ai-elevenlabs'
+import { geminiRealtime, geminiRealtimeToken } from '@tanstack/ai-gemini'
+import type { GeminiRealtimeVoice } from "@tanstack/ai-gemini"
+import type { OpenAIRealtimeVoice } from "@tanstack/ai-openai"
 import { realtimeClientTools } from '@/lib/realtime-tools'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 
 const getRealtimeTokenFn = createServerFn({ method: 'POST' })
-  .inputValidator((data: { provider: Provider; agentId?: string }) => {
+  .inputValidator((data: { provider?: Provider; agentId?: string }) => {
     if (!data.provider) throw new Error('Provider is required')
     return data
   })
@@ -24,6 +27,12 @@ const getRealtimeTokenFn = createServerFn({ method: 'POST' })
       })
     }
 
+    if (data.provider === 'gemini') {
+      return realtimeToken({
+        adapter: geminiRealtimeToken(),
+      })
+    }
+
     if (data.provider === 'elevenlabs') {
       const agentId = data.agentId || process.env.ELEVENLABS_AGENT_ID
       if (!agentId) {
@@ -46,16 +55,22 @@ export function useRealtime({
   temperature,
   maxOutputTokens,
   semanticEagerness,
+  voice
 }: {
   provider: Provider
   agentId: string
   outputModalities?: Array<'audio' | 'text'>
   temperature?: number
   maxOutputTokens?: number | 'inf'
   semanticEagerness?: 'low' | 'medium' | 'high'
+  voice?: OpenAIRealtimeVoice | GeminiRealtimeVoice
 }) {
   const adapter =
-    provider === 'openai' ? openaiRealtime() : elevenlabsRealtime()
+    provider === 'openai'
+      ? openaiRealtime()
+      : provider === 'gemini'
+        ? geminiRealtime()
+        : elevenlabsRealtime()
 
   return useRealtimeChat({
     getToken: () =>
@@ -78,7 +93,7 @@ Keep your responses concise and conversational since this is a voice interface.
 When using tools, briefly explain what you're doing and then share the results naturally.
 If the user sends an image, describe what you see and answer any questions about it.
 Be friendly and engaging!`,
-    voice: 'alloy',
+    voice: voice || provider === 'gemini' ? 'Puck' : 'alloy',
     tools: realtimeClientTools,
     outputModalities,
     temperature,

examples/ts-react-chat/src/routes/realtime.tsx

@@ -13,11 +13,12 @@ import {
 import { AudioSparkline } from '@/components/AudioSparkline'
 import { useRealtime } from '@/lib/use-realtime'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 type OutputMode = 'audio+text' | 'text-only' | 'audio-only'
 
 const PROVIDER_OPTIONS: Array<{ value: Provider; label: string }> = [
   { value: 'openai', label: 'OpenAI Realtime' },
+  { value: 'gemini', label: 'Google Gemini' },
   { value: 'elevenlabs', label: 'ElevenLabs' },
 ]
 
@@ -275,7 +276,7 @@ function RealtimePage() {
         </div>
 
         {/* Tools indicator */}
-        {provider === 'openai' && (
+        {(provider === 'openai' || provider === 'gemini') && (
           <div className="border-b border-orange-500/10 bg-gray-800/50 px-4 py-2">
             <div className="flex items-center gap-2 text-xs text-gray-400">
               <Wrench className="w-3 h-3" />

packages/typescript/ai-client/src/realtime-client.ts

@@ -100,8 +100,30 @@ export class RealtimeClient {
         this.clientTools.size > 0
           ? Array.from(this.clientTools.values())
           : undefined
+
+      const {
+        instructions,
+        voice,
+        vadMode,
+        outputModalities,
+        temperature,
+        maxOutputTokens,
+        semanticEagerness,
+        providerOptions,
+      } = this.options
+
       this.connection = await this.options.adapter.connect(
-        this.token,
+        this.token, 
+        {
+          instructions,
+          voice,
+          vadMode,
+          outputModalities,
+          temperature,
+          maxOutputTokens,
+          semanticEagerness,
+          providerOptions,
+        },
         toolsList,
       )
 
@@ -500,12 +522,12 @@ export class RealtimeClient {
 
     const toolsConfig = tools
       ? Array.from(this.clientTools.values()).map((t) => ({
-          name: t.name,
-          description: t.description,
-          inputSchema: t.inputSchema
-            ? convertSchemaToJsonSchema(t.inputSchema)
-            : undefined,
-        }))
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema
+          ? convertSchemaToJsonSchema(t.inputSchema)
+          : undefined,
+      }))
       : undefined
 
     this.connection.updateSession({

packages/typescript/ai-client/src/realtime-types.ts

@@ -25,11 +25,12 @@ export interface RealtimeAdapter {
   /**
    * Create a connection using the provided token
    * @param token - The ephemeral token from the server
-   * @param clientTools - Optional client-side tools to register with the provider
+   * @param config - Initial session configuration (voice, instructions, etc.)
    * @returns A connection instance
    */
   connect: (
     token: RealtimeToken,
+    config: RealtimeSessionConfig,
     clientTools?: ReadonlyArray<AnyClientTool>,
   ) => Promise<RealtimeConnection>
 }
@@ -148,6 +149,11 @@ export interface RealtimeClientOptions {
    */
   semanticEagerness?: 'low' | 'medium' | 'high'
 
+  /**
+   * Provider-specific options
+   */
+  providerOptions?: Record<string, unknown>
+
   // Callbacks
   onStatusChange?: (status: RealtimeStatus) => void
   onModeChange?: (mode: RealtimeMode) => void

packages/typescript/ai-elevenlabs/src/realtime/adapter.ts

@@ -40,9 +40,10 @@ export function elevenlabsRealtime(
 
     async connect(
       token: RealtimeToken,
+      _config: RealtimeSessionConfig,
       clientToolDefs?: ReadonlyArray<AnyClientTool>,
     ): Promise<RealtimeConnection> {
-      return createElevenLabsConnection(token, options, clientToolDefs)
+      return createElevenLabsConnection(token, clientToolDefs)
     },
   }
 }
@@ -52,7 +53,6 @@ export function elevenlabsRealtime(
  */
 async function createElevenLabsConnection(
   token: RealtimeToken,
-  _options: ElevenLabsRealtimeOptions,
   clientToolDefs?: ReadonlyArray<AnyClientTool>,
 ): Promise<RealtimeConnection> {
   const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()

packages/typescript/ai-gemini/package.json

@@ -40,14 +40,16 @@
     "adapter"
   ],
   "dependencies": {
-    "@google/genai": "^1.43.0"
+    "@google/genai": "^1.46.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "workspace:^"
+    "@tanstack/ai": "workspace:^",
+    "@tanstack/ai-client": "workspace:^"
   },
   "devDependencies": {
     "@tanstack/ai": "workspace:*",
+    "@tanstack/ai-client": "workspace:*",
     "@vitest/coverage-v8": "4.0.14",
-    "vite": "^7.2.7"
+    "vite": "^7.3.1"
   }
 }

packages/typescript/ai-gemini/src/index.ts

@@ -82,3 +82,17 @@ export type {
   GeminiDocumentMetadata,
   GeminiMessageMetadataByModality,
 } from './message-types'
+
+// ============================================================================
+// Realtime (Voice) Adapters
+// ============================================================================
+
+export { geminiRealtime, geminiRealtimeToken } from './realtime/index'
+
+export type {
+  GeminiRealtimeModel,
+  GeminiRealtimeOptions,
+  GeminiRealtimeTokenOptions,
+  GeminiRealtimeVoice,
+  GeminiRealtimeEvent
+} from './realtime/index'