Feat(ai-gemini): Gemini Realtime Adapter

.changeset/huge-lizards-admire.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai-gemini': minor
+---
+
+Added Gemini Realtime Adapter

examples/ts-react-chat/src/lib/use-realtime.ts

@@ -6,9 +6,10 @@ import {
   elevenlabsRealtime,
   elevenlabsRealtimeToken,
 } from '@tanstack/ai-elevenlabs'
+import { geminiRealtime, geminiRealtimeToken } from '@tanstack/ai-gemini'
 import { realtimeClientTools } from '@/lib/realtime-tools'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 
 const getRealtimeTokenFn = createServerFn({ method: 'POST' })
   .inputValidator((data: { provider: Provider; agentId?: string }) => {
@@ -24,6 +25,12 @@ const getRealtimeTokenFn = createServerFn({ method: 'POST' })
       })
     }
 
+    if (data.provider === 'gemini') {
+      return realtimeToken({
+        adapter: geminiRealtimeToken(),
+      })
+    }
+
     if (data.provider === 'elevenlabs') {
       const agentId = data.agentId || process.env.ELEVENLABS_AGENT_ID
       if (!agentId) {
@@ -55,7 +62,11 @@ export function useRealtime({
   semanticEagerness?: 'low' | 'medium' | 'high'
 }) {
   const adapter =
-    provider === 'openai' ? openaiRealtime() : elevenlabsRealtime()
+    provider === 'openai'
+      ? openaiRealtime()
+      : provider === 'gemini'
+        ? geminiRealtime()
+        : elevenlabsRealtime()
 
   return useRealtimeChat({
     getToken: () =>

examples/ts-react-chat/src/routes/realtime.tsx

@@ -13,11 +13,12 @@ import {
 import { AudioSparkline } from '@/components/AudioSparkline'
 import { useRealtime } from '@/lib/use-realtime'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 type OutputMode = 'audio+text' | 'text-only' | 'audio-only'
 
 const PROVIDER_OPTIONS: Array<{ value: Provider; label: string }> = [
   { value: 'openai', label: 'OpenAI Realtime' },
+  { value: 'gemini', label: 'Google Gemini' },
   { value: 'elevenlabs', label: 'ElevenLabs' },
 ]
 
@@ -275,7 +276,7 @@ function RealtimePage() {
         </div>
 
         {/* Tools indicator */}
-        {provider === 'openai' && (
+        {(provider === 'openai' || provider === 'gemini') && (
           <div className="border-b border-orange-500/10 bg-gray-800/50 px-4 py-2">
             <div className="flex items-center gap-2 text-xs text-gray-400">
               <Wrench className="w-3 h-3" />

packages/typescript/ai-client/src/realtime-client.ts

@@ -100,8 +100,30 @@ export class RealtimeClient {
         this.clientTools.size > 0
           ? Array.from(this.clientTools.values())
           : undefined
+
+      const {
+        instructions,
+        voice,
+        vadMode,
+        outputModalities,
+        temperature,
+        maxOutputTokens,
+        semanticEagerness,
+        providerOptions,
+      } = this.options
+
       this.connection = await this.options.adapter.connect(
-        this.token,
+        this.token, 
+        {
+          instructions,
+          voice,
+          vadMode,
+          outputModalities,
+          temperature,
+          maxOutputTokens,
+          semanticEagerness,
+          providerOptions,
+        },
         toolsList,
       )
 
@@ -500,12 +522,12 @@ export class RealtimeClient {
 
     const toolsConfig = tools
       ? Array.from(this.clientTools.values()).map((t) => ({
-          name: t.name,
-          description: t.description,
-          inputSchema: t.inputSchema
-            ? convertSchemaToJsonSchema(t.inputSchema)
-            : undefined,
-        }))
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema
+          ? convertSchemaToJsonSchema(t.inputSchema)
+          : undefined,
+      }))
       : undefined
 
     this.connection.updateSession({

packages/typescript/ai-client/src/realtime-types.ts

@@ -25,11 +25,12 @@ export interface RealtimeAdapter {
   /**
    * Create a connection using the provided token
    * @param token - The ephemeral token from the server
-   * @param clientTools - Optional client-side tools to register with the provider
+   * @param config - Initial session configuration (voice, instructions, etc.)
    * @returns A connection instance
    */
   connect: (
     token: RealtimeToken,
+    config: RealtimeSessionConfig,
     clientTools?: ReadonlyArray<AnyClientTool>,
   ) => Promise<RealtimeConnection>
 }
@@ -148,6 +149,11 @@ export interface RealtimeClientOptions {
    */
   semanticEagerness?: 'low' | 'medium' | 'high'
 
+  /**
+   * Provider-specific options
+   */
+  providerOptions?: Record<string, unknown>
+
   // Callbacks
   onStatusChange?: (status: RealtimeStatus) => void
   onModeChange?: (mode: RealtimeMode) => void

packages/typescript/ai-elevenlabs/src/realtime/adapter.ts

@@ -40,9 +40,10 @@ export function elevenlabsRealtime(
 
     async connect(
       token: RealtimeToken,
+      _config: RealtimeSessionConfig,
       clientToolDefs?: ReadonlyArray<AnyClientTool>,
     ): Promise<RealtimeConnection> {
-      return createElevenLabsConnection(token, options, clientToolDefs)
+      return createElevenLabsConnection(token, clientToolDefs)
     },
   }
 }
@@ -52,7 +53,6 @@ export function elevenlabsRealtime(
  */
 async function createElevenLabsConnection(
   token: RealtimeToken,
-  _options: ElevenLabsRealtimeOptions,
   clientToolDefs?: ReadonlyArray<AnyClientTool>,
 ): Promise<RealtimeConnection> {
   const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()

packages/typescript/ai-gemini/package.json

@@ -40,14 +40,16 @@
     "adapter"
   ],
   "dependencies": {
-    "@google/genai": "^1.43.0"
+    "@google/genai": "^1.46.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "workspace:^"
+    "@tanstack/ai": "workspace:^",
+    "@tanstack/ai-client": "workspace:^"
   },
   "devDependencies": {
     "@tanstack/ai": "workspace:*",
+    "@tanstack/ai-client": "workspace:*",
     "@vitest/coverage-v8": "4.0.14",
-    "vite": "^7.2.7"
+    "vite": "^7.3.1"
   }
 }

packages/typescript/ai-gemini/src/index.ts

@@ -82,3 +82,9 @@ export type {
   GeminiDocumentMetadata,
   GeminiMessageMetadataByModality,
 } from './message-types'
+
+// Realtime adapter
+export {
+  geminiRealtime,
+  geminiRealtimeToken,
+} from './realtime/index'