Feat(ai-gemini): Gemini Realtime Adapter

.changeset/huge-lizards-admire.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai-gemini': minor
+---
+
+Added Gemini Realtime Adapter

examples/ts-react-chat/src/lib/use-realtime.ts

@@ -6,9 +6,10 @@ import {
   elevenlabsRealtime,
   elevenlabsRealtimeToken,
 } from '@tanstack/ai-elevenlabs'
+import { geminiRealtime, geminiRealtimeToken } from '@tanstack/ai-gemini'
 import { realtimeClientTools } from '@/lib/realtime-tools'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 
 const getRealtimeTokenFn = createServerFn({ method: 'POST' })
   .inputValidator((data: { provider: Provider; agentId?: string }) => {
@@ -24,6 +25,12 @@ const getRealtimeTokenFn = createServerFn({ method: 'POST' })
       })
     }
 
+    if (data.provider === 'gemini') {
+      return realtimeToken({
+        adapter: geminiRealtimeToken(),
+      })
+    }
+
     if (data.provider === 'elevenlabs') {
       const agentId = data.agentId || process.env.ELEVENLABS_AGENT_ID
       if (!agentId) {
@@ -55,7 +62,11 @@ export function useRealtime({
   semanticEagerness?: 'low' | 'medium' | 'high'
 }) {
   const adapter =
-    provider === 'openai' ? openaiRealtime() : elevenlabsRealtime()
+    provider === 'openai'
+      ? openaiRealtime()
+      : provider === 'gemini'
+        ? geminiRealtime()
+        : elevenlabsRealtime()
 
   return useRealtimeChat({
     getToken: () =>

examples/ts-react-chat/src/routes/realtime.tsx

@@ -13,11 +13,12 @@ import {
 import { AudioSparkline } from '@/components/AudioSparkline'
 import { useRealtime } from '@/lib/use-realtime'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 type OutputMode = 'audio+text' | 'text-only' | 'audio-only'
 
 const PROVIDER_OPTIONS: Array<{ value: Provider; label: string }> = [
   { value: 'openai', label: 'OpenAI Realtime' },
+  { value: 'gemini', label: 'Google Gemini' },
   { value: 'elevenlabs', label: 'ElevenLabs' },
 ]
 
@@ -275,7 +276,7 @@ function RealtimePage() {
         </div>
 
         {/* Tools indicator */}
-        {provider === 'openai' && (
+        {(provider === 'openai' || provider === 'gemini') && (
           <div className="border-b border-orange-500/10 bg-gray-800/50 px-4 py-2">
             <div className="flex items-center gap-2 text-xs text-gray-400">
               <Wrench className="w-3 h-3" />

packages/typescript/ai-client/src/realtime-client.ts

@@ -96,14 +96,45 @@ export class RealtimeClient {
       this.scheduleTokenRefresh()
 
       // Connect via adapter (pass tools for providers like ElevenLabs that need them at connect time)
-      const toolsList =
-        this.clientTools.size > 0
-          ? Array.from(this.clientTools.values())
-          : undefined
-      this.connection = await this.options.adapter.connect(
-        this.token,
-        toolsList,
-      )
+      // const toolsList =
+      //   this.clientTools.size > 0
+      //     ? Array.from(this.clientTools.values())
+      //     : undefined
+
+      const toolsConfig = this.clientTools.size > 0
+        ? Array.from(this.clientTools.values()).map((t) => ({
+          name: t.name,
+          description: t.description,
+          inputSchema: t.inputSchema
+            ? convertSchemaToJsonSchema(t.inputSchema)
+            : undefined,
+          outputSchema: t.outputSchema
+            ? convertSchemaToJsonSchema(t.outputSchema)
+            : undefined,
+        }))
+        : undefined
+      const {
+        instructions,
+        voice,
+        vadMode,
+        outputModalities,
+        temperature,
+        maxOutputTokens,
+        semanticEagerness,
+        providerOptions,
+      } = this.options
+
+      this.connection = await this.options.adapter.connect(this.token, {
+        instructions,
+        voice,
+        vadMode,
+        outputModalities,
+        temperature,
+        maxOutputTokens,
+        semanticEagerness,
+        providerOptions,
+        tools: toolsConfig,
+      })
 
       // Subscribe to connection events
       this.subscribeToConnectionEvents()

packages/typescript/ai-client/src/realtime-types.ts

@@ -25,12 +25,12 @@ export interface RealtimeAdapter {
   /**
    * Create a connection using the provided token
    * @param token - The ephemeral token from the server
-   * @param clientTools - Optional client-side tools to register with the provider
+   * @param config - Initial session configuration (voice, instructions, etc.)
    * @returns A connection instance
    */
   connect: (
     token: RealtimeToken,
-    clientTools?: ReadonlyArray<AnyClientTool>,
+    config: RealtimeSessionConfig,
   ) => Promise<RealtimeConnection>
 }
 
@@ -148,6 +148,11 @@ export interface RealtimeClientOptions {
    */
   semanticEagerness?: 'low' | 'medium' | 'high'
 
+  /**
+   * Provider-specific options
+   */
+  providerOptions?: Record<string, unknown>
+
   // Callbacks
   onStatusChange?: (status: RealtimeStatus) => void
   onModeChange?: (mode: RealtimeMode) => void

packages/typescript/ai-gemini/package.json

@@ -40,14 +40,16 @@
     "adapter"
   ],
   "dependencies": {
-    "@google/genai": "^1.43.0"
+    "@google/genai": "^1.46.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "workspace:^"
+    "@tanstack/ai": "workspace:^",
+    "@tanstack/ai-client": "workspace:^"
   },
   "devDependencies": {
     "@tanstack/ai": "workspace:*",
+    "@tanstack/ai-client": "workspace:*",
     "@vitest/coverage-v8": "4.0.14",
-    "vite": "^7.2.7"
+    "vite": "^7.3.1"
   }
 }

packages/typescript/ai-gemini/src/index.ts

@@ -82,3 +82,9 @@ export type {
   GeminiDocumentMetadata,
   GeminiMessageMetadataByModality,
 } from './message-types'
+
+// Realtime adapter
+export {
+  geminiRealtime,
+  geminiRealtimeToken,
+} from './realtime/index'