feat(ai-elevenlabs): Add session overrides to ElevenLabs adapter

.changeset/lazy-plums-knock.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai-elevenlabs': minor
+---
+
+add session overrides support

packages/typescript/ai-elevenlabs/README.md

@@ -62,6 +62,78 @@ function VoiceChat() {
 }
 ```
 
+## Session Overrides
+
+You can customize the agent session at two levels: **server-side** (via the token adapter) and **client-side** (via the realtime adapter).
+
+### Server-side overrides (token generation)
+
+Pass overrides to `elevenlabsRealtimeToken()` to bake them into the signed URL:
+
+```typescript
+const token = await realtimeToken({
+  adapter: elevenlabsRealtimeToken({
+    agentId: 'your-agent-id',
+    overrides: {
+      systemPrompt: 'You are a helpful assistant. Be concise.',
+      firstMessage: 'Hi! How can I help you today?',
+      voiceId: 'your-voice-id',
+      language: 'en',
+    },
+  }),
+})
+```
+
+| Option         | Type     | Description                                   |
+| -------------- | -------- | --------------------------------------------- |
+| `systemPrompt` | `string` | Custom system prompt for the agent            |
+| `firstMessage` | `string` | First message the agent speaks when connected |
+| `voiceId`      | `string` | ElevenLabs voice ID                           |
+| `language`     | `string` | Language code (e.g. `'en'`, `'es'`, `'fr'`)   |
+
+### Client-side overrides (adapter options)
+
+Pass overrides to `elevenlabsRealtime()` on the client. These take precedence over token-level overrides for agent prompt, firstMessage, and language.
+
+```typescript
+const client = new RealtimeClient({
+  getToken: () => fetch('/api/realtime-token').then((r) => r.json()),
+  adapter: elevenlabsRealtime({
+    overrides: {
+      agent: {
+        prompt: { prompt: 'You are a helpful assistant.' },
+        firstMessage: 'Hello! How can I assist you?',
+        language: 'en',
+      },
+      tts: {
+        voiceId: 'your-voice-id',
+        speed: 1.0,
+        stability: 0.5,
+        similarityBoost: 0.8,
+      },
+      conversation: {
+        textOnly: false,
+      },
+    },
+  }),
+})
+```
+
+| Option                  | Type      | Description                                  |
+| ----------------------- | --------- | -------------------------------------------- |
+| `agent.prompt.prompt`   | `string`  | System prompt (overrides token instructions) |
+| `agent.firstMessage`    | `string`  | First message the agent speaks               |
+| `agent.language`        | `string`  | Language code                                |
+| `tts.voiceId`           | `string`  | ElevenLabs voice ID                          |
+| `tts.speed`             | `number`  | Speaking speed multiplier                    |
+| `tts.stability`         | `number`  | Voice stability (0–1)                        |
+| `tts.similarityBoost`   | `number`  | Voice similarity boost (0–1)                 |
+| `conversation.textOnly` | `boolean` | Disable audio, use text only                 |
+
+### Override precedence
+
+When both levels are set, client-side `overrides.agent.prompt.prompt` takes precedence over the server-side `systemPrompt`. If only the server-side prompt is set, it is used as the fallback.
+
 ## Environment Variables
 
 Set `ELEVENLABS_API_KEY` in your environment for server-side token generation.

packages/typescript/ai-elevenlabs/package.json

@@ -41,7 +41,7 @@
     "test:types": "tsc"
   },
   "dependencies": {
-    "@11labs/client": "^0.2.0"
+    "@elevenlabs/client": "^1.2.0"
   },
   "peerDependencies": {
     "@tanstack/ai": "workspace:^",

packages/typescript/ai-elevenlabs/src/realtime/adapter.ts

@@ -1,4 +1,5 @@
-import { Conversation } from '@11labs/client'
+import { Conversation } from '@elevenlabs/client'
+import type { Language } from '@elevenlabs/client'
 import type {
   AnyClientTool,
   AudioVisualization,
@@ -16,7 +17,7 @@ import type { ElevenLabsRealtimeOptions } from './types'
 /**
  * Creates an ElevenLabs realtime adapter for client-side use.
  *
- * Wraps the @11labs/client SDK for voice conversations.
+ * Wraps the @elevenlabs/client SDK for voice conversations.
  *
  * @param options - Optional configuration
  * @returns A RealtimeAdapter for use with RealtimeClient
@@ -47,6 +48,56 @@ export function elevenlabsRealtime(
   }
 }
 
+type SessionOverrides = NonNullable<
+  Parameters<typeof Conversation.startSession>[0]['overrides']
+>
+
+/**
+ * Merges token instructions with user-provided overrides into
+ * the shape expected by Conversation.startSession.
+ * Option overrides take precedence over token instructions.
+ */
+function buildOverrides(
+  instructions: string | undefined,
+  optionOverrides: ElevenLabsRealtimeOptions['overrides'],
+): SessionOverrides | undefined {
+  const hasInstructions = instructions !== undefined
+  const hasOptions = optionOverrides !== undefined
+
+  if (!hasInstructions && !hasOptions) return undefined
+
+  const overrides: SessionOverrides = {}
+
+  if (hasInstructions || optionOverrides?.agent) {
+    const agentOverrides = optionOverrides?.agent
+
+    overrides.agent = {
+      prompt: {
+        prompt: agentOverrides?.prompt?.prompt ?? instructions,
+      },
+      firstMessage: agentOverrides?.firstMessage,
+      language: agentOverrides?.language as Language | undefined,
+    }
+  }
+
+  if (optionOverrides?.tts) {
+    overrides.tts = {
+      voiceId: optionOverrides.tts.voiceId,
+      speed: optionOverrides.tts.speed,
+      stability: optionOverrides.tts.stability,
+      similarityBoost: optionOverrides.tts.similarityBoost,
+    }
+  }
+
+  if (optionOverrides?.conversation) {
+    overrides.conversation = {
+      textOnly: optionOverrides.conversation.textOnly,
+    }
+  }
+
+  return overrides
+}
+
 /**
  * Creates a connection to ElevenLabs conversational AI
  */
@@ -161,6 +212,15 @@ async function createElevenLabsConnection(
     sessionOptions.clientTools = elevenLabsClientTools
   }
 
+  const overrides = buildOverrides(
+    token.config.instructions,
+    _options.overrides,
+  )
+
+  if (overrides) {
+    sessionOptions.overrides = overrides
+  }
+
   // Start the conversation session
   conversation = await Conversation.startSession(
     sessionOptions as Parameters<typeof Conversation.startSession>[0],

packages/typescript/ai-elevenlabs/src/realtime/types.ts

@@ -25,6 +25,25 @@ export interface ElevenLabsRealtimeOptions {
   connectionMode?: 'websocket' | 'webrtc'
   /** Enable debug logging */
   debug?: boolean
+  /** Optional overrides passed directly to Conversation.startSession */
+  overrides?: {
+    agent?: {
+      prompt?: {
+        prompt?: string
+      }
+      firstMessage?: string
+      language?: string
+    }
+    tts?: {
+      voiceId?: string
+      speed?: number
+      stability?: number
+      similarityBoost?: number
+    }
+    conversation?: {
+      textOnly?: boolean
+    }
+  }
 }
 
 /**

packages/typescript/ai-elevenlabs/tests/realtime-adapter.test.ts

@@ -5,7 +5,7 @@ import type { AnyClientTool, RealtimeMessage } from '@tanstack/ai'
 // Capture the session options passed to Conversation.startSession
 let capturedSessionOptions: Record<string, any> = {}
 
-vi.mock('@11labs/client', () => ({
+vi.mock('@elevenlabs/client', () => ({
   Conversation: {
     startSession: vi.fn(async (options: Record<string, any>) => {
       capturedSessionOptions = options
@@ -33,6 +33,7 @@ describe('elevenlabsRealtime adapter', () => {
   const fakeToken = {
     token: 'wss://fake-signed-url',
     expiresAt: Date.now() + 60_000,
+    config: {},
   }
 
   async function createConnection(
@@ -145,6 +146,161 @@ describe('elevenlabsRealtime adapter', () => {
     })
   })
 
+  describe('overrides', () => {
+    const fakeTokenWithInstructions = {
+      token: 'wss://fake-signed-url',
+      expiresAt: Date.now() + 60_000,
+      config: { instructions: 'Be concise.' },
+    }
+
+    async function createConnectionWithOptions(
+      adapterOptions: import('../src/realtime/types').ElevenLabsRealtimeOptions,
+      token: typeof fakeToken = fakeToken,
+    ): Promise<RealtimeConnection> {
+      const adapter = elevenlabsRealtime(adapterOptions)
+      return adapter.connect(token as any, undefined)
+    }
+
+    it('should not include overrides when no instructions and no overrides option', async () => {
+      await createConnectionWithOptions({}, { ...fakeToken, config: {} } as any)
+
+      expect(capturedSessionOptions.overrides).toBeUndefined()
+    })
+
+    it('should set agent prompt from token instructions', async () => {
+      await createConnectionWithOptions({}, fakeTokenWithInstructions as any)
+
+      expect(capturedSessionOptions.overrides).toMatchObject({
+        agent: {
+          prompt: { prompt: 'Be concise.' },
+        },
+      })
+    })
+
+    it('should set agent prompt from options, ignoring token instructions', async () => {
+      await createConnectionWithOptions(
+        {
+          overrides: {
+            agent: { prompt: { prompt: 'Custom system prompt.' } },
+          },
+        },
+        fakeTokenWithInstructions as any,
+      )
+
+      expect(capturedSessionOptions.overrides?.agent?.prompt?.prompt).toBe(
+        'Custom system prompt.',
+      )
+    })
+
+    it('should fall back to token instructions when options prompt is absent', async () => {
+      await createConnectionWithOptions(
+        {
+          overrides: {
+            agent: { firstMessage: 'Hello!' },
+          },
+        },
+        fakeTokenWithInstructions as any,
+      )
+
+      expect(capturedSessionOptions.overrides?.agent?.prompt?.prompt).toBe(
+        'Be concise.',
+      )
+      expect(capturedSessionOptions.overrides?.agent?.firstMessage).toBe(
+        'Hello!',
+      )
+    })
+
+    it('should set agent firstMessage and language', async () => {
+      await createConnectionWithOptions(
+        {
+          overrides: {
+            agent: {
+              firstMessage: 'Hola!',
+              language: 'es',
+            },
+          },
+        },
+        { ...fakeToken, config: {} } as any,
+      )
+
+      expect(capturedSessionOptions.overrides?.agent?.firstMessage).toBe(
+        'Hola!',
+      )
+      expect(capturedSessionOptions.overrides?.agent?.language).toBe('es')
+    })
+
+    it('should set tts overrides correctly', async () => {
+      await createConnectionWithOptions(
+        {
+          overrides: {
+            tts: {
+              voiceId: 'voice-abc',
+              speed: 1.2,
+              stability: 0.7,
+              similarityBoost: 0.9,
+            },
+          },
+        },
+        { ...fakeToken, config: {} } as any,
+      )
+
+      expect(capturedSessionOptions.overrides?.tts).toMatchObject({
+        voiceId: 'voice-abc',
+        speed: 1.2,
+        stability: 0.7,
+        similarityBoost: 0.9,
+      })
+    })
+
+    it('should set conversation textOnly override', async () => {
+      await createConnectionWithOptions(
+        {
+          overrides: {
+            conversation: { textOnly: true },
+          },
+        },
+        { ...fakeToken, config: {} } as any,
+      )
+
+      expect(capturedSessionOptions.overrides?.conversation?.textOnly).toBe(
+        true,
+      )
+    })
+
+    it('should set all override sections simultaneously', async () => {
+      await createConnectionWithOptions(
+        {
+          overrides: {
+            agent: {
+              prompt: { prompt: 'Full test prompt.' },
+              firstMessage: 'Hi there!',
+              language: 'fr',
+            },
+            tts: {
+              voiceId: 'voice-xyz',
+              speed: 0.9,
+            },
+            conversation: { textOnly: false },
+          },
+        },
+        { ...fakeToken, config: {} } as any,
+      )
+
+      expect(capturedSessionOptions.overrides).toMatchObject({
+        agent: {
+          prompt: { prompt: 'Full test prompt.' },
+          firstMessage: 'Hi there!',
+          language: 'fr',
+        },
+        tts: {
+          voiceId: 'voice-xyz',
+          speed: 0.9,
+        },
+        conversation: { textOnly: false },
+      })
+    })
+  })
+
   describe('clientTools registration', () => {
     it('should pass client tools as plain functions to @11labs/client', async () => {
       const mockTool: AnyClientTool = {