diff --git a/.changeset/webrtc-device-option-parity.md b/.changeset/webrtc-device-option-parity.md new file mode 100644 index 00000000..037cc0fb --- /dev/null +++ b/.changeset/webrtc-device-option-parity.md @@ -0,0 +1,5 @@ +--- +"@elevenlabs/client": patch +--- + +Align `changeInputDevice`/`changeOutputDevice` across connection types: both the WebSocket and WebRTC paths now reject a `sampleRate` or `format` that differs from the one the connection was created with, instead of the WebSocket path silently ignoring it and the WebRTC path throwing on any value including the one already active. Re-passing the connection's current `sampleRate`/`format` alongside a device id, which callers routinely do, stays a no-op on both paths. diff --git a/packages/client/src/platform/web/input.ts b/packages/client/src/platform/web/input.ts index c83e670a..6319e4d1 100644 --- a/packages/client/src/platform/web/input.ts +++ b/packages/client/src/platform/web/input.ts @@ -115,6 +115,8 @@ export class MediaDeviceInput implements InputController, InputEventTarget { inputStream, source, permissions, + sampleRate, + format, onError ); } catch (error) { @@ -146,6 +148,8 @@ export class MediaDeviceInput implements InputController, InputEventTarget { private inputStream: MediaStream, private mediaStreamSource: MediaStreamAudioSourceNode, private permissions: PermissionStatus, + private readonly configuredSampleRate: number, + private readonly configuredFormat: FormatConfig["format"], private onError: ( message: string, context?: unknown @@ -221,14 +225,29 @@ export class MediaDeviceInput implements InputController, InputEventTarget { } this.settingInput = true; + // sampleRate and format cannot be changed on an existing input (would + // require recreating the AudioContext), so a caller asking for a value + // other than the one this input was created with gets a clear error + // instead of a silently mismatched format. Re-passing the same value + // back, which callers routinely do alongside an inputDeviceId change, + // stays a no-op. preferHeadphonesForIosDevices is a best-effort + // selection hint rather than a format guarantee, so it stays a silent + // no-op regardless, same as inputChunkDurationMs. All are only applied + // during MediaDeviceInput.create(). + if ( + (config?.sampleRate !== undefined && + config.sampleRate !== this.configuredSampleRate) || + (config?.format !== undefined && + config.format !== this.configuredFormat) + ) { + throw new Error( + "Input device does not support changing sampleRate or format after the connection is created" + ); + } + // Extract inputDeviceId from config const inputDeviceId = config?.inputDeviceId; - // Note: sampleRate, format, inputChunkDurationMs, and - // preferHeadphonesForIosDevices cannot be changed on an existing input - // (would require recreating the AudioContext). These options are only used - // during initial MediaDeviceInput.create() - // Create new constraints with the specified device or use default const options: MediaTrackConstraints = { ...defaultConstraints, diff --git a/packages/client/src/platform/web/output.ts b/packages/client/src/platform/web/output.ts index 656ba3f0..a8ce49b3 100644 --- a/packages/client/src/platform/web/output.ts +++ b/packages/client/src/platform/web/output.ts @@ -127,7 +127,9 @@ export class MediaDeviceOutput analyser, gain, worklet, - audioElement + audioElement, + sampleRate, + format ); return newOutput; @@ -155,7 +157,9 @@ export class MediaDeviceOutput private readonly analyser: AnalyserNode, private readonly gain: GainNode, private readonly worklet: AudioWorkletNode, - private readonly audioElement: HTMLAudioElement + private readonly audioElement: HTMLAudioElement, + private readonly configuredSampleRate: number, + private readonly configuredFormat: FormatConfig["format"] ) { // Start the MessagePort to enable addEventListener to work // (required when using addEventListener instead of onmessage) @@ -236,13 +240,25 @@ export class MediaDeviceOutput throw new Error("setSinkId is not supported in this browser"); } + // sampleRate and format cannot be changed on an existing output (would + // require recreating the AudioContext), so a caller asking for a value + // other than the one this output was created with gets a clear error + // instead of a silently mismatched format. Re-passing the same value + // back, which callers routinely do alongside an outputDeviceId change, + // stays a no-op. Only used during initial MediaDeviceOutput.create(). + if ( + (config?.sampleRate !== undefined && + config.sampleRate !== this.configuredSampleRate) || + (config?.format !== undefined && config.format !== this.configuredFormat) + ) { + throw new Error( + "Output device does not support changing sampleRate or format after the connection is created" + ); + } + // Extract outputDeviceId from config const outputDeviceId = config?.outputDeviceId; - // Note: sampleRate and format cannot be changed on an existing output - // (would require recreating the AudioContext). - // These options are only used during initial MediaDeviceOutput.create() - // If deviceId is undefined, use empty string which resets to default device await this.audioElement.setSinkId(outputDeviceId || ""); } diff --git a/packages/client/src/utils/WebRTCConnection.test.ts b/packages/client/src/utils/WebRTCConnection.test.ts index 6d7005c1..db6b2aa2 100644 --- a/packages/client/src/utils/WebRTCConnection.test.ts +++ b/packages/client/src/utils/WebRTCConnection.test.ts @@ -603,4 +603,210 @@ describe("WebRTCConnection", () => { connection.close(); }); }); + + describe("device option parity with the WebSocket path", () => { + function mockConnectedRoom() { + const mockRoom = new Room() as any; + (mockRoom.on as ReturnType).mockImplementation( + (event: string, callback: () => void) => { + if (event === "connected") { + queueMicrotask(callback); + } + } + ); + (mockRoom.once as ReturnType).mockImplementation( + (event: string, callback: () => void) => { + if (event === "signalConnected") { + queueMicrotask(callback); + } + } + ); + return mockRoom; + } + + it("rejects a sampleRate change on the input device, even alongside a device id", async () => { + mockConnectedRoom(); + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + await expect( + connection.input.setDevice({ + sampleRate: 16000, + inputDeviceId: "mic-2", + }) + ).rejects.toThrow(/sampleRate or format/); + + expect(createLocalAudioTrack).not.toHaveBeenCalled(); + + connection.close(); + }); + + it("rejects a format change on the input device", async () => { + mockConnectedRoom(); + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + await expect( + connection.input.setDevice({ format: "ulaw" }) + ).rejects.toThrow(/sampleRate or format/); + + connection.close(); + }); + + it("switches the input device when the negotiated sampleRate and format are re-passed unchanged", async () => { + const mockRoom = mockConnectedRoom(); + ( + mockRoom.localParticipant.getTrackPublication as ReturnType< + typeof vi.fn + > + ).mockReturnValue(undefined); + (createLocalAudioTrack as ReturnType).mockResolvedValue({ + mediaStreamTrack: { id: "new-track", kind: "audio" }, + }); + + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + // WebRTC always negotiates pcm_48000 (see connectionType: "webrtc" + // handling in create()), so re-sending that same pair is not a change + // request, and callers do this routinely alongside a device id. + await expect( + connection.input.setDevice({ + sampleRate: 48000, + format: "pcm", + inputDeviceId: "mic-2", + }) + ).resolves.toBeUndefined(); + + expect(createLocalAudioTrack).toHaveBeenCalledWith( + expect.objectContaining({ deviceId: { exact: "mic-2" } }) + ); + + connection.close(); + }); + + it("switches the input device when only preferHeadphonesForIosDevices is also passed", async () => { + const mockRoom = mockConnectedRoom(); + ( + mockRoom.localParticipant.getTrackPublication as ReturnType< + typeof vi.fn + > + ).mockReturnValue(undefined); + (createLocalAudioTrack as ReturnType).mockResolvedValue({ + mediaStreamTrack: { id: "new-track", kind: "audio" }, + }); + + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + await expect( + connection.input.setDevice({ + preferHeadphonesForIosDevices: true, + inputDeviceId: "mic-2", + }) + ).resolves.toBeUndefined(); + + expect(createLocalAudioTrack).toHaveBeenCalledWith( + expect.objectContaining({ deviceId: { exact: "mic-2" } }) + ); + + connection.close(); + }); + + it("stays a no-op when only preferHeadphonesForIosDevices is passed", async () => { + mockConnectedRoom(); + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + await expect( + connection.input.setDevice({ preferHeadphonesForIosDevices: true }) + ).resolves.toBeUndefined(); + + expect(createLocalAudioTrack).not.toHaveBeenCalled(); + + connection.close(); + }); + + it("rejects a sampleRate or format change on the output device, even alongside a device id", async () => { + mockConnectedRoom(); + const setOutputDevice = vi.fn(() => Promise.resolve()); + setWebRTCAudioAdapterFactory(() => ({ + attachRemoteTrack: vi.fn(() => Promise.resolve()), + setupInputAnalysis: vi.fn(() => ({ volumeProvider: NO_VOLUME })), + setupOutputAnalysis: vi.fn(() => + Promise.resolve({ volumeProvider: NO_VOLUME }) + ), + setVolume: vi.fn(), + setOutputDevice, + cleanup: vi.fn(), + })); + + try { + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + await expect( + connection.output.setDevice({ + sampleRate: 16000, + format: "pcm", + outputDeviceId: "speaker-2", + }) + ).rejects.toThrow(/sampleRate or format/); + + expect(setOutputDevice).not.toHaveBeenCalled(); + + connection.close(); + } finally { + setWebRTCAudioAdapterFactory(() => new WebAudioAdapter()); + } + }); + + it("switches the output device when the negotiated sampleRate and format are re-passed unchanged", async () => { + mockConnectedRoom(); + const setOutputDevice = vi.fn(() => Promise.resolve()); + setWebRTCAudioAdapterFactory(() => ({ + attachRemoteTrack: vi.fn(() => Promise.resolve()), + setupInputAnalysis: vi.fn(() => ({ volumeProvider: NO_VOLUME })), + setupOutputAnalysis: vi.fn(() => + Promise.resolve({ volumeProvider: NO_VOLUME }) + ), + setVolume: vi.fn(), + setOutputDevice, + cleanup: vi.fn(), + })); + + try { + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + await expect( + connection.output.setDevice({ + sampleRate: 48000, + format: "pcm", + outputDeviceId: "speaker-2", + }) + ).resolves.toBeUndefined(); + + expect(setOutputDevice).toHaveBeenCalledWith("speaker-2"); + + connection.close(); + } finally { + setWebRTCAudioAdapterFactory(() => new WebAudioAdapter()); + } + }); + }); }); diff --git a/packages/client/src/utils/WebRTCConnection.ts b/packages/client/src/utils/WebRTCConnection.ts index 82165db2..c6ee178e 100644 --- a/packages/client/src/utils/WebRTCConnection.ts +++ b/packages/client/src/utils/WebRTCConnection.ts @@ -96,15 +96,22 @@ export class WebRTCConnection extends BaseConnection { } }, setDevice: async (config?: Partial & InputDeviceConfig) => { - // WebRTC only supports changing inputDeviceId - // sampleRate, format, and preferHeadphonesForIosDevices are not supported + // sampleRate and format cannot be applied to an in-progress WebRTC or + // WebSocket input, on top of an inputDeviceId change or on their own, so + // both paths reject a value other than the one already negotiated + // rather than silently keeping the old format. Re-passing that same + // value back, which callers routinely do alongside an inputDeviceId + // change, stays a no-op — see the WebSocket input path for why. This is + // the one intentional asymmetry with preferHeadphonesForIosDevices, + // which stays a best-effort hint the caller cannot rely on either way. if ( - config?.sampleRate !== undefined || - config?.format !== undefined || - config?.preferHeadphonesForIosDevices !== undefined + (config?.sampleRate !== undefined && + config.sampleRate !== this.inputFormat.sampleRate) || + (config?.format !== undefined && + config.format !== this.inputFormat.format) ) { throw new Error( - "WebRTC input device does not support sampleRate, format, or preferHeadphonesForIosDevices options" + "WebRTC input device does not support changing sampleRate or format after the connection is created" ); } @@ -183,11 +190,17 @@ export class WebRTCConnection extends BaseConnection { // Audio elements are cleaned up when the connection closes }, setDevice: async (config?: Partial & OutputDeviceConfig) => { - // WebRTC only supports changing outputDeviceId - // sampleRate and format are not supported - if (config?.sampleRate !== undefined || config?.format !== undefined) { + // See the input controller above: sampleRate and format cannot be + // applied after creation on either connection type, so both reject a + // value other than the one already negotiated. + if ( + (config?.sampleRate !== undefined && + config.sampleRate !== this.outputFormat.sampleRate) || + (config?.format !== undefined && + config.format !== this.outputFormat.format) + ) { throw new Error( - "WebRTC output device does not support sampleRate or format options" + "WebRTC output device does not support changing sampleRate or format after the connection is created" ); } diff --git a/packages/client/src/utils/input.test.ts b/packages/client/src/utils/input.test.ts index 5b1ba3bd..6d5d1ae6 100644 --- a/packages/client/src/utils/input.test.ts +++ b/packages/client/src/utils/input.test.ts @@ -25,4 +25,39 @@ describe("MediaDeviceInput", () => { expect(encodedAudio.length).toBeGreaterThan(0); expect(volume).toBeTypeOf("number"); }); + + describe("setDevice", () => { + it("rejects a sampleRate change after creation, even alongside a device id", async () => { + input = await MediaDeviceInput.create({ + sampleRate: 16000, + format: "pcm", + }); + + await expect( + input.setDevice({ sampleRate: 48000, inputDeviceId: "mic-2" }) + ).rejects.toThrow(/sampleRate or format/); + }); + + it("rejects a format change after creation", async () => { + input = await MediaDeviceInput.create({ + sampleRate: 16000, + format: "pcm", + }); + + await expect(input.setDevice({ format: "ulaw" })).rejects.toThrow( + /sampleRate or format/ + ); + }); + + it("stays a no-op when only preferHeadphonesForIosDevices is passed", async () => { + input = await MediaDeviceInput.create({ + sampleRate: 16000, + format: "pcm", + }); + + await expect( + input.setDevice({ preferHeadphonesForIosDevices: true }) + ).resolves.toBeUndefined(); + }); + }); }); diff --git a/packages/client/src/utils/output.test.ts b/packages/client/src/utils/output.test.ts new file mode 100644 index 00000000..cbeb3129 --- /dev/null +++ b/packages/client/src/utils/output.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, beforeAll, vi } from "vitest"; +import { MediaDeviceOutput } from "../platform/web/output.js"; + +// MediaDeviceOutput.create() needs a real AudioContext/AudioWorklet, which +// this package's Node test project does not provide (only src/utils/input.ts +// runs in the browser project, and output's own setup, unlike input's, +// depends on AudioContext.resume() settling, which headless Chromium's +// autoplay policy leaves permanently pending outside a real browser test +// environment). The sampleRate/format guard below runs before setDevice +// touches any instance state, so it is exercised directly against the class's +// real setDevice implementation, without going through create(). +describe("MediaDeviceOutput.setDevice", () => { + beforeAll(() => { + function FakeHTMLAudioElement() {} + FakeHTMLAudioElement.prototype = { setSinkId: () => {} }; + vi.stubGlobal("HTMLAudioElement", FakeHTMLAudioElement); + }); + + function uncreatedOutput(): MediaDeviceOutput { + return Object.create(MediaDeviceOutput.prototype) as MediaDeviceOutput; + } + + it("rejects a sampleRate change, even alongside a device id", async () => { + await expect( + uncreatedOutput().setDevice({ + sampleRate: 48000, + outputDeviceId: "speaker-2", + }) + ).rejects.toThrow(/sampleRate or format/); + }); + + it("rejects a format change", async () => { + await expect( + uncreatedOutput().setDevice({ format: "ulaw" }) + ).rejects.toThrow(/sampleRate or format/); + }); +});