Feat/td 1747 fallback voice

apps/chat-bot/src/utils/audio/mp3-stream-player.ts

@@ -107,6 +107,8 @@ export class Mp3StreamPlayer {
   private networkDone = false;
   // Set once the element plays a Blob URL of the whole audio instead of the MediaSource.
   private usingFullSource = false;
+  // Latest seek target requested before the full source's metadata loaded.
+  private pendingSeekTime: number | null = null;
   private duration = 0;
   private playbackRate = 1;
   private mp3Chunks: Uint8Array<ArrayBufferLike>[] = [];
@@ -191,14 +193,9 @@ export class Mp3StreamPlayer {
     const target = Math.min(Math.max(time, 0), this.duration);
     if (!this.isFullyBuffered()) {
       this.switchToFullSource(audio);
-      // Some browsers ignore seeks before the new source's metadata has loaded.
-      audio.addEventListener(
-        'loadedmetadata',
-        () => {
-          audio.currentTime = target;
-        },
-        { once: true },
-      );
+    }
+    if (this.usingFullSource && audio.readyState < HTMLMediaElement.HAVE_METADATA) {
+      this.pendingSeekTime = target;
     }
     audio.currentTime = target;
     if (this.state === 'paused') {
@@ -280,6 +277,7 @@ export class Mp3StreamPlayer {
 
   dispose(): void {
     this.disposed = true;
+    this.pendingSeekTime = null;
     this.notifyFeeder();
     if (this.audio) {
       this.audio.pause();
@@ -500,6 +498,19 @@ export class Mp3StreamPlayer {
     // Same narrowing as for `appendBuffer`: fetch response bytes are never SharedArrayBuffers.
     const blob = new Blob(this.mp3Chunks as Uint8Array<ArrayBuffer>[], { type: MP3_MIME_TYPE });
     this.objectUrl = URL.createObjectURL(blob);
+    this.pendingSeekTime = null;
+    // Some browsers ignore seeks before the new source's metadata has loaded.
+    audio.addEventListener(
+      'loadedmetadata',
+      () => {
+        if (this.disposed || this.pendingSeekTime === null) {
+          return;
+        }
+        audio.currentTime = this.pendingSeekTime;
+        this.pendingSeekTime = null;
+      },
+      { once: true },
+    );
     audio.src = this.objectUrl;
   }
 

packages/shared/src/speech/speech-service.test.ts

@@ -1,5 +1,4 @@
 import type { LlmModelSelectModel } from '@shared/db/schema';
-import { InvalidArgumentError } from '@shared/error';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 const mocks = vi.hoisted(() => ({
@@ -42,12 +41,16 @@ beforeEach(() => {
 });
 
 describe('generateSpeechPreview', () => {
-  it('rejects an unknown voice without generating speech', async () => {
-    await expect(
-      generateSpeechPreview({ text: 'Hello', voice: 'does-not-exist' }),
-    ).rejects.toBeInstanceOf(InvalidArgumentError);
+  it('falls back to the first configured voice for an unknown voice', async () => {
+    mocks.getCachedSpeechPreview.mockResolvedValue(undefined);
 
-    expect(mocks.generateSpeechById).not.toHaveBeenCalled();
+    await generateSpeechPreview({ text: 'Hello', voice: 'does-not-exist' });
+
+    expect(mocks.generateSpeechById).toHaveBeenCalledWith({
+      modelId: speechModel.id,
+      text: 'Hello',
+      voice: 'alloy',
+    });
   });
 
   it('returns the cached preview without regenerating or re-caching', async () => {

packages/shared/src/speech/speech-service.ts

@@ -11,7 +11,6 @@ import { dbGetSpeechModel } from '@shared/db/functions/llm-model';
 import { dbInsertConversationUsage } from '@shared/db/functions/token-usage';
 import { LlmModelSelectModel } from '@shared/db/schema';
 import { ConversationModel } from '@shared/db/types';
-import { InvalidArgumentError } from '@shared/error';
 import { FederalStateModel } from '@shared/federal-states/types';
 import { constructNewMessageEvent, sendRabbitmqEvent } from '@shared/rabbitmq';
 import { getCachedSpeechPreview, setCachedSpeechPreview } from '@shared/speech/speech-cache';
@@ -21,6 +20,7 @@ import {
 } from '@shared/speech/streaming/speech-pipeline';
 
 import { ChatSpeechStreamResult } from './types';
+import { resolveSpeechVoice } from './utils';
 
 type SpeechContext = {
   user: UserModel;
@@ -142,15 +142,9 @@ export async function generateSpeechPreview({
 
   if (!speechModel) throw new Error('No speech model configured');
 
-  const voices = speechModel.modelConfig?.voices ?? [];
+  const resolvedVoice = resolveSpeechVoice(speechModel, voice);
 
-  if (voices.length === 0) throw new Error('No voices configured for the speech model');
-
-  if (!voices.some((v) => v.name === voice)) {
-    throw new InvalidArgumentError(`Unknown voice: ${voice}`);
-  }
-
-  const cached = await getCachedSpeechPreview(speechModel.id, voice, text);
+  const cached = await getCachedSpeechPreview(speechModel.id, resolvedVoice, text);
   if (cached !== undefined) {
     return {
       buffer: cached,
@@ -161,10 +155,10 @@ export async function generateSpeechPreview({
   const { audioBuffer, mimeType } = await generateSpeechById({
     modelId: speechModel.id,
     text,
-    voice,
+    voice: resolvedVoice,
   });
 
-  await setCachedSpeechPreview(speechModel.id, voice, text, audioBuffer);
+  await setCachedSpeechPreview(speechModel.id, resolvedVoice, text, audioBuffer);
 
   return {
     buffer: audioBuffer,

packages/shared/src/speech/streaming/start-streaming.ts

@@ -2,30 +2,18 @@ import { streamSpeechById } from '@ais-chat/ai-core';
 import { LlmModelSelectModel } from '@shared/db/schema';
 
 import { ChatSpeechStreamResult } from '../types';
-import { calculateSpeechCostsInCent } from '../utils';
+import { calculateSpeechCostsInCent, resolveSpeechVoice } from '../utils';
 import type { SpeechStreamingPipelineParams } from './speech-pipeline';
 
 export const startStream = async (
   params: SpeechStreamingPipelineParams,
   model: LlmModelSelectModel,
   onStreamFinished: (result: ChatSpeechStreamResult) => Promise<void>,
 ): Promise<ChatSpeechStreamResult> => {
-  const voices = model.modelConfig?.voices ?? [];
-  const firstVoice = voices.at(0);
-
-  if (firstVoice === undefined) {
-    throw new Error('No voices configured for the speech model');
-  }
-
-  const selectedVoice =
-    params.voice && voices.some((voice) => voice.name === params.voice)
-      ? params.voice
-      : firstVoice.name;
-
   const { chunks, usage, mimeType } = await streamSpeechById({
     modelId: model.id,
     text: params.input,
-    voice: selectedVoice,
+    voice: resolveSpeechVoice(model, params.voice),
     abortSignal: params.abortSignal,
   });
 

packages/shared/src/speech/utils.ts

@@ -24,3 +24,22 @@ export function calculateSpeechCostsInCent(model: LlmModelSelectModel, usage: Sp
 
   return (inputTextTokenPrice + outputAudioTokenPrice) / TOKEN_AMOUNT_PER_PRICE;
 }
+
+/**
+ * Returns the names of the voices configured for the speech model.
+ * Throws if no voices are configured.
+ */
+export function getConfiguredVoiceNames(model: LlmModelSelectModel): [string, ...string[]] {
+  const [first, ...rest] = (model.modelConfig?.voices ?? []).map((v) => v.name);
+  if (first === undefined) throw new Error('No voices configured for the speech model');
+  return [first, ...rest];
+}
+
+/**
+ * Returns the given voice if it is configured for the speech model,
+ * otherwise falls back to the first configured voice.
+ */
+export function resolveSpeechVoice(model: LlmModelSelectModel, voice?: string): string {
+  const voices = getConfiguredVoiceNames(model);
+  return voice && voices.includes(voice) ? voice : voices[0];
+}