[Widget] Show first message in text mode for Text & Voice agents

.changeset/widget-text-mode-first-message.md

@@ -0,0 +1,23 @@
+---
+"@elevenlabs/convai-widget-core": patch
+---
+
+Show the agent's first message in text mode for agents that support both text and
+voice, not just text-only ones. Text mode drops the server-sent first message —
+it belongs to a turn the user's opening message immediately interrupts — and the
+local re-render that compensates was gated to text-only widgets, so Text & Voice
+agents showed no welcome message at all, including via `override-first-message`,
+which feeds the same suppressed path. It now renders for any agent that supports
+text mode with the text input enabled, as a preview before connecting and through
+a text conversation; voice conversations still use the server-sent copy, so there
+is never a duplicate. Because a voice-capable agent writes `first_message` for
+TTS, the local copy honours `strip_audio_tags` the way voice bubbles do, so tags
+like `[happy]` don't leak into the text bubble.
+
+First-message rich content follows the same rule, so a greeting that offers
+buttons no longer renders without them before the conversation starts.
+
+Side effect: the open-but-disconnected sheet now has a non-empty transcript, so
+the orb shrinks to the corner avatar, the call button moves from the orb into the
+action row, and the resize button appears — the same layout previously reached on
+the user's first message.

packages/convai-widget-core/src/index.test.ts

@@ -213,6 +213,74 @@ describe("elevenlabs-convai", () => {
     }
   );
 
+  describe("first message for voice-capable agents", () => {
+    it("shows the first message before a text conversation starts", async () => {
+      setupWebComponent({ "agent-id": "text_and_voice", variant: "compact" });
+
+      await expect
+        .element(page.getByText("Welcome message"))
+        .toBeInTheDocument();
+    });
+
+    it("shows the first message rich content before a text conversation starts", async () => {
+      // The buttons belong to the first message, so a greeting offering choices
+      // must not render without them.
+      setupWebComponent({
+        "agent-id": "text_and_voice_rich_content",
+        variant: "compact",
+      });
+
+      await expect
+        .element(page.getByText("Welcome message"))
+        .toBeInTheDocument();
+      await expect
+        .element(page.getByRole("button", { name: "Track my order" }))
+        .toBeInTheDocument();
+    });
+
+    it("keeps a single first message when the user starts a text chat", async () => {
+      setupWebComponent({ "agent-id": "text_and_voice", variant: "compact" });
+
+      const textInput = page.getByRole("textbox", {
+        name: "Text message input",
+      });
+      await textInput.fill("Text message");
+      await userEvent.keyboard("{Enter}");
+
+      await expect.element(page.getByText("Text message")).toBeInTheDocument();
+      await expect
+        .element(page.getByText("Another agent response"))
+        .toBeInTheDocument();
+
+      // The server sends the first message too, but it arrives after the user's
+      // opening message, so the conversation drops it in favour of the locally
+      // rendered one rather than showing it twice or out of order.
+      const welcome = page.getByText("Welcome message");
+      await expect.element(welcome).toBeInTheDocument();
+      expect(welcome.elements()).toHaveLength(1);
+    });
+
+    it("does not duplicate the first message when the user starts a call", async () => {
+      setupWebComponent({ "agent-id": "text_and_voice", variant: "compact" });
+
+      // The locally rendered preview has to give way to the server-sent first
+      // message once the conversation starts in voice mode.
+      await expect
+        .element(page.getByText("Welcome message"))
+        .toBeInTheDocument();
+
+      await page.getByRole("button", { name: "Start a call" }).click();
+
+      await expect
+        .element(page.getByText("Another agent response"))
+        .toBeInTheDocument();
+
+      const welcome = page.getByText("Welcome message");
+      await expect.element(welcome).toBeInTheDocument();
+      expect(welcome.elements()).toHaveLength(1);
+    });
+  });
+
   it.each(Variants)(
     "$0 variant should show last message when agent calls end_call",
     async variant => {
@@ -574,6 +642,20 @@ describe("elevenlabs-convai", () => {
       await expect.element(page.getByText(/\[happy\]/)).toBeInTheDocument();
     });
 
+    it("should strip audio tags from the locally rendered first message of a voice-capable agent", async () => {
+      // first_message is written for TTS, so the local copy has to strip tags
+      // even though it renders as a text bubble.
+      setupWebComponent({
+        "agent-id": "text_and_voice_audio_tags",
+        variant: "compact",
+      });
+
+      await expect
+        .element(page.getByText("Hello there! How can I help you today?"))
+        .toBeInTheDocument();
+      await expect.element(page.getByText(/\[happy\]/)).not.toBeInTheDocument();
+    });
+
     it("should not strip audio tags when strip_audio_tags config is false", async () => {
       setupWebComponent({
         "agent-id": "audio_tags_no_strip",

packages/convai-widget-core/src/mocks/browser.ts

@@ -272,6 +272,45 @@ const codeBlock = true;
     default_expanded: true,
     first_message: "",
   },
+  text_and_voice: {
+    ...BASIC_CONFIG,
+    text_only: false,
+    supports_text_only: true,
+    transcript_enabled: true,
+    text_input_enabled: true,
+    terms_html: undefined,
+    default_expanded: true,
+    first_message: "Welcome message",
+  },
+  text_and_voice_rich_content: {
+    ...BASIC_CONFIG,
+    text_only: false,
+    supports_text_only: true,
+    transcript_enabled: true,
+    text_input_enabled: true,
+    terms_html: undefined,
+    default_expanded: true,
+    first_message: "Welcome message",
+    first_message_rich_content: {
+      component: "buttons",
+      props: {
+        buttons: [
+          { type: "message", label: "Track my order", message: "Track" },
+        ],
+      },
+    },
+  },
+  text_and_voice_audio_tags: {
+    ...BASIC_CONFIG,
+    text_only: false,
+    supports_text_only: true,
+    transcript_enabled: true,
+    text_input_enabled: true,
+    strip_audio_tags: true,
+    terms_html: undefined,
+    default_expanded: true,
+    first_message: "[happy] Hello there! [excited] How can I help you today?",
+  },
 } as const satisfies Record<string, WidgetConfig>;
 
 function isValidAgentId(agentId: string): agentId is keyof typeof AGENTS {
@@ -335,8 +374,11 @@ export const Worker = setupWorker(
           },
         })
       );
+      // `text_and_voice` is a voice-capable agent that the widget switches to
+      // text mode when the user types, so it follows the text chat script.
+      const isTextChat = config.text_only || agentId === "text_and_voice";
       if (
-        config.text_only &&
+        isTextChat &&
         agentId !== "end_call_test" &&
         agentId !== "tool_call" &&
         agentId !== "stream_consolidation" &&
@@ -359,7 +401,7 @@ export const Worker = setupWorker(
         );
         await new Promise(resolve => setTimeout(resolve, 1000));
         client.close();
-      } else if (!config.text_only) {
+      } else if (!isTextChat) {
         client.send(
           JSON.stringify({
             type: "user_transcript",

packages/convai-widget-core/src/widget/Sheet.tsx

@@ -2,6 +2,7 @@ import { useComputed, useSignal } from "@preact/signals";
 import {
   useFirstMessage,
   useIsConversationTextOnly,
+  useTextInputEnabled,
   useTextOnly,
   useWidgetConfig,
 } from "../contexts/widget-config";
@@ -22,6 +23,7 @@ import { useSheetContent } from "../contexts/sheet-content";
 import { useWidgetSize } from "../contexts/widget-size";
 import { SheetActions } from "./SheetActions";
 import { AvatarOverlay } from "./AvatarOverlay";
+import { stripAudioTags } from "../utils/stripAudioTags";
 
 interface SheetProps {
   open: Signalish<boolean>;
@@ -50,22 +52,44 @@ export function Sheet({ open }: SheetProps) {
     isExternalAgentMode,
   } = useConversation();
   const firstMessage = useFirstMessage();
+  const textInputEnabled = useTextInputEnabled();
   const { currentContent, currentConfig } = useSheetContent();
   const { variant } = useWidgetSize();
 
+  const localFirstMessage = useComputed(() => {
+    const raw = firstMessage.value;
+    if (!raw) return undefined;
+
+    // Voice-capable agents write first_message for TTS, so strip its audio tags
+    // the way voice bubbles do. Text-only widgets keep them, which the
+    // `audio_tags_strip` test relies on.
+    const message =
+      !textOnly.value && config.value.strip_audio_tags
+        ? stripAudioTags(raw)
+        : raw;
+
+    if (isConversationTextOnly.value) return message;
+
+    const showFirstMessage =
+      isDisconnected.value &&
+      config.value.supports_text_only &&
+      textInputEnabled.value &&
+      transcript.value.every(
+        entry => entry.type !== "message" || entry.isText
+      );
+
+    return showFirstMessage ? message : undefined;
+  });
+
   const filteredTranscript = useComputed<DisplayTranscriptEntry[]>(() => {
     const isTextOnly = textOnly.value || isConversationTextOnly.value;
+    const localMessage = localFirstMessage.value;
     return buildDisplayTranscript(transcript.value, {
       showAgentStatus: config.value.show_agent_status ?? false,
       transcriptEnabled:
         isTextOnly || (config.value.transcript_enabled ?? false),
-      showRichContent: isTextOnly,
-      // Prepend first message only when the widget is text-only
-      // (not when it switched to text-only due to user input)
-      firstMessage:
-        isTextOnly && textOnly.value && firstMessage.value
-          ? firstMessage.value
-          : undefined,
+      showRichContent: isTextOnly || localMessage !== undefined,
+      firstMessage: localMessage,
       firstMessageConversationIndex: conversationIndex.peek(),
       showTypingIndicator: isExternalAgentMode.value && isAgentTyping.value,
     });