From 81128424abfcd7ea3f7a179c1edc75f8de31a1a7 Mon Sep 17 00:00:00 2001 From: Gianna Gerton Date: Thu, 20 Aug 2026 16:57:02 -0400 Subject: [PATCH 1/2] fix missing first message --- .changeset/widget-text-mode-first-message.md | 23 ++++++ packages/convai-widget-core/src/index.test.ts | 82 +++++++++++++++++++ .../convai-widget-core/src/mocks/browser.ts | 46 ++++++++++- .../convai-widget-core/src/widget/Sheet.tsx | 38 +++++++-- 4 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 .changeset/widget-text-mode-first-message.md diff --git a/.changeset/widget-text-mode-first-message.md b/.changeset/widget-text-mode-first-message.md new file mode 100644 index 00000000..41fce7ce --- /dev/null +++ b/.changeset/widget-text-mode-first-message.md @@ -0,0 +1,23 @@ +--- +"@elevenlabs/convai-widget-core": patch +--- + +Show the agent's first message in text mode for agents that support both text and +voice, not just text-only ones. Text mode drops the server-sent first message — +it belongs to a turn the user's opening message immediately interrupts — and the +local re-render that compensates was gated to text-only widgets, so Text & Voice +agents showed no welcome message at all, including via `override-first-message`, +which feeds the same suppressed path. It now renders for any agent that supports +text mode with the text input enabled, as a preview before connecting and through +a text conversation; voice conversations still use the server-sent copy, so there +is never a duplicate. Because a voice-capable agent writes `first_message` for +TTS, the local copy honours `strip_audio_tags` the way voice bubbles do, so tags +like `[happy]` don't leak into the text bubble. + +First-message rich content follows the same rule, so a greeting that offers +buttons no longer renders without them before the conversation starts. + +Side effect: the open-but-disconnected sheet now has a non-empty transcript, so +the orb shrinks to the corner avatar, the call button moves from the orb into the +action row, and the resize button appears — the same layout previously reached on +the user's first message. diff --git a/packages/convai-widget-core/src/index.test.ts b/packages/convai-widget-core/src/index.test.ts index e5325993..1c9412bf 100644 --- a/packages/convai-widget-core/src/index.test.ts +++ b/packages/convai-widget-core/src/index.test.ts @@ -213,6 +213,74 @@ describe("elevenlabs-convai", () => { } ); + describe("first message for voice-capable agents", () => { + it("shows the first message before a text conversation starts", async () => { + setupWebComponent({ "agent-id": "text_and_voice", variant: "compact" }); + + await expect + .element(page.getByText("Welcome message")) + .toBeInTheDocument(); + }); + + it("shows the first message rich content before a text conversation starts", async () => { + // The buttons belong to the first message, so a greeting offering choices + // must not render without them. + setupWebComponent({ + "agent-id": "text_and_voice_rich_content", + variant: "compact", + }); + + await expect + .element(page.getByText("Welcome message")) + .toBeInTheDocument(); + await expect + .element(page.getByRole("button", { name: "Track my order" })) + .toBeInTheDocument(); + }); + + it("keeps a single first message when the user starts a text chat", async () => { + setupWebComponent({ "agent-id": "text_and_voice", variant: "compact" }); + + const textInput = page.getByRole("textbox", { + name: "Text message input", + }); + await textInput.fill("Text message"); + await userEvent.keyboard("{Enter}"); + + await expect.element(page.getByText("Text message")).toBeInTheDocument(); + await expect + .element(page.getByText("Another agent response")) + .toBeInTheDocument(); + + // The server sends the first message too, but it arrives after the user's + // opening message, so the conversation drops it in favour of the locally + // rendered one rather than showing it twice or out of order. + const welcome = page.getByText("Welcome message"); + await expect.element(welcome).toBeInTheDocument(); + expect(welcome.elements()).toHaveLength(1); + }); + + it("does not duplicate the first message when the user starts a call", async () => { + setupWebComponent({ "agent-id": "text_and_voice", variant: "compact" }); + + // The locally rendered preview has to give way to the server-sent first + // message once the conversation starts in voice mode. + await expect + .element(page.getByText("Welcome message")) + .toBeInTheDocument(); + + await page.getByRole("button", { name: "Start a call" }).click(); + + await expect + .element(page.getByText("Another agent response")) + .toBeInTheDocument(); + + const welcome = page.getByText("Welcome message"); + await expect.element(welcome).toBeInTheDocument(); + expect(welcome.elements()).toHaveLength(1); + }); + }); + it.each(Variants)( "$0 variant should show last message when agent calls end_call", async variant => { @@ -574,6 +642,20 @@ describe("elevenlabs-convai", () => { await expect.element(page.getByText(/\[happy\]/)).toBeInTheDocument(); }); + it("should strip audio tags from the locally rendered first message of a voice-capable agent", async () => { + // first_message is written for TTS, so the local copy has to strip tags + // even though it renders as a text bubble. + setupWebComponent({ + "agent-id": "text_and_voice_audio_tags", + variant: "compact", + }); + + await expect + .element(page.getByText("Hello there! How can I help you today?")) + .toBeInTheDocument(); + await expect.element(page.getByText(/\[happy\]/)).not.toBeInTheDocument(); + }); + it("should not strip audio tags when strip_audio_tags config is false", async () => { setupWebComponent({ "agent-id": "audio_tags_no_strip", diff --git a/packages/convai-widget-core/src/mocks/browser.ts b/packages/convai-widget-core/src/mocks/browser.ts index 89032f89..0547ff7b 100644 --- a/packages/convai-widget-core/src/mocks/browser.ts +++ b/packages/convai-widget-core/src/mocks/browser.ts @@ -272,6 +272,45 @@ const codeBlock = true; default_expanded: true, first_message: "", }, + text_and_voice: { + ...BASIC_CONFIG, + text_only: false, + supports_text_only: true, + transcript_enabled: true, + text_input_enabled: true, + terms_html: undefined, + default_expanded: true, + first_message: "Welcome message", + }, + text_and_voice_rich_content: { + ...BASIC_CONFIG, + text_only: false, + supports_text_only: true, + transcript_enabled: true, + text_input_enabled: true, + terms_html: undefined, + default_expanded: true, + first_message: "Welcome message", + first_message_rich_content: { + component: "buttons", + props: { + buttons: [ + { type: "message", label: "Track my order", message: "Track" }, + ], + }, + }, + }, + text_and_voice_audio_tags: { + ...BASIC_CONFIG, + text_only: false, + supports_text_only: true, + transcript_enabled: true, + text_input_enabled: true, + strip_audio_tags: true, + terms_html: undefined, + default_expanded: true, + first_message: "[happy] Hello there! [excited] How can I help you today?", + }, } as const satisfies Record; function isValidAgentId(agentId: string): agentId is keyof typeof AGENTS { @@ -335,8 +374,11 @@ export const Worker = setupWorker( }, }) ); + // `text_and_voice` is a voice-capable agent that the widget switches to + // text mode when the user types, so it follows the text chat script. + const isTextChat = config.text_only || agentId === "text_and_voice"; if ( - config.text_only && + isTextChat && agentId !== "end_call_test" && agentId !== "tool_call" && agentId !== "stream_consolidation" && @@ -359,7 +401,7 @@ export const Worker = setupWorker( ); await new Promise(resolve => setTimeout(resolve, 1000)); client.close(); - } else if (!config.text_only) { + } else if (!isTextChat) { client.send( JSON.stringify({ type: "user_transcript", diff --git a/packages/convai-widget-core/src/widget/Sheet.tsx b/packages/convai-widget-core/src/widget/Sheet.tsx index 5fc82f6c..f9e1c581 100644 --- a/packages/convai-widget-core/src/widget/Sheet.tsx +++ b/packages/convai-widget-core/src/widget/Sheet.tsx @@ -2,6 +2,7 @@ import { useComputed, useSignal } from "@preact/signals"; import { useFirstMessage, useIsConversationTextOnly, + useTextInputEnabled, useTextOnly, useWidgetConfig, } from "../contexts/widget-config"; @@ -22,6 +23,7 @@ import { useSheetContent } from "../contexts/sheet-content"; import { useWidgetSize } from "../contexts/widget-size"; import { SheetActions } from "./SheetActions"; import { AvatarOverlay } from "./AvatarOverlay"; +import { stripAudioTags } from "../utils/stripAudioTags"; interface SheetProps { open: Signalish; @@ -50,22 +52,44 @@ export function Sheet({ open }: SheetProps) { isExternalAgentMode, } = useConversation(); const firstMessage = useFirstMessage(); + const textInputEnabled = useTextInputEnabled(); const { currentContent, currentConfig } = useSheetContent(); const { variant } = useWidgetSize(); + const localFirstMessage = useComputed(() => { + const raw = firstMessage.value; + if (!raw) return undefined; + + // Voice-capable agents write first_message for TTS, so strip its audio tags + // the way voice bubbles do. Text-only widgets keep them, which the + // `audio_tags_strip` test relies on. + const message = + !textOnly.value && config.value.strip_audio_tags + ? stripAudioTags(raw) + : raw; + + if (isConversationTextOnly.value) return message; + + const showFirstMessage = + isDisconnected.value && + config.value.supports_text_only && + textInputEnabled.value && + !transcript.value.some( + entry => entry.type === "message" && !entry.isText + ); + + return showFirstMessage ? message : undefined; + }); + const filteredTranscript = useComputed(() => { const isTextOnly = textOnly.value || isConversationTextOnly.value; + const localMessage = localFirstMessage.value; return buildDisplayTranscript(transcript.value, { showAgentStatus: config.value.show_agent_status ?? false, transcriptEnabled: isTextOnly || (config.value.transcript_enabled ?? false), - showRichContent: isTextOnly, - // Prepend first message only when the widget is text-only - // (not when it switched to text-only due to user input) - firstMessage: - isTextOnly && textOnly.value && firstMessage.value - ? firstMessage.value - : undefined, + showRichContent: isTextOnly || localMessage !== undefined, + firstMessage: localMessage, firstMessageConversationIndex: conversationIndex.peek(), showTypingIndicator: isExternalAgentMode.value && isAgentTyping.value, }); From 3d8142665524e6d47f00d970c150012ea30afb7b Mon Sep 17 00:00:00 2001 From: Gianna Date: Fri, 21 Aug 2026 10:34:00 -0400 Subject: [PATCH 2/2] Update packages/convai-widget-core/src/widget/Sheet.tsx Co-authored-by: Kamil K --- packages/convai-widget-core/src/widget/Sheet.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/convai-widget-core/src/widget/Sheet.tsx b/packages/convai-widget-core/src/widget/Sheet.tsx index f9e1c581..93c097ac 100644 --- a/packages/convai-widget-core/src/widget/Sheet.tsx +++ b/packages/convai-widget-core/src/widget/Sheet.tsx @@ -74,8 +74,8 @@ export function Sheet({ open }: SheetProps) { isDisconnected.value && config.value.supports_text_only && textInputEnabled.value && - !transcript.value.some( - entry => entry.type === "message" && !entry.isText + transcript.value.every( + entry => entry.type !== "message" || entry.isText ); return showFirstMessage ? message : undefined;