From 75cc842b866f9dc85273667d2d32a9acaac88002 Mon Sep 17 00:00:00 2001 From: tsushanth <78000697+tsushanth@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:12:40 -0700 Subject: [PATCH 1/2] fix(plugin-openai): skip agent_config_update items in realtime chat-ctx sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AgentActivity` inserts `agent_config_update` chat items on enter and on tool / instructions changes. The OpenAI Realtime plugin's chat-context sync (`createChatCtxUpdateEvents`) fed these directly into `livekitItemToOpenAIItem`, which has no case for `agent_config_update` and throws `Unsupported item type: agent_config_update`. The throw is caught upstream — non-fatal — but the never-played-message sync bails mid-way, leaving degraded post-tool speech behavior and missing transcript items. Filter `agent_config_update` items out of `newChatCtx.items` before the diff. Mirrors the non-realtime path's `chatCtx.copy({ excludeConfigUpdate: true })` (agent_activity.ts:666) and the wire filter in `remote_session.ts` (`item.type !== 'agent_config_update'`). Two regression tests added in `realtime_model.test.ts`: - Mixed ChatContext (system + agent_config_update + user) emits create events only for system and user. - ChatContext containing only `agent_config_update` items returns no events and no longer throws. Fixes #1855. --- .../fix-realtime-agent-config-update.md | 5 ++ .../src/realtime/realtime_model.test.ts | 61 +++++++++++++++++++ plugins/openai/src/realtime/realtime_model.ts | 6 ++ 3 files changed, 72 insertions(+) create mode 100644 .changeset/fix-realtime-agent-config-update.md diff --git a/.changeset/fix-realtime-agent-config-update.md b/.changeset/fix-realtime-agent-config-update.md new file mode 100644 index 000000000..0bf6e929b --- /dev/null +++ b/.changeset/fix-realtime-agent-config-update.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-openai': patch +--- + +Fix `Unsupported item type: agent_config_update` thrown by the OpenAI Realtime plugin when an `Agent` is constructed with tools or instructions. `AgentActivity` inserts internal `agent_config_update` chat items on enter and on tool / instructions changes; the realtime plugin's chat-context sync now filters them out before computing the diff, matching the non-realtime path's `excludeConfigUpdate` behavior and the existing wire filter in `remote_session.ts`. diff --git a/plugins/openai/src/realtime/realtime_model.test.ts b/plugins/openai/src/realtime/realtime_model.test.ts index 6080cdbd6..0068cedb1 100644 --- a/plugins/openai/src/realtime/realtime_model.test.ts +++ b/plugins/openai/src/realtime/realtime_model.test.ts @@ -696,6 +696,67 @@ describe('RealtimeSession.updateOptions', () => { }); }); +describe('RealtimeSession.createChatCtxUpdateEvents', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('skips agent_config_update items so the realtime sync does not throw', async () => { + stubTaskRuntime(); + + const model = new RealtimeModel({ apiKey: 'test-key' }); + const session = model.session() as unknown as { + createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise; + }; + + const systemMessage = llm.ChatMessage.create({ + id: 'item_system', + role: 'system', + content: ['You are a helpful agent.'], + }); + const configUpdate = llm.AgentConfigUpdate.create({ + id: 'item_config', + instructions: 'updated instructions', + toolsAdded: ['lookup'], + }); + const userMessage = llm.ChatMessage.create({ + id: 'item_user', + role: 'user', + content: ['hello there'], + }); + const chatCtx = llm.ChatContext.empty(); + chatCtx.items.push(systemMessage, configUpdate, userMessage); + + const events = await session.createChatCtxUpdateEvents(chatCtx); + + const createdIds = events + .filter( + (e): e is api_proto.ConversationItemCreateEvent => e.type === 'conversation.item.create', + ) + .map((e) => e.item.id); + + expect(createdIds).toEqual(expect.arrayContaining(['item_system', 'item_user'])); + expect(createdIds).not.toContain('item_config'); + }); + + it('returns no create events for a ChatContext containing only agent_config_update items', async () => { + stubTaskRuntime(); + + const model = new RealtimeModel({ apiKey: 'test-key' }); + const session = model.session() as unknown as { + createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise; + }; + + const chatCtx = llm.ChatContext.empty(); + chatCtx.items.push( + llm.AgentConfigUpdate.create({ id: 'item_config_a', toolsAdded: ['a'] }), + llm.AgentConfigUpdate.create({ id: 'item_config_b', toolsRemoved: ['b'] }), + ); + + await expect(session.createChatCtxUpdateEvents(chatCtx)).resolves.toEqual([]); + }); +}); + describe('processBaseURL', () => { it('upgrades https baseURL to wss and appends /realtime with model', () => { const url = new URL( diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index e8f88035d..cb974a0b6 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -664,6 +664,12 @@ export class RealtimeSession extends llm.RealtimeSession { addMockAudio: boolean = false, ): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> { const newChatCtx = chatCtx.copy(); + // Drop `agent_config_update` items — they're framework-internal (inserted + // by `AgentActivity` on enter / tool / instructions changes) and have no + // OpenAI Realtime representation; `livekitItemToOpenAIItem` throws on + // them. Mirrors the non-realtime path's `excludeConfigUpdate: true` and + // the wire filter in `remote_session.ts`. + newChatCtx.items = newChatCtx.items.filter((item) => item.type !== 'agent_config_update'); if (addMockAudio) { newChatCtx.items.push(createMockAudioItem()); } else { From 9957723f84421d5af3ebbaf6f92a6e64589a74e0 Mon Sep 17 00:00:00 2001 From: tsushanth <78000697+tsushanth@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:19:51 -0700 Subject: [PATCH 2/2] fix(plugin-openai): also filter agent_handoff in realtime chat-ctx sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Devin Review on #1856 flagged that `livekitItemToOpenAIItem`'s default arm also throws for `agent_handoff` items — which are part of the `ChatItem` union but not handled in the realtime conversion. The non-realtime path already excludes both via `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })` (agent_activity.ts:666). Filter `agent_handoff` alongside `agent_config_update` in the realtime sync as defense-in-depth so the symptom doesn't re-emerge if a future code path ever surfaces handoff items to the realtime session. Added a regression test that pushes an `AgentHandoffItem` next to a user message and asserts the user item still gets a create event while the handoff id is absent. --- .../fix-realtime-agent-config-update.md | 2 +- .../src/realtime/realtime_model.test.ts | 32 +++++++++++++++++++ plugins/openai/src/realtime/realtime_model.ts | 18 +++++++---- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.changeset/fix-realtime-agent-config-update.md b/.changeset/fix-realtime-agent-config-update.md index 0bf6e929b..eb1f21f95 100644 --- a/.changeset/fix-realtime-agent-config-update.md +++ b/.changeset/fix-realtime-agent-config-update.md @@ -2,4 +2,4 @@ '@livekit/agents-plugin-openai': patch --- -Fix `Unsupported item type: agent_config_update` thrown by the OpenAI Realtime plugin when an `Agent` is constructed with tools or instructions. `AgentActivity` inserts internal `agent_config_update` chat items on enter and on tool / instructions changes; the realtime plugin's chat-context sync now filters them out before computing the diff, matching the non-realtime path's `excludeConfigUpdate` behavior and the existing wire filter in `remote_session.ts`. +Fix `Unsupported item type: agent_config_update` thrown by the OpenAI Realtime plugin when an `Agent` is constructed with tools or instructions. `AgentActivity` inserts internal `agent_config_update` chat items on enter and on tool / instructions changes; the realtime plugin's chat-context sync now filters them out before computing the diff. `agent_handoff` items are also filtered as defense-in-depth, matching the non-realtime path's `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })` (agent_activity.ts:666). diff --git a/plugins/openai/src/realtime/realtime_model.test.ts b/plugins/openai/src/realtime/realtime_model.test.ts index 0068cedb1..969a29302 100644 --- a/plugins/openai/src/realtime/realtime_model.test.ts +++ b/plugins/openai/src/realtime/realtime_model.test.ts @@ -755,6 +755,38 @@ describe('RealtimeSession.createChatCtxUpdateEvents', () => { await expect(session.createChatCtxUpdateEvents(chatCtx)).resolves.toEqual([]); }); + + it('also skips agent_handoff items (defense-in-depth, no OpenAI Realtime representation)', async () => { + stubTaskRuntime(); + + const model = new RealtimeModel({ apiKey: 'test-key' }); + const session = model.session() as unknown as { + createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise; + }; + + const userMessage = llm.ChatMessage.create({ + id: 'item_user', + role: 'user', + content: ['hand me off please'], + }); + const handoff = llm.AgentHandoffItem.create({ + id: 'item_handoff', + oldAgentId: 'triage', + newAgentId: 'billing', + }); + const chatCtx = llm.ChatContext.empty(); + chatCtx.items.push(userMessage, handoff); + + const events = await session.createChatCtxUpdateEvents(chatCtx); + const createdIds = events + .filter( + (e): e is api_proto.ConversationItemCreateEvent => e.type === 'conversation.item.create', + ) + .map((e) => e.item.id); + + expect(createdIds).toContain('item_user'); + expect(createdIds).not.toContain('item_handoff'); + }); }); describe('processBaseURL', () => { diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index cb974a0b6..c469ab66c 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -664,12 +664,18 @@ export class RealtimeSession extends llm.RealtimeSession { addMockAudio: boolean = false, ): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> { const newChatCtx = chatCtx.copy(); - // Drop `agent_config_update` items — they're framework-internal (inserted - // by `AgentActivity` on enter / tool / instructions changes) and have no - // OpenAI Realtime representation; `livekitItemToOpenAIItem` throws on - // them. Mirrors the non-realtime path's `excludeConfigUpdate: true` and - // the wire filter in `remote_session.ts`. - newChatCtx.items = newChatCtx.items.filter((item) => item.type !== 'agent_config_update'); + // Drop framework-internal items that have no OpenAI Realtime + // representation. `livekitItemToOpenAIItem`'s default arm throws on + // anything other than `function_call` / `function_call_output` / + // `message`. `agent_config_update` is the reachable case today (inserted + // by `AgentActivity` on enter / tool / instructions changes — the + // original #1855 crash); `agent_handoff` is filtered as defense-in-depth + // to match the non-realtime path, which excludes both via + // `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })` + // (agent_activity.ts:666). + newChatCtx.items = newChatCtx.items.filter( + (item) => item.type !== 'agent_config_update' && item.type !== 'agent_handoff', + ); if (addMockAudio) { newChatCtx.items.push(createMockAudioItem()); } else {