diff --git a/.changeset/fix-realtime-agent-config-update.md b/.changeset/fix-realtime-agent-config-update.md new file mode 100644 index 000000000..eb1f21f95 --- /dev/null +++ b/.changeset/fix-realtime-agent-config-update.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-openai': patch +--- + +Fix `Unsupported item type: agent_config_update` thrown by the OpenAI Realtime plugin when an `Agent` is constructed with tools or instructions. `AgentActivity` inserts internal `agent_config_update` chat items on enter and on tool / instructions changes; the realtime plugin's chat-context sync now filters them out before computing the diff. `agent_handoff` items are also filtered as defense-in-depth, matching the non-realtime path's `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })` (agent_activity.ts:666). diff --git a/plugins/openai/src/realtime/realtime_model.test.ts b/plugins/openai/src/realtime/realtime_model.test.ts index 6080cdbd6..969a29302 100644 --- a/plugins/openai/src/realtime/realtime_model.test.ts +++ b/plugins/openai/src/realtime/realtime_model.test.ts @@ -696,6 +696,99 @@ describe('RealtimeSession.updateOptions', () => { }); }); +describe('RealtimeSession.createChatCtxUpdateEvents', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('skips agent_config_update items so the realtime sync does not throw', async () => { + stubTaskRuntime(); + + const model = new RealtimeModel({ apiKey: 'test-key' }); + const session = model.session() as unknown as { + createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise; + }; + + const systemMessage = llm.ChatMessage.create({ + id: 'item_system', + role: 'system', + content: ['You are a helpful agent.'], + }); + const configUpdate = llm.AgentConfigUpdate.create({ + id: 'item_config', + instructions: 'updated instructions', + toolsAdded: ['lookup'], + }); + const userMessage = llm.ChatMessage.create({ + id: 'item_user', + role: 'user', + content: ['hello there'], + }); + const chatCtx = llm.ChatContext.empty(); + chatCtx.items.push(systemMessage, configUpdate, userMessage); + + const events = await session.createChatCtxUpdateEvents(chatCtx); + + const createdIds = events + .filter( + (e): e is api_proto.ConversationItemCreateEvent => e.type === 'conversation.item.create', + ) + .map((e) => e.item.id); + + expect(createdIds).toEqual(expect.arrayContaining(['item_system', 'item_user'])); + expect(createdIds).not.toContain('item_config'); + }); + + it('returns no create events for a ChatContext containing only agent_config_update items', async () => { + stubTaskRuntime(); + + const model = new RealtimeModel({ apiKey: 'test-key' }); + const session = model.session() as unknown as { + createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise; + }; + + const chatCtx = llm.ChatContext.empty(); + chatCtx.items.push( + llm.AgentConfigUpdate.create({ id: 'item_config_a', toolsAdded: ['a'] }), + llm.AgentConfigUpdate.create({ id: 'item_config_b', toolsRemoved: ['b'] }), + ); + + await expect(session.createChatCtxUpdateEvents(chatCtx)).resolves.toEqual([]); + }); + + it('also skips agent_handoff items (defense-in-depth, no OpenAI Realtime representation)', async () => { + stubTaskRuntime(); + + const model = new RealtimeModel({ apiKey: 'test-key' }); + const session = model.session() as unknown as { + createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise; + }; + + const userMessage = llm.ChatMessage.create({ + id: 'item_user', + role: 'user', + content: ['hand me off please'], + }); + const handoff = llm.AgentHandoffItem.create({ + id: 'item_handoff', + oldAgentId: 'triage', + newAgentId: 'billing', + }); + const chatCtx = llm.ChatContext.empty(); + chatCtx.items.push(userMessage, handoff); + + const events = await session.createChatCtxUpdateEvents(chatCtx); + const createdIds = events + .filter( + (e): e is api_proto.ConversationItemCreateEvent => e.type === 'conversation.item.create', + ) + .map((e) => e.item.id); + + expect(createdIds).toContain('item_user'); + expect(createdIds).not.toContain('item_handoff'); + }); +}); + describe('processBaseURL', () => { it('upgrades https baseURL to wss and appends /realtime with model', () => { const url = new URL( diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index e8f88035d..c469ab66c 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -664,6 +664,18 @@ export class RealtimeSession extends llm.RealtimeSession { addMockAudio: boolean = false, ): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> { const newChatCtx = chatCtx.copy(); + // Drop framework-internal items that have no OpenAI Realtime + // representation. `livekitItemToOpenAIItem`'s default arm throws on + // anything other than `function_call` / `function_call_output` / + // `message`. `agent_config_update` is the reachable case today (inserted + // by `AgentActivity` on enter / tool / instructions changes — the + // original #1855 crash); `agent_handoff` is filtered as defense-in-depth + // to match the non-realtime path, which excludes both via + // `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })` + // (agent_activity.ts:666). + newChatCtx.items = newChatCtx.items.filter( + (item) => item.type !== 'agent_config_update' && item.type !== 'agent_handoff', + ); if (addMockAudio) { newChatCtx.items.push(createMockAudioItem()); } else {