fix(plugin-openai): skip agent_config_update items in realtime chat-ctx sync

.changeset/fix-realtime-agent-config-update.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-openai': patch
+---
+
+Fix `Unsupported item type: agent_config_update` thrown by the OpenAI Realtime plugin when an `Agent` is constructed with tools or instructions. `AgentActivity` inserts internal `agent_config_update` chat items on enter and on tool / instructions changes; the realtime plugin's chat-context sync now filters them out before computing the diff. `agent_handoff` items are also filtered as defense-in-depth, matching the non-realtime path's `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })` (agent_activity.ts:666).

plugins/openai/src/realtime/realtime_model.test.ts

@@ -696,6 +696,99 @@ describe('RealtimeSession.updateOptions', () => {
   });
 });
 
+describe('RealtimeSession.createChatCtxUpdateEvents', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('skips agent_config_update items so the realtime sync does not throw', async () => {
+    stubTaskRuntime();
+
+    const model = new RealtimeModel({ apiKey: 'test-key' });
+    const session = model.session() as unknown as {
+      createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise<api_proto.ClientEvent[]>;
+    };
+
+    const systemMessage = llm.ChatMessage.create({
+      id: 'item_system',
+      role: 'system',
+      content: ['You are a helpful agent.'],
+    });
+    const configUpdate = llm.AgentConfigUpdate.create({
+      id: 'item_config',
+      instructions: 'updated instructions',
+      toolsAdded: ['lookup'],
+    });
+    const userMessage = llm.ChatMessage.create({
+      id: 'item_user',
+      role: 'user',
+      content: ['hello there'],
+    });
+    const chatCtx = llm.ChatContext.empty();
+    chatCtx.items.push(systemMessage, configUpdate, userMessage);
+
+    const events = await session.createChatCtxUpdateEvents(chatCtx);
+
+    const createdIds = events
+      .filter(
+        (e): e is api_proto.ConversationItemCreateEvent => e.type === 'conversation.item.create',
+      )
+      .map((e) => e.item.id);
+
+    expect(createdIds).toEqual(expect.arrayContaining(['item_system', 'item_user']));
+    expect(createdIds).not.toContain('item_config');
+  });
+
+  it('returns no create events for a ChatContext containing only agent_config_update items', async () => {
+    stubTaskRuntime();
+
+    const model = new RealtimeModel({ apiKey: 'test-key' });
+    const session = model.session() as unknown as {
+      createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise<api_proto.ClientEvent[]>;
+    };
+
+    const chatCtx = llm.ChatContext.empty();
+    chatCtx.items.push(
+      llm.AgentConfigUpdate.create({ id: 'item_config_a', toolsAdded: ['a'] }),
+      llm.AgentConfigUpdate.create({ id: 'item_config_b', toolsRemoved: ['b'] }),
+    );
+
+    await expect(session.createChatCtxUpdateEvents(chatCtx)).resolves.toEqual([]);
+  });
+
+  it('also skips agent_handoff items (defense-in-depth, no OpenAI Realtime representation)', async () => {
+    stubTaskRuntime();
+
+    const model = new RealtimeModel({ apiKey: 'test-key' });
+    const session = model.session() as unknown as {
+      createChatCtxUpdateEvents: (chatCtx: llm.ChatContext) => Promise<api_proto.ClientEvent[]>;
+    };
+
+    const userMessage = llm.ChatMessage.create({
+      id: 'item_user',
+      role: 'user',
+      content: ['hand me off please'],
+    });
+    const handoff = llm.AgentHandoffItem.create({
+      id: 'item_handoff',
+      oldAgentId: 'triage',
+      newAgentId: 'billing',
+    });
+    const chatCtx = llm.ChatContext.empty();
+    chatCtx.items.push(userMessage, handoff);
+
+    const events = await session.createChatCtxUpdateEvents(chatCtx);
+    const createdIds = events
+      .filter(
+        (e): e is api_proto.ConversationItemCreateEvent => e.type === 'conversation.item.create',
+      )
+      .map((e) => e.item.id);
+
+    expect(createdIds).toContain('item_user');
+    expect(createdIds).not.toContain('item_handoff');
+  });
+});
+
 describe('processBaseURL', () => {
   it('upgrades https baseURL to wss and appends /realtime with model', () => {
     const url = new URL(

plugins/openai/src/realtime/realtime_model.ts

@@ -664,6 +664,18 @@ export class RealtimeSession extends llm.RealtimeSession {
     addMockAudio: boolean = false,
   ): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> {
     const newChatCtx = chatCtx.copy();
+    // Drop framework-internal items that have no OpenAI Realtime
+    // representation. `livekitItemToOpenAIItem`'s default arm throws on
+    // anything other than `function_call` / `function_call_output` /
+    // `message`. `agent_config_update` is the reachable case today (inserted
+    // by `AgentActivity` on enter / tool / instructions changes — the
+    // original #1855 crash); `agent_handoff` is filtered as defense-in-depth
+    // to match the non-realtime path, which excludes both via
+    // `chatCtx.copy({ excludeHandoff: true, excludeConfigUpdate: true })`
+    // (agent_activity.ts:666).
+    newChatCtx.items = newChatCtx.items.filter(
+      (item) => item.type !== 'agent_config_update' && item.type !== 'agent_handoff',
+    );
     if (addMockAudio) {
       newChatCtx.items.push(createMockAudioItem());
     } else {