Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/graphs/Graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -831,6 +831,13 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
if (bedrockOptions?.promptCache === true) {
finalMessages = addBedrockCacheControl<BaseMessage>(finalMessages);
}
} else if (agentContext.provider === Providers.OPENROUTER) {
const openRouterOptions = agentContext.clientOptions as
| t.ProviderOptionsMap[Providers.OPENROUTER]
| undefined;
if (openRouterOptions?.promptCache === true) {
finalMessages = addCacheControl<BaseMessage>(finalMessages);
}
}

if (
Expand Down
15 changes: 11 additions & 4 deletions src/llm/openrouter/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ export interface ChatOpenRouterCallOptions
include_reasoning?: boolean;
reasoning?: OpenRouterReasoning;
modelKwargs?: OpenAIChatInput['modelKwargs'];
promptCache?: boolean;
}

/** invocationParams return type extended with OpenRouter reasoning */
Expand Down Expand Up @@ -352,12 +353,18 @@ export class ChatOpenRouter extends ChatOpenAI {
);
}
if (usage) {
const promptDetails = usage.prompt_tokens_details as
| (typeof usage.prompt_tokens_details & { cache_write_tokens?: number })
| undefined;
const inputTokenDetails = {
...(usage.prompt_tokens_details?.audio_tokens != null && {
audio: usage.prompt_tokens_details.audio_tokens,
...(promptDetails?.audio_tokens != null && {
audio: promptDetails.audio_tokens,
}),
...(promptDetails?.cached_tokens != null && {
cache_read: promptDetails.cached_tokens,
}),
...(usage.prompt_tokens_details?.cached_tokens != null && {
cache_read: usage.prompt_tokens_details.cached_tokens,
...(promptDetails?.cache_write_tokens != null && {
cache_creation: promptDetails.cache_write_tokens,
}),
};
const outputTokenDetails = {
Expand Down
129 changes: 129 additions & 0 deletions src/messages/cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
addBedrockCacheControl,
addCacheControl,
} from './cache';
import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
import { ContentTypes } from '@/common/enum';

describe('addCacheControl', () => {
Expand Down Expand Up @@ -1381,3 +1382,131 @@ describe('LangChain message type preservation', () => {
expect((result[1] as AIMessage).tool_calls![0].name).toBe('navigate');
});
});

describe('OpenRouter prompt caching (reuses addCacheControl)', () => {
it('adds cache_control to LangChain messages for OpenRouter (same format as Anthropic)', () => {
const messages: BaseMessage[] = [
new HumanMessage({ content: [{ type: 'text', text: 'System context' }] }),
new AIMessage({ content: [{ type: 'text', text: 'Acknowledged' }] }),
new HumanMessage({ content: [{ type: 'text', text: 'User query' }] }),
];

const result = addCacheControl(messages);

const firstContent = result[0].content as MessageContentComplex[];
const lastContent = result[2].content as MessageContentComplex[];

expect((firstContent[0] as Record<string, unknown>).cache_control).toEqual({
type: 'ephemeral',
});
expect((lastContent[0] as Record<string, unknown>).cache_control).toEqual({
type: 'ephemeral',
});
});

it('preserves cache_control through OpenAI message conversion used by OpenRouter', () => {
const messages: BaseMessage[] = [
new HumanMessage({
content: [
{
type: 'text',
text: 'Hello',
cache_control: { type: 'ephemeral' },
},
],
}),
new AIMessage({ content: 'Hi there' }),
new HumanMessage({
content: [
{
type: 'text',
text: 'Follow-up',
cache_control: { type: 'ephemeral' },
},
],
}),
];

const converted = _convertMessagesToOpenAIParams(messages);

const firstUserContent = converted[0].content as unknown as Record<
string,
unknown
>[];
const lastUserContent = converted[2].content as unknown as Record<
string,
unknown
>[];

expect(firstUserContent[0]).toHaveProperty('cache_control');
expect(firstUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
expect(lastUserContent[0]).toHaveProperty('cache_control');
expect(lastUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
});

it('end-to-end: addCacheControl then convert preserves breakpoints for OpenRouter', () => {
const messages: BaseMessage[] = [
new HumanMessage({ content: 'First message with context' }),
new AIMessage({ content: 'Response' }),
new HumanMessage({ content: 'Second question' }),
];

const cached = addCacheControl(messages);
const converted = _convertMessagesToOpenAIParams(
cached,
'anthropic/claude-sonnet-4-20250514'
);

const firstUser = converted[0];
const lastUser = converted[2];

expect(Array.isArray(firstUser.content)).toBe(true);
expect(
(firstUser.content as unknown as Record<string, unknown>[])[0]
).toHaveProperty('cache_control');

expect(Array.isArray(lastUser.content)).toBe(true);
expect(
(lastUser.content as unknown as Record<string, unknown>[])[0]
).toHaveProperty('cache_control');
});

it('strips Bedrock cache before applying OpenRouter/Anthropic cache', () => {
const messages: TestMsg[] = [
{
role: 'user',
content: [
{ type: ContentTypes.TEXT, text: 'First message' },
{ cachePoint: { type: 'default' } },
],
},
{
role: 'assistant',
content: [
{ type: ContentTypes.TEXT, text: 'Response' },
{ cachePoint: { type: 'default' } },
],
},
{
role: 'user',
content: [{ type: ContentTypes.TEXT, text: 'Follow-up' }],
},
];

/** @ts-expect-error - Testing cross-provider compatibility */
const result = addCacheControl(messages);

for (const msg of result) {
if (Array.isArray(msg.content)) {
expect(
(msg.content as MessageContentComplex[]).some(
(b) => 'cachePoint' in b
)
).toBe(false);
}
}

const lastContent = result[2].content as MessageContentComplex[];
expect('cache_control' in lastContent[0]).toBe(true);
});
});
Loading