Repository navigation
Expand file tree
/
Copy pathchat-size-limit.ts
More file actions
108 lines (97 loc) · 5.14 KB
/
Copy pathchat-size-limit.ts
File metadata and controls
108 lines (97 loc) · 5.14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
import { prefixBytes } from "../context-rebuild.ts";
import type { ChatContentPart } from "./chat-images.ts";
import type { OpenAIChatMessage } from "./openai-chat.ts";
import { object } from "./openai-chat-protocol.ts";
/** nginx's "413 Request Entity Too Large" page, Express's "request entity too large", "Payload Too Large", and
* OpenAI's "Request too large for <model>". */
const TOO_LARGE = /\b(?:request entity|payload|request) too large\b/i;
export function refusesRequestSize(status: number, body: string): boolean {
if (status === 413 || TOO_LARGE.test(body)) return true;
if (status !== 400) return false;
try {
const error = object(object(JSON.parse(body))?.error);
return error?.code === "context_length_exceeded" || error?.type === "context_length_exceeded";
} catch {
return false;
}
}
export class RequestTooLargeError extends Error {
readonly bytes: number;
constructor(message: string, bytes: number) {
super(message);
this.bytes = bytes;
}
}
export const IMAGE_OMITTED = "[image omitted to fit the provider's size limit]";
export const TOOL_OUTPUT_SHORTENED = "\n[tool output shortened to fit the provider's size limit]";
const omittedHistory = (count: number) =>
`[${count} earlier message${count === 1 ? "" : "s"} omitted to fit the provider's size limit; full history remains in the app.]`;
const LONG_TOOL_OUTPUT_BYTES = 4_000;
const SHORT_TOOL_OUTPUT_BYTES = 1_000;
export interface RequestSize {
bytes: number;
messages: number;
images: number;
}
const bytesOf = (value: unknown) => Buffer.byteLength(JSON.stringify(value));
const isImage = (part: ChatContentPart) => part.type === "image_url";
const imagesIn = (message: OpenAIChatMessage) => Array.isArray(message.content) ? message.content.filter(isImage).length : 0;
export function requestSize(messages: readonly OpenAIChatMessage[], overheadBytes = 0): RequestSize {
return {
bytes: bytesOf(messages) + overheadBytes,
messages: messages.length,
images: messages.reduce((total, message) => total + imagesIn(message), 0),
};
}
function withoutImages(message: OpenAIChatMessage): OpenAIChatMessage {
if (!Array.isArray(message.content)) return message;
return { ...message, content: message.content.map((part) => isImage(part) ? { type: "text" as const, text: IMAGE_OMITTED } : part) };
}
function shortenedToolOutput(message: OpenAIChatMessage): OpenAIChatMessage {
const content = typeof message.content === "string" ? message.content : "";
const shorten = (text: string) => prefixBytes(text, SHORT_TOOL_OUTPUT_BYTES) + TOOL_OUTPUT_SHORTENED;
let envelope: Record<string, unknown> | undefined;
try { envelope = object(JSON.parse(content)); } catch { envelope = undefined; }
return { ...message, content: typeof envelope?.result === "string" ? JSON.stringify({ ...envelope, result: shorten(envelope.result) }) : shorten(content) };
}
/** Aims at half of `refusedBytes`; null when nothing but the system prompt and `request` could shrink. */
export function smallerResend(
messages: readonly OpenAIChatMessage[],
request: OpenAIChatMessage,
refusedBytes: number,
): { messages: OpenAIChatMessage[]; before: RequestSize; after: RequestSize } | null {
const requestAt = messages.indexOf(request);
if (requestAt < 0) throw new Error("the turn's request is not among the messages to resend");
const before = requestSize(messages);
const overhead = Math.max(0, refusedBytes - before.bytes);
const target = Math.floor(refusedBytes / 2) - overhead;
const out: Array<OpenAIChatMessage | null> = [...messages];
const sizes = out.map(bytesOf);
let total = sizes.reduce((sum, size) => sum + size, 0);
const replace = (index: number, next: OpenAIChatMessage | null) => {
const size = next ? bytesOf(next) : 0;
total += size - sizes[index]!;
sizes[index] = size;
out[index] = next;
};
const shrinkable = messages.flatMap((message, index) => index === requestAt || message.role === "system" ? [] : [index]);
const steps = [
...shrinkable.filter((index) => imagesIn(messages[index]!)).map((index) => () => replace(index, withoutImages(out[index]!))),
...shrinkable.filter((index) => index > requestAt && messages[index]!.role === "tool" && sizes[index]! > LONG_TOOL_OUTPUT_BYTES)
.map((index) => () => replace(index, shortenedToolOutput(out[index]!))),
...shrinkable.filter((index) => index < requestAt).map((index) => () => replace(index, null)),
];
let taken = 0;
for (; taken < steps.length && total > target; taken++) steps[taken]!();
if (!taken) return null;
const dropped = out.slice(0, requestAt).filter((message) => message === null).length;
const firstDropped = out.indexOf(null);
const kept = out.flatMap((message, index) => [
...(dropped && index === firstDropped ? [{ role: "assistant" as const, content: omittedHistory(dropped) }] : []),
...(message ? [message] : []),
]);
return { messages: kept, before: { ...before, bytes: refusedBytes }, after: requestSize(kept, overhead) };
}
export function sizeChange(before: RequestSize, after: RequestSize): string {
return `${before.bytes} → ${after.bytes} bytes, ${before.messages} → ${after.messages} messages, ${before.images} → ${after.images} images`;
}