fix: reranked websearch content don't overload the model anymore

src/tools/search/highlights.ts

@@ -240,7 +240,8 @@ export function expandHighlights(
         !result.highlights ||
         result.highlights.length === 0
       ) {
-        return result; // No modification needed
+        const { content: _content, ...resultWithoutContent } = result;
+        return resultWithoutContent as typeof result;
       }
 
       // Create a shallow copy with expanded highlights

src/tools/search/search.ts

@@ -65,6 +65,12 @@ const chunker = {
   },
 };
 
+/** Maximum chars of scraped content passed to the chunker/reranker per source.
+ *  Overridable via SEARCH_MAX_CONTENT_LENGTH env var. */
+const DEFAULT_MAX_CONTENT_LENGTH = 50000;
+const MAX_CONTENT_LENGTH =
+  Number(process.env.SEARCH_MAX_CONTENT_LENGTH) || DEFAULT_MAX_CONTENT_LENGTH;
+
 function createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {
   return (link: string, update?: Partial<t.ValidSource>): void => {
     const source = sourceMap.get(link);
@@ -82,12 +88,14 @@ const getHighlights = async ({
   content,
   reranker,
   topResults = 5,
+  maxContentLength = MAX_CONTENT_LENGTH,
   logger,
 }: {
   content: string;
   query: string;
   reranker?: BaseReranker;
   topResults?: number;
+  maxContentLength?: number;
   logger?: t.Logger;
 }): Promise<t.Highlight[] | undefined> => {
   const logger_ = logger || createDefaultLogger();
@@ -102,7 +110,11 @@ const getHighlights = async ({
   }
 
   try {
-    const documents = await chunker.splitText(content);
+    const cappedContent =
+      content.length > maxContentLength
+        ? content.slice(0, maxContentLength)
+        : content;
+    const documents = await chunker.splitText(cappedContent);
     if (Array.isArray(documents)) {
       return await reranker.rerank(query, documents, topResults);
     } else {
@@ -445,6 +457,7 @@ export const createSourceProcessor = (
   }
   const {
     topResults = 5,
+    maxContentLength = MAX_CONTENT_LENGTH,
     // strategies = ['no_extraction'],
     // filterContent = true,
     reranker,
@@ -479,11 +492,15 @@ export const createSourceProcessor = (
               );
               if (response.success && response.data) {
                 const [content, references] = scraper.extractContent(response);
+                const cleanedContent = chunker.cleanText(content);
                 return {
                   url,
                   references,
                   attribution,
-                  content: chunker.cleanText(content),
+                  content:
+                    cleanedContent.length > maxContentLength
+                      ? cleanedContent.slice(0, maxContentLength)
+                      : cleanedContent,
                 } as t.ScrapeResult;
               } else {
                 logger_.error(
@@ -512,6 +529,7 @@ export const createSourceProcessor = (
                   query,
                   reranker,
                   content: result.content,
+                  maxContentLength,
                   logger: logger_,
                 });
                 if (onGetHighlights) {
@@ -693,6 +711,7 @@ export const createSourceProcessor = (
 
       if (news && topStories.length > 0) {
         updateSourcesWithContent(topStories, sourceMap);
+        result.data.topStories = topStories.slice(0, numElements);
       }
 
       return result.data;

src/tools/search/tool.ts

@@ -353,6 +353,7 @@ export const createSearchTool = (
     searxngApiKey,
     rerankerType = 'cohere',
     topResults = 5,
+    maxContentLength,
     strategies = ['no_extraction'],
     filterContent = true,
     safeSearch = 1,
@@ -435,6 +436,7 @@ export const createSearchTool = (
     {
       reranker: selectedReranker,
       topResults,
+      maxContentLength,
       strategies,
       filterContent,
       logger,

src/tools/search/types.ts

@@ -85,6 +85,8 @@ export interface ScrapeResult {
 
 export interface ProcessSourcesConfig {
   topResults?: number;
+  /** Max chars of scraped content per source before chunking. Overridable via SEARCH_MAX_CONTENT_LENGTH. */
+  maxContentLength?: number;
   strategies?: string[];
   filterContent?: boolean;
   reranker?: BaseReranker;