feat: filter base64 image artifacts based on agent vision capability

src/agents/AgentContext.ts

@@ -42,6 +42,7 @@ export class AgentContext {
       maxContextTokens,
       reasoningKey,
       useLegacyContent,
+      vision,
       discoveredTools,
     } = agentConfig;
 
@@ -63,6 +64,7 @@ export class AgentContext {
       instructionTokens: 0,
       tokenCounter,
       useLegacyContent,
+      vision,
       discoveredTools,
     });
 
@@ -167,6 +169,11 @@ export class AgentContext {
   tokenCalculationPromise?: Promise<void>;
   /** Format content blocks as strings (for legacy compatibility) */
   useLegacyContent: boolean = false;
+  /**
+   * Whether this agent supports vision capabilities (image processing).
+   * Defaults to false if not explicitly set.
+   */
+  vision?: boolean;
   /**
    * Handoff context when this agent receives control via handoff.
    * Contains source and parallel execution info for system message context.
@@ -196,6 +203,7 @@ export class AgentContext {
     toolEnd,
     instructionTokens,
     useLegacyContent,
+    vision,
     discoveredTools,
   }: {
     agentId: string;
@@ -215,6 +223,7 @@ export class AgentContext {
     toolEnd?: boolean;
     instructionTokens?: number;
     useLegacyContent?: boolean;
+    vision?: boolean;
     discoveredTools?: string[];
   }) {
     this.agentId = agentId;
@@ -230,6 +239,7 @@ export class AgentContext {
     this.toolDefinitions = toolDefinitions;
     this.instructions = instructions;
     this.additionalInstructions = additionalInstructions;
+    this.vision = vision;
     if (reasoningKey) {
       this.reasoningKey = reasoningKey;
     }

src/graphs/Graph.ts

@@ -4,13 +4,8 @@ import { nanoid } from 'nanoid';
 import { concat } from '@langchain/core/utils/stream';
 import { ToolNode } from '@langchain/langgraph/prebuilt';
 import { ChatVertexAI } from '@langchain/google-vertexai';
-import {
-  START,
-  END,
-  StateGraph,
-  Annotation,
-  messagesStateReducer,
-} from '@langchain/langgraph';
+import { START, END, StateGraph, Annotation } from '@langchain/langgraph';
+import { messagesStateReducer } from '@/messages/reducer';
 import {
   Runnable,
   RunnableConfig,
@@ -173,7 +168,6 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
   agentContexts: Map<string, AgentContext> = new Map();
   /** Default agent ID to use */
   defaultAgentId: string;
-
   constructor({
     // parent-level graph inputs
     runId,
@@ -658,6 +652,12 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       throw new Error('No model found');
     }
 
+    if ((_tools?.length ?? 0) > 0 && manualToolStreamProviders.has(provider)) {
+      if (!model.stream) {
+        throw new Error('Model does not support stream');
+      }
+    }
+
     if (model.stream) {
       /**
        * Process all model output through a local ChatModelStreamHandler in the
@@ -758,12 +758,16 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       }
 
       const toolsForBinding = agentContext.getToolsForBinding();
+      const clientOptionsWithVision = {
+        ...agentContext.clientOptions,
+        vision: agentContext.vision,
+      } as unknown as t.ClientOptions;
       let model =
         this.overrideModel ??
         this.initializeModel({
           tools: toolsForBinding,
           provider: agentContext.provider,
-          clientOptions: agentContext.clientOptions,
+          clientOptions: clientOptionsWithVision,
         });
 
       if (agentContext.systemRunnable) {
@@ -779,6 +783,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       this.config = config;
 
       let messagesToUse = messages;
+
       if (
         !agentContext.pruneMessages &&
         agentContext.tokenCounter &&
@@ -842,13 +847,23 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
 
       const isLatestToolMessage = lastMessageY instanceof ToolMessage;
 
+      // Check if any ToolMessage in current turn has artifacts
+      // This is more robust than only checking if the last message is a ToolMessage
+      const hasToolMessagesWithArtifacts = finalMessages.some((msg) => {
+        if (msg._getType() !== 'tool') return false;
+        const toolMsg = msg as ToolMessage & { artifact?: t.MCPArtifact };
+        return (
+          toolMsg.artifact != null || toolMsg.additional_kwargs.artifact != null
+        );
+      });
+
       if (
         isLatestToolMessage &&
         agentContext.provider === Providers.ANTHROPIC
       ) {
         formatAnthropicArtifactContent(finalMessages);
       } else if (
-        isLatestToolMessage &&
+        hasToolMessagesWithArtifacts &&
         ((isOpenAILike(agentContext.provider) &&
           agentContext.provider !== Providers.DEEPSEEK) ||
           isGoogleLike(agentContext.provider))
@@ -1092,6 +1107,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       }
 
       agentContext.currentUsage = this.getUsageMetadata(result.messages?.[0]);
+
       this.cleanupSignalListener();
       return result;
     };

src/graphs/MultiAgentGraph.ts

@@ -13,8 +13,8 @@ import {
   StateGraph,
   Annotation,
   getCurrentTaskInput,
-  messagesStateReducer,
 } from '@langchain/langgraph';
+import { messagesStateReducer } from '@/messages/reducer';
 import type { LangGraphRunnableConfig } from '@langchain/langgraph';
 import type { BaseMessage, AIMessageChunk } from '@langchain/core/messages';
 import type { ToolRunnableConfig } from '@langchain/core/tools';
@@ -821,7 +821,9 @@ export class MultiAgentGraph extends StandardGraph {
             }
           }
 
-          /** Update token map if we have a token counter */
+          /**
+           * Update token map if we have a token counter
+           */
           if (agentContext?.tokenCounter && hasInstructions) {
             const freshTokenMap: Record<string, number> = {};
             for (

src/llm/anthropic/index.ts

@@ -234,22 +234,24 @@ export class CustomAnthropic extends ChatAnthropicMessages {
         );
       }
 
+      const maxTokens = Math.max(1, this.maxTokens ?? 4096);
       return {
         model: this.model,
         stop_sequences: options?.stop ?? this.stopSequences,
         stream: this.streaming,
-        max_tokens: this.maxTokens,
+        max_tokens: maxTokens,
         ...sharedParams,
       };
     }
+    const maxTokens = Math.max(1, this.maxTokens ?? 4096);
     return {
       model: this.model,
       temperature: this.temperature,
       top_k: this.top_k,
       top_p: this.topP,
       stop_sequences: options?.stop ?? this.stopSequences,
       stream: this.streaming,
-      max_tokens: this.maxTokens,
+      max_tokens: maxTokens,
       ...sharedParams,
     };
   }

src/llm/openai/index.ts

@@ -199,13 +199,19 @@ export class CustomAzureOpenAIClient extends AzureOpenAIClient {
 export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
   _lc_stream_delay?: number;
 
+  /** When false, image_url parts are stripped before sending (avoids non-multimodal API errors). */
+  protected visionCapable: boolean;
+
   constructor(
     fields?: t.ChatOpenAICallOptions & {
       _lc_stream_delay?: number;
+      vision?: boolean;
     } & t.OpenAIChatInput['modelKwargs']
   ) {
-    super(fields);
-    this._lc_stream_delay = fields?._lc_stream_delay;
+    const { vision, _lc_stream_delay, ...rest } = fields ?? {};
+    super(rest as typeof fields);
+    this._lc_stream_delay = _lc_stream_delay;
+    this.visionCapable = vision ?? true;
   }
 
   public get exposedClient(): CustomOpenAIClient {
@@ -287,7 +293,8 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
         input: _convertMessagesToOpenAIResponsesParams(
           messages,
           this.model,
-          this.zdrEnabled
+          this.zdrEnabled,
+          this.visionCapable
         ),
         stream: true,
       },
@@ -322,7 +329,9 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     const messagesMapped: OpenAICompletionParam[] =
-      _convertMessagesToOpenAIParams(messages, this.model);
+      _convertMessagesToOpenAIParams(messages, this.model, {
+        visionCapable: this.visionCapable,
+      });
 
     const params = {
       ...this.invocationParams(options, {
@@ -459,9 +468,18 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
 export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
   _lc_stream_delay?: number;
 
-  constructor(fields?: t.AzureOpenAIInput & { _lc_stream_delay?: number }) {
-    super(fields);
-    this._lc_stream_delay = fields?._lc_stream_delay;
+  protected visionCapable: boolean;
+
+  constructor(
+    fields?: t.AzureOpenAIInput & {
+      _lc_stream_delay?: number;
+      vision?: boolean;
+    }
+  ) {
+    const { vision, _lc_stream_delay, ...rest } = fields ?? {};
+    super(rest as typeof fields);
+    this._lc_stream_delay = _lc_stream_delay;
+    this.visionCapable = vision ?? true;
   }
 
   public get exposedClient(): CustomOpenAIClient {
@@ -581,7 +599,8 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
         input: _convertMessagesToOpenAIResponsesParams(
           messages,
           this.model,
-          this.zdrEnabled
+          this.zdrEnabled,
+          this.visionCapable
         ),
         stream: true,
       },
@@ -611,6 +630,17 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
   }
 }
 export class ChatDeepSeek extends OriginalChatDeepSeek {
+  protected visionCapable: boolean;
+
+  constructor(
+    fields?: ConstructorParameters<typeof OriginalChatDeepSeek>[0] & {
+      vision?: boolean;
+    }
+  ) {
+    const { vision, ...rest } = fields ?? {};
+    super(rest as ConstructorParameters<typeof OriginalChatDeepSeek>[0]);
+    this.visionCapable = vision ?? true;
+  }
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
@@ -758,6 +788,7 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
     const messagesMapped: OpenAICompletionParam[] =
       _convertMessagesToOpenAIParams(messages, this.model, {
         includeReasoningContent: true,
+        visionCapable: this.visionCapable,
       });
 
     const params = {
@@ -1138,15 +1169,19 @@ export class ChatMoonshot extends ChatOpenAI {
 export class ChatXAI extends OriginalChatXAI {
   _lc_stream_delay?: number;
 
+  protected visionCapable: boolean;
+
   constructor(
     fields?: Partial<ChatXAIInput> & {
       configuration?: { baseURL?: string };
       clientConfig?: { baseURL?: string };
       _lc_stream_delay?: number;
+      vision?: boolean;
     }
   ) {
     super(fields);
     this._lc_stream_delay = fields?._lc_stream_delay;
+    this.visionCapable = fields?.vision ?? true;
     const customBaseURL =
       fields?.configuration?.baseURL ?? fields?.clientConfig?.baseURL;
     if (customBaseURL != null && customBaseURL) {
@@ -1202,7 +1237,9 @@ export class ChatXAI extends OriginalChatXAI {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     const messagesMapped: OpenAICompletionParam[] =
-      _convertMessagesToOpenAIParams(messages, this.model);
+      _convertMessagesToOpenAIParams(messages, this.model, {
+        visionCapable: this.visionCapable,
+      });
 
     const params = {
       ...this.invocationParams(options, {