Add Z.ai GLM benchmark model support

.env.example

@@ -2,6 +2,11 @@ BROWSER_USE_API_KEY=
 GOOGLE_API_KEY=
 OPENAI_API_KEY=
 ANTHROPIC_API_KEY=
+ZAI_API_KEY=
+# Standard pay-as-you-go endpoint:
+# ZAI_BASE_URL=https://api.z.ai/api/paas/v4
+# GLM Coding Plan endpoint:
+ZAI_BASE_URL=https://api.z.ai/api/coding/paas/v4
 
 # Optional model providers used by some framework/model combinations
 GOOGLE_GENERATIVE_AI_API_KEY=

llms/__init__.py

@@ -0,0 +1,3 @@
+from llms.zai import ChatZAI
+
+__all__ = ["ChatZAI"]

llms/zai.py

@@ -0,0 +1,196 @@
+"""z.ai chat model support for browser-use.
+
+The z.ai OpenAI-compatible endpoint supports native function calling and
+GLM-specific thinking controls. Browser-use's generic ChatOpenAI wrapper does
+not pass the z.ai `thinking` object, and its structured-output path relies on
+OpenAI JSON schema responses. This wrapper uses z.ai function calls for
+structured output instead.
+"""
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, Literal, TypeVar, overload
+
+import httpx
+from openai import APIConnectionError, APIStatusError, AsyncOpenAI, RateLimitError
+from openai.types.chat.chat_completion import ChatCompletion
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.openai.serializer import OpenAIMessageSerializer
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar("T", bound=BaseModel)
+
+
+@dataclass
+class ChatZAI(BaseChatModel):
+    model: str
+    api_key: str | None = None
+    base_url: str | httpx.URL | None = None
+    timeout: float | httpx.Timeout | None = 180
+    max_retries: int = 5
+    temperature: float | None = 0.1
+    top_p: float | None = None
+    max_tokens: int | None = 8192
+    thinking: Literal["enabled", "disabled"] = "enabled"
+    reasoning_effort: Literal["max", "xhigh", "high", "medium", "low", "minimal", "none"] | None = "max"
+    default_headers: Mapping[str, str] | None = None
+    default_query: Mapping[str, object] | None = None
+    http_client: httpx.AsyncClient | None = None
+
+    @property
+    def provider(self) -> str:
+        return "zai"
+
+    @property
+    def name(self) -> str:
+        return self.model
+
+    def _client(self) -> AsyncOpenAI:
+        params: dict[str, Any] = {
+            "api_key": self.api_key,
+            "base_url": self.base_url,
+            "timeout": self.timeout,
+            "max_retries": self.max_retries,
+            "default_headers": self.default_headers,
+            "default_query": self.default_query,
+        }
+        if self.http_client is not None:
+            params["http_client"] = self.http_client
+        return AsyncOpenAI(**{k: v for k, v in params.items() if v is not None})
+
+    def _model_params(self) -> dict[str, Any]:
+        extra_body: dict[str, Any] = {"thinking": {"type": self.thinking}}
+        if self.reasoning_effort is not None:
+            extra_body["reasoning_effort"] = self.reasoning_effort
+        params: dict[str, Any] = {"extra_body": extra_body}
+        if self.temperature is not None:
+            params["temperature"] = self.temperature
+        if self.top_p is not None:
+            params["top_p"] = self.top_p
+        if self.max_tokens is not None:
+            params["max_tokens"] = self.max_tokens
+        return params
+
+    def _usage(self, response: ChatCompletion) -> ChatInvokeUsage | None:
+        if response.usage is None:
+            return None
+        prompt_details = response.usage.prompt_tokens_details
+        return ChatInvokeUsage(
+            prompt_tokens=response.usage.prompt_tokens,
+            prompt_cached_tokens=prompt_details.cached_tokens if prompt_details else None,
+            prompt_cache_creation_tokens=None,
+            prompt_image_tokens=None,
+            completion_tokens=response.usage.completion_tokens,
+            total_tokens=response.usage.total_tokens,
+        )
+
+    def _thinking_from_response(self, response: ChatCompletion) -> str | None:
+        if not response.choices:
+            return None
+        message = response.choices[0].message
+        reasoning = getattr(message, "reasoning_content", None)
+        if reasoning is None and getattr(message, "model_extra", None):
+            reasoning = message.model_extra.get("reasoning_content")
+        return reasoning
+
+    def _tool_for_output(self, output_format: type[BaseModel]) -> dict[str, Any]:
+        schema = SchemaOptimizer.create_optimized_json_schema(
+            output_format,
+            remove_min_items=True,
+            remove_defaults=True,
+        )
+        return {
+            "type": "function",
+            "function": {
+                "name": "agent_output",
+                "description": f"Return the browser-use action result as {output_format.__name__}.",
+                "parameters": schema,
+            },
+        }
+
+    def _parse_tool_arguments(self, arguments: Any, output_format: type[T]) -> T:
+        if isinstance(arguments, str):
+            data = json.loads(arguments)
+        elif isinstance(arguments, dict):
+            data = arguments
+        else:
+            raise TypeError(f"Unexpected function arguments type: {type(arguments).__name__}")
+        return output_format.model_validate(data)
+
+    def _parse_content_json(self, content: str, output_format: type[T]) -> T:
+        try:
+            return output_format.model_validate_json(content)
+        except Exception:
+            start = content.find("{")
+            end = content.rfind("}")
+            if start == -1 or end == -1 or end <= start:
+                raise
+            return output_format.model_validate_json(content[start : end + 1])
+
+    @overload
+    async def ainvoke(
+        self, messages: list[BaseMessage], output_format: None = None, **kwargs: Any
+    ) -> ChatInvokeCompletion[str]: ...
+
+    @overload
+    async def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...
+
+    async def ainvoke(
+        self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+    ) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+        openai_messages = OpenAIMessageSerializer.serialize_messages(messages)
+        try:
+            params = self._model_params()
+            if output_format is None:
+                response = await self._client().chat.completions.create(
+                    model=self.model,
+                    messages=openai_messages,
+                    **params,
+                )
+                return ChatInvokeCompletion(
+                    completion=response.choices[0].message.content or "",
+                    thinking=self._thinking_from_response(response),
+                    usage=self._usage(response),
+                    stop_reason=response.choices[0].finish_reason if response.choices else None,
+                )
+
+            response = await self._client().chat.completions.create(
+                model=self.model,
+                messages=openai_messages,
+                tools=[self._tool_for_output(output_format)],
+                tool_choice="auto",
+                **params,
+            )
+            message = response.choices[0].message
+            if message.tool_calls:
+                completion = self._parse_tool_arguments(message.tool_calls[0].function.arguments, output_format)
+            elif message.content:
+                completion = self._parse_content_json(message.content, output_format)
+            else:
+                raise ModelProviderError(
+                    message="Expected z.ai function call or JSON content but got neither",
+                    status_code=500,
+                    model=self.name,
+                )
+
+            return ChatInvokeCompletion(
+                completion=completion,
+                thinking=self._thinking_from_response(response),
+                usage=self._usage(response),
+                stop_reason=response.choices[0].finish_reason if response.choices else None,
+            )
+
+        except RateLimitError as e:
+            raise ModelRateLimitError(message=e.message, model=self.name) from e
+        except APIConnectionError as e:
+            raise ModelProviderError(message=str(e), model=self.name) from e
+        except APIStatusError as e:
+            raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+        except Exception as e:
+            raise ModelProviderError(message=str(e), model=self.name) from e

run_batch.py

@@ -10,10 +10,13 @@
 from dotenv import load_dotenv
 from browser_use import ChatGoogle
 from browser_use.llm import ChatBrowserUse, ChatOpenAI, ChatAnthropic
+from llms import ChatZAI
 from run_eval import load_tasks, run_task
 
 load_dotenv()
 
+ZAI_BASE_URL = os.getenv("ZAI_BASE_URL") or "https://api.z.ai/api/coding/paas/v4"
+
 def interleave(tasks: list) -> list:
     """Reorder 100 tasks, 20 per section to balance difficulty."""
     reordered = []
@@ -30,6 +33,17 @@ def interleave(tasks: list) -> list:
     "gpt-5-mini": lambda: ChatOpenAI(model="gpt-5-mini", api_key=os.getenv("OPENAI_API_KEY")),
     "gpt-5.1-codex-mini": lambda: ChatOpenAI(model="gpt-5.1-codex-mini", api_key=os.getenv("OPENAI_API_KEY")),
     "gpt-5": lambda: ChatOpenAI(model="gpt-5", api_key=os.getenv("OPENAI_API_KEY")),
+    "GLM-5.1": lambda: ChatZAI(
+        model="glm-5.1",
+        api_key=os.getenv("ZAI_API_KEY"),
+        base_url=ZAI_BASE_URL,
+        reasoning_effort=None,
+    ),
+    "GLM-5.2": lambda: ChatZAI(
+        model="glm-5.2",
+        api_key=os.getenv("ZAI_API_KEY"),
+        base_url=ZAI_BASE_URL,
+    ),
 
     "claude-3-5-haiku": lambda: ChatAnthropic(model="claude-3-5-haiku", api_key=os.getenv("ANTHROPIC_API_KEY")),
     "claude-haiku-4-5": lambda: ChatAnthropic(model="claude-haiku-4-5", api_key=os.getenv("ANTHROPIC_API_KEY")),

run_eval.py

@@ -36,18 +36,39 @@
 from browser_use.llm import ChatBrowserUse
 from browsers import PROVIDERS, get_provider
 from judge import construct_judge_messages, JudgementResult
+from llms import ChatZAI
 
 load_dotenv()
 
 # Judge LLM - always use gemini-2.5-flash for consistent judging across all evaluations
 JUDGE_LLM = ChatGoogle(model="gemini-2.5-flash", api_key=os.getenv("GOOGLE_API_KEY"))
-TASKS_FILE = Path(__file__).parent / "BU_Bench_V1.enc"
+DEFAULT_BENCHMARK = "BU_Bench_V1"
+BENCHMARKS = {
+    "BU_Bench_V1": Path(__file__).parent / "BU_Bench_V1.enc",
+    "Stealth_Bench_V1": Path(__file__).parent / "Stealth_Bench_V1.enc",
+}
 MAX_CONCURRENT = 3
 TASK_TIMEOUT = 1800  # 30 minutes max per task
 
 AGENT_FRAMEWORK_NAME = "BrowserUse"
 AGENT_FRAMEWORK_VERSION = "0.11.5"
 MODEL_NAME = "bu-2-0"
+ZAI_BASE_URL = os.getenv("ZAI_BASE_URL") or "https://api.z.ai/api/coding/paas/v4"
+
+MODELS = {
+    "bu-2-0": lambda: ChatBrowserUse(model="bu-2-0"),
+    "GLM-5.1": lambda: ChatZAI(
+        model="glm-5.1",
+        api_key=os.getenv("ZAI_API_KEY"),
+        base_url=ZAI_BASE_URL,
+        reasoning_effort=None,
+    ),
+    "GLM-5.2": lambda: ChatZAI(
+        model="glm-5.2",
+        api_key=os.getenv("ZAI_API_KEY"),
+        base_url=ZAI_BASE_URL,
+    ),
+}
 
 
 def encode_screenshots(paths: list[str]) -> list[str]:
@@ -60,9 +81,9 @@ def encode_screenshots(paths: list[str]) -> list[str]:
     return result
 
 
-def load_tasks() -> list[dict]:
-    key = base64.urlsafe_b64encode(hashlib.sha256(b"BU_Bench_V1").digest())
-    encrypted = base64.b64decode(TASKS_FILE.read_text())
+def load_tasks(benchmark: str = DEFAULT_BENCHMARK) -> list[dict]:
+    key = base64.urlsafe_b64encode(hashlib.sha256(benchmark.encode()).digest())
+    encrypted = base64.b64decode(BENCHMARKS[benchmark].read_text())
     return json.loads(Fernet(key).decrypt(encrypted))
 
 
@@ -211,7 +232,13 @@ async def run_task(
 
 
 async def main():
-    parser = argparse.ArgumentParser(description="Run BU_Bench_V1 evaluation")
+    parser = argparse.ArgumentParser(description="Run benchmark evaluation")
+    parser.add_argument(
+        "--benchmark",
+        default=DEFAULT_BENCHMARK,
+        choices=list(BENCHMARKS.keys()),
+        help=f"Benchmark to run (default: {DEFAULT_BENCHMARK})",
+    )
     parser.add_argument(
         "--browser",
         default="browser-use-cloud",
@@ -224,6 +251,12 @@ async def main():
         default=None,
         help="Number of tasks to run (default: all)",
     )
+    parser.add_argument(
+        "--model",
+        default=MODEL_NAME,
+        choices=list(MODELS.keys()),
+        help=f"Model to use (default: {MODEL_NAME})",
+    )
     args = parser.parse_args()
 
     # Resolve browser provider (None = use native browser-use-cloud path)
@@ -232,23 +265,37 @@ async def main():
         browser_provider = None
     else:
         browser_provider = get_provider(browser_name)
+    model_name = args.model
+    llm = MODELS[model_name]()
+    stealth = (
+        bool(browser_provider)
+        and getattr(browser_provider, "STEALTH_CAPABLE", False)
+        and browser_provider.stealth_enabled()
+    )
 
     # Build run key and paths
     run_start = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_key = f"{AGENT_FRAMEWORK_NAME}_{AGENT_FRAMEWORK_VERSION}_browser_{browser_name}_model_{MODEL_NAME}"
+    if args.benchmark == DEFAULT_BENCHMARK:
+        run_key = f"{AGENT_FRAMEWORK_NAME}_{AGENT_FRAMEWORK_VERSION}_browser_{browser_name}_model_{model_name}"
+    else:
+        run_key = f"{args.benchmark}_browser_{browser_name}_model_{model_name}"
     run_data_dir = (
         Path(__file__).parent / "run_data" / f"{run_key}_start_at_{run_start}"
     )
     results_file = Path(__file__).parent / "results" / f"{run_key}.json"
 
-    tasks = load_tasks()
+    tasks = load_tasks(args.benchmark)
     if args.tasks:
         tasks = tasks[: args.tasks]
     sem = asyncio.Semaphore(MAX_CONCURRENT)
     results = await asyncio.gather(
         *[
             run_task(
-                t, sem, browser_provider=browser_provider, run_data_dir=run_data_dir
+                t,
+                sem,
+                browser_provider=browser_provider,
+                llm=llm,
+                run_data_dir=run_data_dir,
             )
             for t in tasks
         ]
@@ -266,6 +313,10 @@ async def main():
     runs.append(
         {
             "run_start": run_start,
+            "benchmark": args.benchmark,
+            "browser": browser_name,
+            "model": model_name,
+            "stealth": stealth,
             "tasks_completed": len(results),
             "tasks_successful": successful,
             "total_steps": total_steps,