diff --git a/src/benchflow/agents/install.py b/src/benchflow/agents/install.py index a65d8d6df..dd9343586 100644 --- a/src/benchflow/agents/install.py +++ b/src/benchflow/agents/install.py @@ -255,6 +255,28 @@ async def install_agent( diagnostics=diag.stdout or "", log_path=str(install_log), ) + if agent_cfg and agent_cfg.install_setup_cmd: + setup_cmd = agent_cfg.install_setup_cmd + setup_result = await env.exec(setup_cmd, timeout_sec=install_timeout) + setup_parts = [f"$ {setup_cmd}\n"] + if setup_result.stdout: + setup_parts.extend(["=== stdout ===\n", setup_result.stdout]) + if not setup_result.stdout.endswith("\n"): + setup_parts.append("\n") + if setup_result.stderr: + setup_parts.extend(["=== stderr ===\n", setup_result.stderr]) + if not setup_result.stderr.endswith("\n"): + setup_parts.append("\n") + with install_log.open("a") as handle: + handle.write("".join(setup_parts)) + if setup_result.return_code != 0: + raise AgentInstallError( + agent=agent_base, + return_code=setup_result.return_code, + stdout="".join(setup_parts), + diagnostics="post-install setup failed", + log_path=str(install_log), + ) return agent_cfg diff --git a/src/benchflow/agents/manifest.py b/src/benchflow/agents/manifest.py index daebc8cc0..83fb495ff 100644 --- a/src/benchflow/agents/manifest.py +++ b/src/benchflow/agents/manifest.py @@ -91,6 +91,8 @@ "disallow_web_tools_launch_suffix", "task_mcp_transport", "task_mcp_config_path", + "install_setup_cmd", + "launch_override_cmd", } ) diff --git a/src/benchflow/agents/openhands_settings_writer.py b/src/benchflow/agents/openhands_settings_writer.py new file mode 100644 index 000000000..3787816c3 --- /dev/null +++ b/src/benchflow/agents/openhands_settings_writer.py @@ -0,0 +1,133 @@ +"""Standalone OpenHands settings-writer source embedded into sandboxes.""" + +OPENHANDS_SETTINGS_WRITER = r"""import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +def optional_int(name): + value = os.environ.get(name, "").strip() + if not value: + return None + parsed = int(value) + if parsed <= 0: + raise ValueError(f"{name} must be positive") + return parsed + + +def optional_non_negative_int(name): + value = os.environ.get(name, "").strip() + if not value: + return None + if not value.isdigit(): + raise ValueError(f"{name} must be a non-negative integer") + return int(value) + + +def optional_bool(name): + value = os.environ.get(name, "").strip().lower() + if not value: + return None + if value in {"1", "true", "yes"}: + return True + if value in {"0", "false", "no"}: + return False + raise ValueError(f"{name} must be a boolean") + + +def disable_subagents_if_requested(): + if os.environ.get("BENCHFLOW_OPENHANDS_DISABLE_SUBAGENTS", "0") != "1": + return + + openhands_bin = shutil.which("openhands") + if not openhands_bin: + raise RuntimeError("Cannot locate OpenHands executable") + openhands_python = Path(os.path.realpath(openhands_bin)).parent / "python" + if not openhands_python.is_file() or not os.access(openhands_python, os.X_OK): + raise RuntimeError("Cannot locate OpenHands tool interpreter") + + subprocess.run( + [ + str(openhands_python), + "-c", + "from pathlib import Path\n" + "import openhands_cli.utils as u\n" + "p = Path(u.__file__)\n" + "s = p.read_text()\n" + "old = ' Tool(name=task_tool_name),\\n'\n" + "new = ' # BenchFlow: delegation disabled for this run.\\n'\n" + "assert old in s or new in s\n" + "p.write_text(s.replace(old, new, 1))\n", + ], + check=True, + ) + + +llm = { + "model": os.environ["LLM_MODEL"], + "api_key": os.environ["LLM_API_KEY"], + "usage_id": "agent", +} +for env_name, field_name in ( + ("LLM_BASE_URL", "base_url"), + ("LLM_API_VERSION", "api_version"), +): + value = os.environ.get(env_name, "").strip() + if value: + llm[field_name] = value + +for env_name, field_name in ( + ("LLM_NATIVE_TOOL_CALLING", "native_tool_calling"), + ("LLM_CACHING_PROMPT", "caching_prompt"), + ("LLM_DROP_PARAMS", "drop_params"), + ("LLM_MODIFY_PARAMS", "modify_params"), +): + value = optional_bool(env_name) + if value is not None: + llm[field_name] = value + +timeout = optional_non_negative_int("LLM_TIMEOUT") +if timeout is not None: + llm["timeout"] = timeout + +reasoning_effort = os.environ.get("LLM_REASONING_EFFORT", "").strip() +if reasoning_effort == "max": + llm["litellm_extra_body"] = {"reasoning": {"effort": "max"}} +elif reasoning_effort in {"none", "low", "medium", "high", "xhigh"}: + llm["reasoning_effort"] = reasoning_effort + llm["litellm_extra_body"] = {"reasoning_effort": reasoning_effort} +elif reasoning_effort: + llm["litellm_extra_body"] = {"reasoning_effort": reasoning_effort} + +context_limit = optional_int("BENCHFLOW_OPENHANDS_CONTEXT_LIMIT") +output_limit = optional_int("BENCHFLOW_OPENHANDS_OUTPUT_LIMIT") +if context_limit is not None: + llm["max_input_tokens"] = context_limit +if output_limit is not None: + llm["max_output_tokens"] = output_limit + +condenser = { + "llm": {**llm, "usage_id": "condenser"}, + "max_size": 80, + "keep_first": 4, + "kind": "LLMSummarizingCondenser", +} +if context_limit is not None and output_limit is not None: + reserve = optional_int("BENCHFLOW_OPENHANDS_CONTEXT_RESERVE") or 4096 + condenser_limit = context_limit - output_limit - reserve + if condenser_limit <= 0: + raise ValueError("OpenHands context budget leaves no room for input") + condenser["max_tokens"] = condenser_limit + +settings = { + "llm": llm, + "tools": [], + "condenser": condenser, + "kind": "Agent", +} +Path(sys.argv[1]).write_text(json.dumps(settings, separators=(",", ":"))) +disable_subagents_if_requested() +""" diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 09c56fb01..43ff3aac6 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -53,6 +53,10 @@ from dataclasses import dataclass, field from pathlib import Path +from benchflow.agents.openhands_settings_writer import ( + OPENHANDS_SETTINGS_WRITER as _OPENHANDS_SETTINGS_WRITER, +) + def _install_python_script(container_path: str, source: str) -> str: """Shell snippet that ensures python3 and writes `source` to container_path. @@ -120,6 +124,7 @@ def _apt_install(*packages: str) -> str: _OPENHANDS_CLI_GIT_REV = "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271" _OPENHANDS_SDK_VERSION = "1.28.1" _OPENHANDS_TOOLS_VERSION = "1.28.1" +_OPENHANDS_SETTINGS_WRITER_PATH = "/opt/benchflow/bin/openhands-settings-writer" _JS_AGENT_PATH = ( f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:" f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH" @@ -508,6 +513,11 @@ class AgentConfig: task_mcp_transport: str = "acp" # Native-config target path, relative to $HOME unless absolute. task_mcp_config_path: str = "" + # Host-owned install step for provider/harness compatibility shims that + # cannot be represented in the data-only agent manifest contract. + install_setup_cmd: str = "" + # Host-owned launch override paired with install_setup_cmd. + launch_override_cmd: str = "" # Agent registry — all supported agents @@ -966,6 +976,16 @@ class AgentConfig: "fi && " "openhands acp --always-approve --override-with-envs" ), + install_setup_cmd=_install_python_script( + _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER + ), + launch_override_cmd=( + 'export PATH="$HOME/.local/bin:$PATH" && ' + "mkdir -p ~/.openhands && " + f"python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " + "~/.openhands/agent_settings.json && " + "openhands acp --always-approve --override-with-envs" + ), protocol="acp", requires_env=["LLM_API_KEY"], api_protocol="", diff --git a/src/benchflow/cli/agent.py b/src/benchflow/cli/agent.py index 3c8be9457..36bef6c1c 100644 --- a/src/benchflow/cli/agent.py +++ b/src/benchflow/cli/agent.py @@ -84,7 +84,8 @@ def agent_show( console.print(f" Aliases: {', '.join(aliases)}") console.print(f" Description: {cfg.description}") console.print(f" Protocol: {cfg.protocol}") - console.print(f" Launch: {cfg.launch_cmd}") + launch_cmd = cfg.launch_override_cmd or cfg.launch_cmd + console.print(f" Launch: {launch_cmd}") console.print(f" Requires: {_format_requires(cfg) or '(none)'}") console.print(f" Provider auth: {_PROVIDER_AUTH_MESSAGE}") if cfg.subscription_auth: diff --git a/src/benchflow/rollout_planes.py b/src/benchflow/rollout_planes.py index 6d2272d9a..5aff10115 100644 --- a/src/benchflow/rollout_planes.py +++ b/src/benchflow/rollout_planes.py @@ -54,10 +54,14 @@ class DefaultRolloutPlanes: """Default bindings for the four concrete planes.""" def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: - launch = AGENT_LAUNCH.get(agent, agent) + agent_cfg = AGENTS.get(agent) + launch = ( + agent_cfg.launch_override_cmd + if agent_cfg and agent_cfg.launch_override_cmd + else AGENT_LAUNCH.get(agent, agent) + ) if not disallow_web_tools: return launch - agent_cfg = AGENTS.get(agent) if agent_cfg and agent_cfg.disallow_web_tools_launch_suffix: return launch + agent_cfg.disallow_web_tools_launch_suffix return launch diff --git a/src/benchflow/runtime.py b/src/benchflow/runtime.py index 1c24c8320..40c111b50 100644 --- a/src/benchflow/runtime.py +++ b/src/benchflow/runtime.py @@ -195,7 +195,7 @@ def launch_cmd(self) -> str: config = self.config if config is None: return self.name - return config.launch_cmd + return config.launch_override_cmd or config.launch_cmd def __repr__(self) -> str: return f"Agent({self.name!r}, model={self.model!r})" diff --git a/tests/test_agent_cli.py b/tests/test_agent_cli.py index 7fe2a4a80..0575849ff 100644 --- a/tests/test_agent_cli.py +++ b/tests/test_agent_cli.py @@ -27,3 +27,13 @@ def test_agent_show_mentions_provider_specific_azure_auth() -> None: assert "Provider auth:" in output assert "AZURE_API_KEY" in output assert "AZURE_API_ENDPOINT" in output + + +def test_agent_show_uses_launch_override_when_present() -> None: + """Guards PR #927: agent diagnostics show the effective runtime launch.""" + result = CliRunner().invoke(app, ["agent", "show", "openhands"]) + + assert result.exit_code == 0 + output = click.unstyle(result.output) + assert "Launch:" in output + assert "/opt/benchflow/bin/openhands-settings-writer" in output diff --git a/tests/test_openhands_context_budget.py b/tests/test_openhands_context_budget.py new file mode 100644 index 000000000..bf8af2dbb --- /dev/null +++ b/tests/test_openhands_context_budget.py @@ -0,0 +1,189 @@ +import json +import os +import subprocess +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from benchflow.agents.install import install_agent +from benchflow.agents.registry import _OPENHANDS_SETTINGS_WRITER, AGENTS +from benchflow.rollout_planes import DefaultRolloutPlanes + + +def test_openhands_settings_reserve_context_for_output(tmp_path): + """Guards PR #927 against losing OpenHands context/output budget settings.""" + target = tmp_path / "agent_settings.json" + env = { + **os.environ, + "LLM_MODEL": "openai/qwen35-9b-base", + "LLM_API_KEY": "placeholder", + "LLM_BASE_URL": "http://example.test/v1", + "LLM_NATIVE_TOOL_CALLING": "true", + "LLM_CACHING_PROMPT": "false", + "LLM_DROP_PARAMS": "true", + "LLM_MODIFY_PARAMS": "true", + "BENCHFLOW_OPENHANDS_CONTEXT_LIMIT": "262144", + "BENCHFLOW_OPENHANDS_OUTPUT_LIMIT": "32768", + "BENCHFLOW_OPENHANDS_CONTEXT_RESERVE": "4096", + } + + completed = subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + capture_output=True, + text=True, + check=True, + ) + settings = json.loads(target.read_text()) + + assert completed.stdout == "" + assert settings["kind"] == "Agent" + assert settings["llm"]["max_input_tokens"] == 262144 + assert settings["llm"]["max_output_tokens"] == 32768 + assert settings["llm"]["caching_prompt"] is False + assert settings["condenser"]["kind"] == "LLMSummarizingCondenser" + assert settings["condenser"]["max_tokens"] == 225280 + assert settings["condenser"]["llm"]["usage_id"] == "condenser" + + +def test_openhands_settings_preserves_override_launch_llm_options(tmp_path): + """Guards PR #927 follow-up against commit 64edbc1f bypassing PR #921 settings.""" + target = tmp_path / "agent_settings.json" + env = { + **os.environ, + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_BASE_URL": "http://127.0.0.1:4000/v1", + "LLM_API_VERSION": "preview", + "LLM_TIMEOUT": "115200", + "LLM_REASONING_EFFORT": "max", + } + + subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + check=True, + ) + + settings = json.loads(target.read_text()) + assert settings["llm"]["timeout"] == 115200 + assert settings["llm"]["litellm_extra_body"] == {"reasoning": {"effort": "max"}} + assert "reasoning_effort" not in settings["llm"] + assert settings["condenser"]["llm"]["timeout"] == 115200 + assert settings["condenser"]["llm"]["litellm_extra_body"] == { + "reasoning": {"effort": "max"} + } + + +def test_openhands_settings_preserves_typed_reasoning_effort(tmp_path): + """Guards PR #927 against launch_override_cmd dropping typed OpenHands effort.""" + target = tmp_path / "agent_settings.json" + env = { + **os.environ, + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_REASONING_EFFORT": "xhigh", + } + + subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + check=True, + ) + + settings = json.loads(target.read_text()) + assert settings["llm"]["reasoning_effort"] == "xhigh" + assert settings["llm"]["litellm_extra_body"] == {"reasoning_effort": "xhigh"} + + +def test_openhands_settings_can_disable_subagents(tmp_path): + """Guards PR #927 against launch_override_cmd bypassing subagent disable.""" + target = tmp_path / "agent_settings.json" + tool_root = tmp_path / "tools" + package_root = tool_root / "openhands" / "site-packages" + package_dir = package_root / "openhands_cli" + package_dir.mkdir(parents=True) + (package_dir / "__init__.py").write_text("") + utils_path = package_dir / "utils.py" + utils_path.write_text( + "def get_default_cli_tools():\n" + " return [\n" + " Tool(name=task_tool_name),\n" + " ]\n" + ) + bin_dir = tool_root / "openhands" / "bin" + bin_dir.mkdir(parents=True) + python_wrapper = bin_dir / "python" + python_wrapper.write_text( + f'#!/bin/sh\nPYTHONPATH={package_root} exec {sys.executable} "$@"\n' + ) + python_wrapper.chmod(0o755) + openhands = bin_dir / "openhands" + openhands.write_text("#!/bin/sh\nexit 0\n") + openhands.chmod(0o755) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + (fake_bin / "openhands").symlink_to(openhands) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "BENCHFLOW_OPENHANDS_DISABLE_SUBAGENTS": "1", + } + + subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + check=True, + ) + + patched = utils_path.read_text() + assert "Tool(name=task_tool_name)" not in patched + assert "BenchFlow: delegation disabled for this run." in patched + + +def test_openhands_settings_rejects_non_numeric_timeout(tmp_path): + """Guards PR #927 against malformed timeout JSON in override launch settings.""" + target = tmp_path / "agent_settings.json" + env = { + **os.environ, + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_TIMEOUT": "none", + } + + result = subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "LLM_TIMEOUT must be a non-negative integer" in result.stderr + + +def test_openhands_launch_installs_and_runs_settings_writer(): + """Guards PR #927 against bypassing the effective OpenHands launch override.""" + path = "/opt/benchflow/bin/openhands-settings-writer" + launch = DefaultRolloutPlanes().agent_launch("openhands", disallow_web_tools=False) + assert path in AGENTS["openhands"].install_setup_cmd + assert path in AGENTS["openhands"].launch_override_cmd + assert "mkdir -p ~/.openhands" in AGENTS["openhands"].launch_override_cmd + assert path in launch + assert launch == AGENTS["openhands"].launch_override_cmd + + +@pytest.mark.asyncio +async def test_openhands_install_runs_root_owned_settings_setup(tmp_path): + """Guards PR #927 against installing the settings writer after user launch.""" + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0, stdout="", stderr="")) + + await install_agent(env, "openhands", tmp_path) + + assert env.exec.await_count == 2 + setup_cmd = env.exec.await_args_list[1].args[0] + assert "/opt/benchflow/bin/openhands-settings-writer" in setup_cmd diff --git a/uv.lock b/uv.lock index e55180107..dec20d7f7 100644 --- a/uv.lock +++ b/uv.lock @@ -1982,7 +1982,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.27.2" +version = "1.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2000,9 +2000,9 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/27/3c/347cf965d313f5d41764e7d46bea6ffe7d9ef13b983cc429b0340962a082/mcp-1.27.2.tar.gz", hash = "sha256:8e02db104096d1c25b28e64bde29a5c32b31bc241710213e12fd4d84985bdfef", size = 621116, upload-time = "2026-05-29T17:16:04.039Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/77/9450b8f251a13affb6281997d0523c4615f8a8b35d0b21ff30db3a5aac9d/mcp-1.28.1.tar.gz", hash = "sha256:d51e36a5f5644faea4f85ea649bfffa6bc6c26770d42798ad6a3de3d2ba69683", size = 638501, upload-time = "2026-06-26T12:57:29.093Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/11/252c6f971dc4f16af1d98a1c469d8ba523aab00d1bb76b4d3bc1ff32eacc/mcp-1.27.2-py3-none-any.whl", hash = "sha256:d6ff5160c6ca65d93013626efb3fc249de683c30b2d8570755ceddd490344de5", size = 220498, upload-time = "2026-05-29T17:16:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5e/d118fce19f87a2e7d8101c35c8ae0ec289098a4df0ff244cec23e415aca0/mcp-1.28.1-py3-none-any.whl", hash = "sha256:2726bca5e7193f61c5dde8b12500a6de2d9acf6d1a1c0be9e8c2e706437991df", size = 222620, upload-time = "2026-06-26T12:57:27.218Z" }, ] [[package]]