From db239a0afb072298f86e0dd76b1c01936a511240 Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 25 Jul 2026 06:56:28 +0300 Subject: [PATCH 1/3] fix: route optimizer-trial harbor flags so tau3 survives teardown The tau3 optimizer trial kept dying ~22 min in, after codex had finished, with `grpclib StreamTerminatedError: Connection lost` raised from harbor's finalization verifier exec (harbor/verifier/verifier.py:199 -> environments/dind_compose.py -> modal.py `_sdk_exec`). Harbor logs its own remedy at trial start: "DinD mode uses host networking ... Use --ek modal_vm_runtime=true to use the VM runtime instead". There was no way to declare that. `extra_harbor_args` reaches the NESTED `harbor run` that scores a candidate (build/config.py -> HarborBackendConfig -> backend.py `_harbor_command`), not the OUTER `harbor run` that hosts the optimizer trial, which is where the stream drops. Add `optimizer_harbor_args`, forwarded by `vero harbor run` to the outer command ahead of any trailing CLI args (harbor's `--ek` is last-value-wins, so the command line still overrides), and set tau3's to `--ek modal_vm_runtime=true`. This supersedes the previous version of this PR, which raised tau3's `max_retries` 1 -> 3. That knob is the nested per-case retry and was proven not to be the failing layer: the 2026-07-25 run still errored with `retries=0` and "Not retrying trial because the maximum number of retries has been reached", because the outer trial never receives `--max-retries`. Reverted to 1. Co-Authored-By: Claude Opus 5 (1M context) --- .../tau3/baseline/build.yaml | 6 ++ vero/src/vero/harbor/build/config.py | 28 +++++++++ vero/src/vero/harbor/cli.py | 3 + vero/tests/test_v05_cli.py | 58 +++++++++++++++++++ vero/tests/test_v05_harbor_build.py | 6 ++ 5 files changed, 101 insertions(+) diff --git a/harness-engineering-bench/tau3/baseline/build.yaml b/harness-engineering-bench/tau3/baseline/build.yaml index 8e07caa7..74e2d528 100644 --- a/harness-engineering-bench/tau3/baseline/build.yaml +++ b/harness-engineering-bench/tau3/baseline/build.yaml @@ -49,6 +49,12 @@ model: fireworks_ai/deepseek-v4-flash environment_name: ${inner_env:-modal} # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +# The optimizer trial itself is long (>20 min of codex) and kept dying at teardown +# with grpclib StreamTerminatedError ("Connection lost") while the finalization +# verifier exec'd into the sandbox. Harbor's DinD strategy runs the whole sandbox +# over one host-networked gRPC stream, so a single drop kills the trial; the VM +# runtime does not. Harbor emits this exact suggestion in its own startup warning. +optimizer_harbor_args: ["--ek", "modal_vm_runtime=true"] harbor_python_version: "3.12" n_attempts: 1 max_retries: 1 diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py index d6bc857b..14630987 100644 --- a/vero/src/vero/harbor/build/config.py +++ b/vero/src/vero/harbor/build/config.py @@ -424,6 +424,10 @@ class HarborBuildConfig(StrictModel): task_environment, which is that sub-run's environment. Use it for things like UV_TOOL_BIN_DIR, so `uv tool install` targets a writable directory on a non-root sandbox. + optimizer_harbor_args: Extra flags for the outer harbor run that hosts + the optimizer trial. Distinct from extra_harbor_args, which tunes + the nested evaluation sub-run. Rejected if they override a flag + `vero harbor run` controls. timeout_seconds: Wall clock for one evaluation. case_timeout_seconds: Wall clock for one case. task_agent_timeout_seconds: Wall clock declared for the target agent. @@ -534,8 +538,16 @@ class HarborBuildConfig(StrictModel): feedback_transcripts: bool = False feedback_max_bytes: int = Field(default=3000, ge=0) expose_attempt_detail: bool = False + # Extra flags for the NESTED `harbor run` that evaluates a candidate. extra_harbor_args: list[str] = Field(default_factory=list) agent_env: dict[str, str] = Field(default_factory=dict) + # Extra flags for the OUTER `harbor run` that hosts the optimizer trial + # (`vero harbor run`). Distinct from `extra_harbor_args`: that one tunes the + # nested eval sub-run, this one tunes the environment the optimizer itself + # lives in. A build declares here what its optimizer trial needs to survive, + # e.g. `--ek modal_vm_runtime=true` for a long trial whose teardown keeps + # losing the DinD gRPC stream. + optimizer_harbor_args: list[str] = Field(default_factory=list) timeout_seconds: float = Field(default=1800.0, gt=0) case_timeout_seconds: float = Field(default=180.0, gt=0) @@ -651,6 +663,22 @@ def validate_extra_harbor_args(cls, value: list[str]) -> list[str]: ) return value + @field_validator("optimizer_harbor_args") + @classmethod + def validate_optimizer_harbor_args(cls, value: list[str]) -> list[str]: + # `vero harbor run` owns the outer command's task, agent, environment and + # model; a build must not fight the CLI over them. + controlled = {"-a", "-e", "-m", "-p"} + conflicts = [ + argument for argument in value if argument.split("=", 1)[0] in controlled + ] + if conflicts: + raise ValueError( + "optimizer_harbor_args override controlled flags: " + + ", ".join(conflicts) + ) + return value + @field_validator("partitions") @classmethod def validate_partitions( diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 8b700184..72730a66 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -437,6 +437,9 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) # for a deterministic command line. for key in sorted(config.agent_env): command.extend(["--ae", f"{key}={config.agent_env[key]}"]) + # Build-declared outer-trial flags first, so a command-line arg can still + # override them (harbor's `--ek` takes the last value for a key). + command.extend(config.optimizer_harbor_args) command.extend(extra) click.echo(shlex.join(command)) completed = subprocess.run( diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index d18534c8..c19f9ddb 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -309,3 +309,61 @@ def test_cli_rejects_options_that_do_not_apply(tmp_path: Path): assert "are only valid with --produce" in producer_timeout.output assert wandb.exit_code == 2 assert "require --wandb-project" in wandb.output + + +def test_harbor_run_forwards_build_declared_optimizer_args(tmp_path, monkeypatch): + """`optimizer_harbor_args` tunes the OUTER trial, `extra_harbor_args` the nested eval. + + Regression guard for the tau3 teardown failure: the build declared + `--ek modal_vm_runtime=true` and it has to reach the `harbor run` that hosts + the optimizer, not the `harbor run` that scores a candidate. + """ + from vero.harbor import build as harbor_build + from vero.harbor import cli as harbor_cli + + config_path = tmp_path / "build.yaml" + config_path.write_text("name: org/task\n", encoding="utf-8") + + class _Config: + harbor_requirement = "harbor[modal]==0.20.0" + optimizer_harbor_args = ["--ek", "modal_vm_runtime=true"] + extra_harbor_args = ["--ek", "app_name=nested-only"] + + monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: _Config()) + monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda config, output: output) + monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") + monkeypatch.setattr( + harbor_cli, "_compiled_run_environment", lambda task, overrides: {} + ) + + recorded: list[list[str]] = [] + + def _record(command, env=None): + recorded.append(command) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(harbor_cli.subprocess, "run", _record) + + result = CliRunner().invoke( + main, + [ + "harbor", + "run", + "--config", + str(config_path), + "--agent", + "codex", + "--model", + "gpt-5.3-codex", + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + assert len(recorded) == 1 + command = recorded[0] + assert "modal_vm_runtime=true" in command + # The nested-eval flags stay out of the outer command. + assert "app_name=nested-only" not in command + # Build-declared flags come first so a command-line `--ek` can override them. + assert command.index("modal_vm_runtime=true") < command.index("--yes") diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 43b9c61c..c3770ee4 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -391,6 +391,12 @@ def test_build_config_requires_pins_and_valid_partition_references(tmp_path): _config(tmp_path / "unknown", selection_partition="missing") with pytest.raises(ValidationError, match="controlled flags"): _config(tmp_path / "flags", extra_harbor_args=["--jobs-dir=/forged"]) + with pytest.raises(ValidationError, match="controlled flags"): + _config(tmp_path / "outer-flags", optimizer_harbor_args=["-a", "forged"]) + assert _config( + tmp_path / "outer-ek", + optimizer_harbor_args=["--ek", "modal_vm_runtime=true"], + ).optimizer_harbor_args == ["--ek", "modal_vm_runtime=true"] with pytest.raises(ValidationError, match="explicit version"): _config(tmp_path / "source", task_source="org/unversioned") From f66c6179d656c0f183dd9d4696a0659c9bfb344c Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 25 Jul 2026 10:26:46 +0300 Subject: [PATCH 2/3] fix: stop claiming a tau3 fix no experiment supports Four live runs, each ~40 min on Modal, all ending the same way: default DinD 08:13 StreamTerminatedError (verifier exec) modal_vm_runtime 09:18 StreamTerminatedError (codex agent exec) modal_sandbox_v2 10:02 StreamTerminatedError --max-retries 1 10:22 StreamTerminatedError on both attempts So tau3's optimizer trial gets no `optimizer_harbor_args` value. None of the four is a fix, and a committed config value that implies otherwise is worse than an empty one. What the experiments did establish: - The mechanism works. With `--max-retries 1` the trial logged "failed with exception StreamTerminatedError. Retrying in 1.00 seconds" where every earlier run logged "Not retrying ... maximum number of retries has been reached". The outer layer is now configurable, which it was not before. - Retry is not a cure here. The second attempt died the same way, so this converts one guaranteed loss into two. - It is not a networking mode. The failure moved between phases under vm_runtime but kept an identical bottom of stack, and sandbox_v2 changed nothing. - It looks deterministic, not flaky: 39m55s, 39m45s and 38m14s to failure across three independent runs is too tight for a random stream drop. The common factor is that harbor runs each phase as one exec and streams its stdout for that phase's whole duration. No harbor timeout matches ~40 min, so the deadline, if there is one, is not in harbor's config surface. This PR is therefore just the `optimizer_harbor_args` plumbing plus its tests. The tau3 failure belongs upstream with harbor. Refs #55. Co-Authored-By: Claude Opus 5 (1M context) --- harness-engineering-bench/tau3/baseline/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/harness-engineering-bench/tau3/baseline/build.yaml b/harness-engineering-bench/tau3/baseline/build.yaml index 74e2d528..beacdb45 100644 --- a/harness-engineering-bench/tau3/baseline/build.yaml +++ b/harness-engineering-bench/tau3/baseline/build.yaml @@ -49,12 +49,12 @@ model: fireworks_ai/deepseek-v4-flash environment_name: ${inner_env:-modal} # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"] -# The optimizer trial itself is long (>20 min of codex) and kept dying at teardown -# with grpclib StreamTerminatedError ("Connection lost") while the finalization -# verifier exec'd into the sandbox. Harbor's DinD strategy runs the whole sandbox -# over one host-networked gRPC stream, so a single drop kills the trial; the VM -# runtime does not. Harbor emits this exact suggestion in its own startup warning. -optimizer_harbor_args: ["--ek", "modal_vm_runtime=true"] +# NOTE: tau3's optimizer trial reliably dies at ~38-40 min with grpclib +# StreamTerminatedError ("Connection lost"). Four live experiments (default +# DinD, --ek modal_vm_runtime=true, --ek modal_sandbox_v2=true, and +# --max-retries 1) all failed identically, so no `optimizer_harbor_args` value +# is set here: none of them is a fix, and a config value that claims to be one +# is worse than nothing. Tracked upstream; see the PR that added this field. harbor_python_version: "3.12" n_attempts: 1 max_retries: 1 From 28a533153c2250fae6e4de5684dac3bf752ae8ce Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sun, 26 Jul 2026 09:28:38 +0300 Subject: [PATCH 3/3] test: teach the run-command config stubs about optimizer_harbor_args Two tests in test_v05_harbor_http.py build the build-config as a SimpleNamespace carrying only the attributes run_command reads, so adding a field to HarborBuildConfig breaks them with an AttributeError that surfaces only as a non-zero exit code. Same convention as when agent_env was added: the stub grows with the config. The mirror of that also applied to this branch's own test, whose _Config predates agent_env and so broke once both fields were in the same code path. Suite now matches the base branch exactly: 11 pre-existing failures, none new. Co-Authored-By: Claude Opus 5 (1M context) --- vero/tests/test_v05_cli.py | 1 + vero/tests/test_v05_harbor_http.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index c19f9ddb..5d48c9a8 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -326,6 +326,7 @@ def test_harbor_run_forwards_build_declared_optimizer_args(tmp_path, monkeypatch class _Config: harbor_requirement = "harbor[modal]==0.20.0" + agent_env: dict[str, str] = {} optimizer_harbor_args = ["--ek", "modal_vm_runtime=true"] extra_harbor_args = ["--ek", "app_name=nested-only"] diff --git a/vero/tests/test_v05_harbor_http.py b/vero/tests/test_v05_harbor_http.py index a4e17a52..0cbc1ffb 100644 --- a/vero/tests/test_v05_harbor_http.py +++ b/vero/tests/test_v05_harbor_http.py @@ -260,7 +260,11 @@ def test_harbor_run_uses_current_python_and_pinned_harbor_extra(tmp_path, monkey config_path = tmp_path / "build.yaml" config_path.write_text("task_name: unused\n") - config = SimpleNamespace(harbor_requirement="harbor[modal]==0.20.0", agent_env={}) + config = SimpleNamespace( + harbor_requirement="harbor[modal]==0.20.0", + agent_env={}, + optimizer_harbor_args=[], + ) observed = {} def compile_task(_config, output): @@ -313,7 +317,11 @@ def test_harbor_run_env_file_secrets_reach_subprocess_not_command_line( config_path.write_text("task_name: unused\n") env_file = tmp_path / "secrets.env" env_file.write_text("MODAL_TOKEN_ID=mt-id\nMODAL_TOKEN_SECRET=mt-secret\n") - config = SimpleNamespace(harbor_requirement="harbor[modal]==0.20.0", agent_env={}) + config = SimpleNamespace( + harbor_requirement="harbor[modal]==0.20.0", + agent_env={}, + optimizer_harbor_args=[], + ) observed = {} def compile_task(_config, output):