diff --git a/harness-engineering-bench/tau3/baseline/build.yaml b/harness-engineering-bench/tau3/baseline/build.yaml index 8e07caa..beacdb4 100644 --- a/harness-engineering-bench/tau3/baseline/build.yaml +++ b/harness-engineering-bench/tau3/baseline/build.yaml @@ -49,6 +49,12 @@ model: fireworks_ai/deepseek-v4-flash environment_name: ${inner_env:-modal} # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +# NOTE: tau3's optimizer trial reliably dies at ~38-40 min with grpclib +# StreamTerminatedError ("Connection lost"). Four live experiments (default +# DinD, --ek modal_vm_runtime=true, --ek modal_sandbox_v2=true, and +# --max-retries 1) all failed identically, so no `optimizer_harbor_args` value +# is set here: none of them is a fix, and a config value that claims to be one +# is worse than nothing. Tracked upstream; see the PR that added this field. harbor_python_version: "3.12" n_attempts: 1 max_retries: 1 diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py index d6bc857..1463098 100644 --- a/vero/src/vero/harbor/build/config.py +++ b/vero/src/vero/harbor/build/config.py @@ -424,6 +424,10 @@ class HarborBuildConfig(StrictModel): task_environment, which is that sub-run's environment. Use it for things like UV_TOOL_BIN_DIR, so `uv tool install` targets a writable directory on a non-root sandbox. + optimizer_harbor_args: Extra flags for the outer harbor run that hosts + the optimizer trial. Distinct from extra_harbor_args, which tunes + the nested evaluation sub-run. Rejected if they override a flag + `vero harbor run` controls. timeout_seconds: Wall clock for one evaluation. case_timeout_seconds: Wall clock for one case. task_agent_timeout_seconds: Wall clock declared for the target agent. @@ -534,8 +538,16 @@ class HarborBuildConfig(StrictModel): feedback_transcripts: bool = False feedback_max_bytes: int = Field(default=3000, ge=0) expose_attempt_detail: bool = False + # Extra flags for the NESTED `harbor run` that evaluates a candidate. extra_harbor_args: list[str] = Field(default_factory=list) agent_env: dict[str, str] = Field(default_factory=dict) + # Extra flags for the OUTER `harbor run` that hosts the optimizer trial + # (`vero harbor run`). Distinct from `extra_harbor_args`: that one tunes the + # nested eval sub-run, this one tunes the environment the optimizer itself + # lives in. A build declares here what its optimizer trial needs to survive, + # e.g. `--ek modal_vm_runtime=true` for a long trial whose teardown keeps + # losing the DinD gRPC stream. + optimizer_harbor_args: list[str] = Field(default_factory=list) timeout_seconds: float = Field(default=1800.0, gt=0) case_timeout_seconds: float = Field(default=180.0, gt=0) @@ -651,6 +663,22 @@ def validate_extra_harbor_args(cls, value: list[str]) -> list[str]: ) return value + @field_validator("optimizer_harbor_args") + @classmethod + def validate_optimizer_harbor_args(cls, value: list[str]) -> list[str]: + # `vero harbor run` owns the outer command's task, agent, environment and + # model; a build must not fight the CLI over them. + controlled = {"-a", "-e", "-m", "-p"} + conflicts = [ + argument for argument in value if argument.split("=", 1)[0] in controlled + ] + if conflicts: + raise ValueError( + "optimizer_harbor_args override controlled flags: " + + ", ".join(conflicts) + ) + return value + @field_validator("partitions") @classmethod def validate_partitions( diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 8b70018..72730a6 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -437,6 +437,9 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) # for a deterministic command line. for key in sorted(config.agent_env): command.extend(["--ae", f"{key}={config.agent_env[key]}"]) + # Build-declared outer-trial flags first, so a command-line arg can still + # override them (harbor's `--ek` takes the last value for a key). + command.extend(config.optimizer_harbor_args) command.extend(extra) click.echo(shlex.join(command)) completed = subprocess.run( diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index d18534c..5d48c9a 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -309,3 +309,62 @@ def test_cli_rejects_options_that_do_not_apply(tmp_path: Path): assert "are only valid with --produce" in producer_timeout.output assert wandb.exit_code == 2 assert "require --wandb-project" in wandb.output + + +def test_harbor_run_forwards_build_declared_optimizer_args(tmp_path, monkeypatch): + """`optimizer_harbor_args` tunes the OUTER trial, `extra_harbor_args` the nested eval. + + Regression guard for the tau3 teardown failure: the build declared + `--ek modal_vm_runtime=true` and it has to reach the `harbor run` that hosts + the optimizer, not the `harbor run` that scores a candidate. + """ + from vero.harbor import build as harbor_build + from vero.harbor import cli as harbor_cli + + config_path = tmp_path / "build.yaml" + config_path.write_text("name: org/task\n", encoding="utf-8") + + class _Config: + harbor_requirement = "harbor[modal]==0.20.0" + agent_env: dict[str, str] = {} + optimizer_harbor_args = ["--ek", "modal_vm_runtime=true"] + extra_harbor_args = ["--ek", "app_name=nested-only"] + + monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: _Config()) + monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda config, output: output) + monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") + monkeypatch.setattr( + harbor_cli, "_compiled_run_environment", lambda task, overrides: {} + ) + + recorded: list[list[str]] = [] + + def _record(command, env=None): + recorded.append(command) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(harbor_cli.subprocess, "run", _record) + + result = CliRunner().invoke( + main, + [ + "harbor", + "run", + "--config", + str(config_path), + "--agent", + "codex", + "--model", + "gpt-5.3-codex", + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + assert len(recorded) == 1 + command = recorded[0] + assert "modal_vm_runtime=true" in command + # The nested-eval flags stay out of the outer command. + assert "app_name=nested-only" not in command + # Build-declared flags come first so a command-line `--ek` can override them. + assert command.index("modal_vm_runtime=true") < command.index("--yes") diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 43b9c61..c3770ee 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -391,6 +391,12 @@ def test_build_config_requires_pins_and_valid_partition_references(tmp_path): _config(tmp_path / "unknown", selection_partition="missing") with pytest.raises(ValidationError, match="controlled flags"): _config(tmp_path / "flags", extra_harbor_args=["--jobs-dir=/forged"]) + with pytest.raises(ValidationError, match="controlled flags"): + _config(tmp_path / "outer-flags", optimizer_harbor_args=["-a", "forged"]) + assert _config( + tmp_path / "outer-ek", + optimizer_harbor_args=["--ek", "modal_vm_runtime=true"], + ).optimizer_harbor_args == ["--ek", "modal_vm_runtime=true"] with pytest.raises(ValidationError, match="explicit version"): _config(tmp_path / "source", task_source="org/unversioned") diff --git a/vero/tests/test_v05_harbor_http.py b/vero/tests/test_v05_harbor_http.py index a4e17a5..0cbc1ff 100644 --- a/vero/tests/test_v05_harbor_http.py +++ b/vero/tests/test_v05_harbor_http.py @@ -260,7 +260,11 @@ def test_harbor_run_uses_current_python_and_pinned_harbor_extra(tmp_path, monkey config_path = tmp_path / "build.yaml" config_path.write_text("task_name: unused\n") - config = SimpleNamespace(harbor_requirement="harbor[modal]==0.20.0", agent_env={}) + config = SimpleNamespace( + harbor_requirement="harbor[modal]==0.20.0", + agent_env={}, + optimizer_harbor_args=[], + ) observed = {} def compile_task(_config, output): @@ -313,7 +317,11 @@ def test_harbor_run_env_file_secrets_reach_subprocess_not_command_line( config_path.write_text("task_name: unused\n") env_file = tmp_path / "secrets.env" env_file.write_text("MODAL_TOKEN_ID=mt-id\nMODAL_TOKEN_SECRET=mt-secret\n") - config = SimpleNamespace(harbor_requirement="harbor[modal]==0.20.0", agent_env={}) + config = SimpleNamespace( + harbor_requirement="harbor[modal]==0.20.0", + agent_env={}, + optimizer_harbor_args=[], + ) observed = {} def compile_task(_config, output):