From 6091efddb74c834415d40e19cf8c413feeb65877 Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 25 Jul 2026 08:45:24 +0300 Subject: [PATCH] fix: repair a scheme-less WANDB_BASE_URL and stop losing sink failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every harbor benchmark run has reported nothing to the self-hosted W&B: `shehab-yasser` listed zero projects, and each run's exported `artifacts/wandb/state.json` showed a run_id minted with `evaluation_ids: []`, `next_step: 0`, `request_log_files: {}` — even for a session that completed 11 evaluations. That fingerprint is exactly what `SidecarWandbSink.__init__` leaves when `wandb.init()` raises: state is written (wandb.py:225) before the run is opened (wandb.py:230), and `deployment.py` catches anything from the constructor and continues without W&B. Reproduced locally against scaleai.wandb.io, $0, deterministic: WANDB_BASE_URL=scaleai.wandb.io pydantic_core.ValidationError: 1 validation error for Settings base_url Input should be a valid URL, relative URL without a base [type=url_parsing, input_value='scaleai.wandb.io'] W&B parses `base_url` as a URL, so the natural way to write a self-hosted host silently costs the run all of its reporting. With `https://` prepended, the same script logs metrics, uploads an artifact and auto-creates the project. Egress was never the problem. Two changes: - `normalize_wandb_base_url()` prepends `https://` to a scheme-less `WANDB_BASE_URL` (and warns), called before both `wandb.init()` sites. Already-qualified values, including plain `http://localhost`, pass through. - The swallowed init failure now also lands in `session/artifacts/wandb/init-error.json`. The existing `logger.warning` goes to the sidecar container's stderr, which no run artifact captures, so a disabled sink was indistinguishable from a healthy run that logged nothing. Observability still never takes the eval path down. Refs #52. Co-Authored-By: Claude Opus 5 (1M context) --- vero/src/vero/harbor/deployment.py | 18 +++++- vero/src/vero/runtime/wandb.py | 28 +++++++++ vero/tests/test_v05_harbor_deployment.py | 73 ++++++++++++++++++++++++ vero/tests/test_v05_wandb.py | 29 ++++++++++ 4 files changed, 147 insertions(+), 1 deletion(-) diff --git a/vero/src/vero/harbor/deployment.py b/vero/src/vero/harbor/deployment.py index e77aaf81..0bf66710 100644 --- a/vero/src/vero/harbor/deployment.py +++ b/vero/src/vero/harbor/deployment.py @@ -25,6 +25,7 @@ from vero.evaluation.engine import EvaluationEngine from vero.harbor.backend import HarborBackend, HarborBackendConfig from vero.models import StrictModel +from vero.runtime.artifacts import ArtifactStore from vero.sandbox import LocalSandbox from vero.sidecar.serve import SidecarComponents from vero.sidecar.session import initialize_harbor_session_manifest @@ -323,13 +324,28 @@ async def build_harbor_components(config: dict) -> SidecarComponents: log_traces=parsed.wandb.log_traces, ) engine.listeners.append(wandb_sink) - except Exception: + except Exception as error: logger.warning( "W&B reporting disabled: the sidecar sink could not be " "initialized; the evaluation sidecar continues without it", exc_info=True, ) wandb_sink = None + # The warning above goes to the sidecar container's stderr, which no + # run artifact captures, so a silently disabled sink looks exactly + # like a healthy run that logged nothing. Record why in the session + # artifacts, which are archived into the exported run record. + try: + ArtifactStore(session_dir / "artifacts").write_json( + "wandb/init-error.json", + { + "project": parsed.wandb.project, + "error_type": type(error).__name__, + "error": str(error), + }, + ) + except OSError: + logger.warning("could not record the W&B init failure", exc_info=True) usage_path = ( Path(parsed.inference_usage_path) diff --git a/vero/src/vero/runtime/wandb.py b/vero/src/vero/runtime/wandb.py index e467e3f3..c9fc4094 100644 --- a/vero/src/vero/runtime/wandb.py +++ b/vero/src/vero/runtime/wandb.py @@ -6,6 +6,7 @@ import hashlib import json import logging +import os import tempfile from pathlib import Path from typing import Any @@ -19,6 +20,31 @@ logger = logging.getLogger(__name__) +def normalize_wandb_base_url(environment: dict[str, str] | None = None) -> str | None: + """Give a scheme-less ``WANDB_BASE_URL`` the ``https://`` it needs. + + A self-hosted host is naturally written ``scaleai.wandb.io``, but W&B's + settings model parses ``base_url`` as a URL and rejects that with + ``Input should be a valid URL, relative URL without a base``. The error + surfaces out of ``wandb.init()``, which callers treat as "W&B unavailable", + so one missing scheme silently costs a run all of its reporting. + + Returns the value in effect, or None when the variable is unset. + """ + environ = os.environ if environment is None else environment + value = (environ.get("WANDB_BASE_URL") or "").strip() + if not value or "://" in value: + return value or None + corrected = f"https://{value}" + environ["WANDB_BASE_URL"] = corrected + logger.warning( + "WANDB_BASE_URL %r has no scheme and would be rejected by W&B; using %r", + value, + corrected, + ) + return corrected + + def _open_wandb_run( *, project: str, @@ -43,6 +69,7 @@ def _open_wandb_run( raise RuntimeError( "W&B reporting requires `pip install scale-vero[wandb]`" ) from error + normalize_wandb_base_url() wandb_dir.mkdir(parents=True, exist_ok=True) stable_id = run_id or ("vero-" + hashlib.sha256(session_id.encode()).hexdigest()[:16]) init_kwargs: dict[str, Any] = { @@ -96,6 +123,7 @@ def __init__( "W&B reporting requires `pip install scale-vero[wandb]`" ) from error + normalize_wandb_base_url() wandb_dir = session_dir / "artifacts" / "wandb" wandb_dir.mkdir(parents=True, exist_ok=True) self.artifacts = ArtifactStore(session_dir / "artifacts") diff --git a/vero/tests/test_v05_harbor_deployment.py b/vero/tests/test_v05_harbor_deployment.py index da61367c..25c0a4b2 100644 --- a/vero/tests/test_v05_harbor_deployment.py +++ b/vero/tests/test_v05_harbor_deployment.py @@ -270,3 +270,76 @@ class Result: assert ( exported / "requests/requests-00001.jsonl" ).read_text() == '{"scope":"producer"}\n' + + +@pytest.mark.asyncio +async def test_wandb_init_failure_is_recorded_in_the_session_artifacts(tmp_path): + # A sink that cannot start only logs to the sidecar container's stderr, which + # no run artifact captures, so a disabled sink is indistinguishable from a + # healthy run that logged nothing. Leave the reason behind in the session. + trusted = tmp_path / "trusted" + agent = tmp_path / "agent" + _repo(trusted, "VALUE = 1\n") + _repo(agent, "VALUE = 1\n") + cases = tmp_path / "cases.json" + cases.write_text('[{"id":"task","task_name":"org/task"}]') + evaluation_set = EvaluationSet(name="benchmark") + objective = ObjectiveSpec( + selector=MetricSelector(metric="score"), + direction="maximize", + ) + session_dir = tmp_path / "state/session" + config = { + "repo_path": str(trusted), + "agent_repo_path": str(agent), + "session_dir": str(session_dir), + "backends": { + "backend": { + "task_source": "org/benchmark@1.0", + "agent_import_path": "program:Agent", + "cases_path": str(cases), + "harbor_requirement": "harbor==0.1.17", + "uv_executable": sys.executable, + } + }, + "access_policies": [], + "budgets": [], + "selection": { + "mode": "auto_best", + "backend_id": "backend", + "evaluation_set": evaluation_set.model_dump(mode="json"), + "objective": objective.model_dump(mode="json"), + "baseline_version": "HEAD", + }, + "targets": [ + { + "reward_key": "reward", + "backend_id": "backend", + "evaluation_set": evaluation_set.model_dump(mode="json"), + "objective": objective.model_dump(mode="json"), + } + ], + "admin_volume": str(tmp_path / "state/admin"), + "wandb": {"project": "vero-tests"}, + } + + import vero.runtime.wandb as wandb_module + + def _explode(**kwargs): + raise ValueError("Input should be a valid URL, relative URL without a base") + + original = wandb_module.SidecarWandbSink + wandb_module.SidecarWandbSink = _explode + try: + components = await build_harbor_components(config) + finally: + wandb_module.SidecarWandbSink = original + + # The eval path survives: observability never takes the sidecar down. + assert components.telemetry is None + recorded = json.loads( + (session_dir / "artifacts/wandb/init-error.json").read_text() + ) + assert recorded["project"] == "vero-tests" + assert recorded["error_type"] == "ValueError" + assert "valid URL" in recorded["error"] diff --git a/vero/tests/test_v05_wandb.py b/vero/tests/test_v05_wandb.py index 963a59d1..2db9aa62 100644 --- a/vero/tests/test_v05_wandb.py +++ b/vero/tests/test_v05_wandb.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os from datetime import UTC, datetime, timedelta from pathlib import Path @@ -402,3 +403,31 @@ def test_sidecar_wandb_telemetry_is_best_effort(tmp_path: Path): poller.poll_once() # must not raise assert client.run.logged == [] assert client.run.artifacts == [] + + +def test_scheme_less_wandb_base_url_is_repaired_before_init(tmp_path: Path, monkeypatch): + """A self-hosted host written without a scheme must not silently kill reporting. + + W&B parses `base_url` as a URL, so `WANDB_BASE_URL=scaleai.wandb.io` raises + out of `wandb.init()` and the sidecar disables W&B for the whole run. + """ + from vero.runtime.wandb import normalize_wandb_base_url + + monkeypatch.setenv("WANDB_BASE_URL", "scaleai.wandb.io") + assert normalize_wandb_base_url() == "https://scaleai.wandb.io" + assert os.environ["WANDB_BASE_URL"] == "https://scaleai.wandb.io" + + # Already-qualified values, including plain http, are left exactly as given. + monkeypatch.setenv("WANDB_BASE_URL", "http://localhost:8080") + assert normalize_wandb_base_url() == "http://localhost:8080" + assert os.environ["WANDB_BASE_URL"] == "http://localhost:8080" + + monkeypatch.delenv("WANDB_BASE_URL", raising=False) + assert normalize_wandb_base_url() is None + + # And the repair happens before a sink opens its run. + monkeypatch.setenv("WANDB_BASE_URL", "scaleai.wandb.io") + SidecarWandbSink( + project="v", session_id="s", session_dir=tmp_path / "session", client=FakeWandb() + ) + assert os.environ["WANDB_BASE_URL"] == "https://scaleai.wandb.io"