diff --git a/tests/trace_server/test_environment.py b/tests/trace_server/test_environment.py index 9d75447c597f..a78696ba7b78 100644 --- a/tests/trace_server/test_environment.py +++ b/tests/trace_server/test_environment.py @@ -6,6 +6,7 @@ DEFAULT_REMOTE_SCORER_HTTP_TIMEOUT_SECONDS, REMOTE_SCORER_ALLOW_INSECURE_HTTP_ENV, REMOTE_SCORER_ALLOWED_HOSTS_ENV, + REMOTE_SCORER_REQUIRE_STRUCTURED_RESULT_SCHEMA_ENV, REMOTE_SCORER_VALIDATE_HOSTS_ENV, VALID_CALLS_SHARD_KEYS, kafka_producer_max_buffer_size, @@ -19,6 +20,7 @@ wf_scoring_worker_remote_scorer_bearer_token, wf_scoring_worker_remote_scorer_enabled, wf_scoring_worker_remote_scorer_http_timeout_seconds, + wf_scoring_worker_remote_scorer_require_structured_result_schema, wf_scoring_worker_remote_scorer_validate_hosts, ) @@ -269,3 +271,25 @@ def test_wf_scoring_worker_remote_scorer_allow_insecure_http(monkeypatch): assert wf_scoring_worker_remote_scorer_allow_insecure_http() is False monkeypatch.setenv(key, "1") assert wf_scoring_worker_remote_scorer_allow_insecure_http() is False + + +@pytest.mark.disable_logging_error_check +def test_wf_scoring_worker_remote_scorer_require_structured_result_schema( + monkeypatch, +): + """Structured result schema enforcement is enabled by default; only false disables it.""" + key = REMOTE_SCORER_REQUIRE_STRUCTURED_RESULT_SCHEMA_ENV + monkeypatch.delenv(key, raising=False) + assert wf_scoring_worker_remote_scorer_require_structured_result_schema() is True + + monkeypatch.setenv(key, "false") + assert wf_scoring_worker_remote_scorer_require_structured_result_schema() is False + monkeypatch.setenv(key, "False") + assert wf_scoring_worker_remote_scorer_require_structured_result_schema() is False + + monkeypatch.setenv(key, "true") + assert wf_scoring_worker_remote_scorer_require_structured_result_schema() is True + monkeypatch.setenv(key, "") + assert wf_scoring_worker_remote_scorer_require_structured_result_schema() is True + monkeypatch.setenv(key, "0") + assert wf_scoring_worker_remote_scorer_require_structured_result_schema() is True diff --git a/weave/trace_server/environment.py b/weave/trace_server/environment.py index 24fd9c6562e2..23377dff84b1 100644 --- a/weave/trace_server/environment.py +++ b/weave/trace_server/environment.py @@ -159,6 +159,9 @@ def wf_scoring_worker_remote_scorer_bearer_token() -> str | None: REMOTE_SCORER_ALLOW_INSECURE_HTTP_ENV = ( "WF_SCORING_WORKER_REMOTE_SCORER_ALLOW_INSECURE_HTTP" ) +REMOTE_SCORER_REQUIRE_STRUCTURED_RESULT_SCHEMA_ENV = ( + "WF_SCORING_WORKER_REMOTE_SCORER_REQUIRE_STRUCTURED_RESULT_SCHEMA" +) def wf_scoring_worker_remote_scorer_enabled() -> bool: @@ -223,6 +226,21 @@ def wf_scoring_worker_remote_scorer_allow_insecure_http() -> bool: ) +def wf_scoring_worker_remote_scorer_require_structured_result_schema() -> bool: + """Whether remote scorer results must match the structured scorer schema. + + Defaults to true to nudge new RemoteScorer users onto the typed scorer + result shape. Set to ``false`` only for controlled compatibility testing or + emergency operational bypasses. + """ + return ( + os.environ.get( + REMOTE_SCORER_REQUIRE_STRUCTURED_RESULT_SCHEMA_ENV, "true" + ).lower() + != "false" + ) + + # Clickhouse Settings