From 9457bce8b7008c1b686ec34934d123cf5bf6e849 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Fri, 29 May 2026 22:42:31 -0400 Subject: [PATCH 01/18] Add synthetic readers for lexical context (closes #497) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Templates collect synthetic read-only Tools for non-Tool symbols in their lexical scope, alongside the existing real-Tool collection. The LLM can call these readers to inspect lexical state on demand instead of having the entire scope dumped into the system prompt. Two reader flavors via singledispatch on the value's type: - Definition-readers for classes and functions return text via pydoc.render_doc (level="short", default — byte-equivalent to help(obj)) or inspect.getsource (level="full"). They bypass Encodable and just return str. - Value-readers for everything else return the live value, encoded through the existing Encodable pipeline. Probe is TypeAdapter(Encodable[T]).json_schema(); on any failure (Pydantic schema error, unencodable types like Term/Operation/TypeVar) the symbol is silently skipped. _collect_synthetic_readers is wired in two places: call_assistant (sees template.__context__ + bound args, mirroring Python call semantics) and Template.tools (sees template.__context__ only). Real Tools collected by _collect_tools take precedence — synthetic readers fill the gap. A short static preface sentence is appended to Template.__system_prompt__ so the LLM knows the read-only-readers category exists. The structured tools array carries per-tool semantics; the preface does not enumerate them. Two existing assertions in test_handlers_llm_template.py flip from 'local_variable not in a.f.tools' to 'in', reflecting the new behavior. 19 new unit tests cover the singledispatch matrix, the probe contract, live-read semantics, the BaseModel-via-metaclass dispatch case, the Box-via-TypeError-chain skip path, and the system-prompt preface. One recorded-fixture integration test exercises the end-to-end LLM-reads-lexical-value path. hide=/expose= knob deferred to a follow-up. --- effectful/handlers/llm/completions.py | 115 ++++++- effectful/handlers/llm/template.py | 27 +- ...gration__test_llm_reads_lexical_value.json | 55 ++++ ...ation__test_llm_reads_lexical_value_1.json | 55 ++++ ...ation__test_llm_reads_lexical_value_2.json | 55 ++++ ...ation__test_llm_reads_lexical_value_3.json | 55 ++++ ...ation__test_llm_reads_lexical_value_4.json | 55 ++++ ...ation__test_llm_reads_lexical_value_5.json | 46 +++ tests/test_handlers_llm_provider.py | 33 ++ tests/test_handlers_llm_template.py | 309 +++++++++++++++++- 10 files changed, 796 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 2e169d932..b839e3e46 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -5,9 +5,11 @@ import functools import inspect import json +import pydoc import string import textwrap import traceback +import types import typing import uuid @@ -209,6 +211,112 @@ def _collect_tools( return result +def _build_definition_reader( + value: typing.Any, name: str, env: collections.abc.Mapping[str, typing.Any] +) -> Tool | None: + """Build a Tool that returns the source / help() output of a class or + function. Probe is `inspect.getsource` reachability; symbols whose + source is unreachable (builtin C, REPL lambdas, etc.) are skipped.""" + try: + inspect.getsource(value) + except (OSError, TypeError): + return None + + kind = "class" if inspect.isclass(value) else "function" + + def body(level: typing.Literal["short", "full"] = "short") -> str: + obj = env[name] + if level == "short": + return pydoc.render_doc(obj, renderer=pydoc.plaintext) + return inspect.getsource(obj) + + body.__name__ = name + body.__doc__ = ( + f"Read the definition of `{name}` (a {kind}). " + f'`level="short"` (default) returns `help({name})` output; ' + f'`level="full"` returns `inspect.getsource({name})`. ' + f"Calls must include `level` explicitly under OpenAI strict mode." + ) + body.__annotations__ = { + "level": typing.Literal["short", "full"], + "return": str, + } + return Tool.define(body) + + +@functools.singledispatch +def _build_synthetic_reader( + value: typing.Any, + name: str, + env: collections.abc.Mapping[str, typing.Any], +) -> Tool | None: + """Build a synthetic read-only Tool for a lexical symbol. + + Default branch: value-reader. Probe is the §2c Encodable schema check; + on failure (unencodable value), return None and let the symbol be + skipped silently. + + Registrations route classes and functions through the + definition-reader path (`_build_definition_reader`), and route Tool, + Agent, and module values to a no-op (already collected by + `_collect_tools` or intentionally excluded). + """ + try: + inferred = nested_type(value).value + adapter = pydantic.TypeAdapter(Encodable[inferred]) + adapter.json_schema() + except Exception: + # The probe chains through several third-party libraries + # (nested_type → inspect.signature → typing.get_overloads → + # Pydantic schema generation). Any failure means "this symbol + # cannot be exposed as a synthetic reader" — the contract is + # skip-on-probe-failure, so we catch broadly. The cost of a + # too-narrow catch is a crash mid-call_assistant; the cost of a + # too-broad catch is silently skipping a symbol that might have + # worked. + return None + + def body(): + return env[name] + + body.__name__ = name + body.__doc__ = f"Read the value of lexical variable `{name}` (type `{inferred}`)." + body.__annotations__ = {"return": inferred} + return Tool.define(body) + + +_build_synthetic_reader.register(type, _build_definition_reader) +_build_synthetic_reader.register(types.FunctionType, _build_definition_reader) +_build_synthetic_reader.register(types.BuiltinFunctionType, _build_definition_reader) +_build_synthetic_reader.register(types.MethodType, _build_definition_reader) + + +@_build_synthetic_reader.register(types.ModuleType) +@_build_synthetic_reader.register(Tool) +@_build_synthetic_reader.register(Agent) +def _no_synthetic_reader(value, name, env): + return None + + +def _collect_synthetic_readers( + env: collections.abc.Mapping[str, typing.Any], + already_collected: collections.abc.Set[str], +) -> collections.abc.Mapping[str, Tool]: + """Synthetic readers for lexical symbols not already covered by + `_collect_tools`. Skips dunder names and any name already in + `already_collected`.""" + result: dict[str, Tool] = {} + for name, obj in env.items(): + if name in already_collected: + continue + if name.startswith("__"): + continue + tool = _build_synthetic_reader(obj, name, env) + if tool is not None: + result[name] = tool + return result + + @Operation.define @functools.wraps(litellm.completion) def completion(*args, **kwargs) -> typing.Any: @@ -243,7 +351,12 @@ def call_assistant[T]( ResultDecodingError: If the result cannot be decoded. The error includes the raw assistant message for retry handling. """ - tools = _collect_tools(env) + tools = dict(_collect_tools(env)) + # Add synthetic readers for non-Tool lexical symbols. These mirror the + # bound-args layer that LiteLLMProvider._call adds to env, so the LLM + # sees readers for the Template's arguments alongside other lexical + # context. + tools.update(_collect_synthetic_readers(env, set(tools))) tool_specs = { k: typing.cast( pydantic.TypeAdapter[typing.Any], diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index f56d6fad7..8b2a96194 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -11,6 +11,12 @@ from effectful.ops.types import Annotation, Operation +_LEXICAL_READERS_PREFACE = ( + "You also have access to read-only tools for inspecting the lexical " + "scope where this template is defined. Their names match variable " + "names from that scope; calling one returns the current value." +) + class _IsRecursiveAnnotation(Annotation): """ @@ -215,15 +221,19 @@ def __prompt_template__(self) -> str: @property def tools(self) -> Mapping[str, Tool]: - """Operations and Templates available as tools. Auto-capture from lexical context.""" - from effectful.handlers.llm.completions import _collect_tools + """Operations and Templates available as tools, plus synthetic + readers for other lexical symbols. Auto-captured from lexical context.""" + from effectful.handlers.llm.completions import ( + _collect_synthetic_readers, + _collect_tools, + ) - result = _collect_tools(self.__context__) + result = dict(_collect_tools(self.__context__)) + result.update(_collect_synthetic_readers(self.__context__, set(result))) # We remove the template itself from the tool map unless it is explicitly # marked as recursive (see test_template_method, test_template_method_nested_class). if not _is_recursive_signature(self.__signature__): - result = dict(result) # copy to allow mutation for name, tool in tuple(result.items()): if tool is self: del result[name] @@ -312,7 +322,14 @@ def define[**Q, V]( op = super().define(default, *args, **kwargs) op.__context__ = context # type: ignore[attr-defined] mod = inspect.getmodule(_fn) - op.__system_prompt__ = inspect.getdoc(mod) if mod is not None else "" # type: ignore[attr-defined] + op.__system_prompt__ = "\n\n".join( # type: ignore[attr-defined] + part + for part in ( + inspect.getdoc(mod) if mod is not None else None, + _LEXICAL_READERS_PREFACE, + ) + if part + ) # Keep validation on original define-time callables, but skip the bound wrapper path. # to avoid dropping `self` from the signature and falsely rejecting valid prompt fields like `{self.name}`. is_bound_wrapper = ( diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json new file mode 100644 index 000000000..403d200f5 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-Dl3k0fEKWeTJpZDy3rcqAKPKE46Cu", + "created": 1780108168, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_df8c8d3b43", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{}", + "name": "_known_data" + }, + "id": "call_DoVBVCnKiU5k7FTzWRo5nnzm", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 11, + "prompt_tokens": 6043, + "total_tokens": 6054, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json new file mode 100644 index 000000000..b6e7666a3 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-Dl3k2MUxG3A1S3HrmgwMXFmDNScCt", + "created": 1780108170, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_df8c8d3b43", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":10,\"b\":20}", + "name": "add_numbers" + }, + "id": "call_vypFiLTjH5MEpMKSL3VLoD4z", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 6077, + "total_tokens": 6095, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 6016, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json new file mode 100644 index 000000000..5683bba8f --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-Dl3k5bfkN00lZh8L5D82Cr9YhOJsA", + "created": 1780108173, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_df8c8d3b43", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":30,\"b\":30}", + "name": "add_numbers" + }, + "id": "call_EmO3tVtqZBqdL6K4dUqLmfgn", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 6104, + "total_tokens": 6122, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 6016, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json new file mode 100644 index 000000000..a1c38794e --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-Dl3k7gKpQZx4an1oAXAhdYViG1NjR", + "created": 1780108175, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_df8c8d3b43", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":60,\"b\":40}", + "name": "add_numbers" + }, + "id": "call_WhTflV4rMdUJUJOY2qqOHv7x", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 6131, + "total_tokens": 6149, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 6016, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json new file mode 100644 index 000000000..08c5bc100 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-Dl3k86CG3iPs5P7110XgQ3Eso2CfZ", + "created": 1780108176, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_df8c8d3b43", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":100,\"b\":50}", + "name": "add_numbers" + }, + "id": "call_LTy1WHCfWTFnwZ5GSAUElfDA", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 6158, + "total_tokens": 6176, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 6016, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json new file mode 100644 index 000000000..e1fe0b3c4 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json @@ -0,0 +1,46 @@ +{ + "id": "chatcmpl-Dl3k9lFSuyqAQbfM8J1H3BQmgYopC", + "created": 1780108177, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_df8c8d3b43", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"value\":150}", + "role": "assistant", + "tool_calls": null, + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 11, + "prompt_tokens": 6185, + "total_tokens": 6196, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 6144, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index b56fd7bbd..a994645d2 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -2158,3 +2158,36 @@ def _completion(self, model, messages=None, **kwargs): assert messages[0]["role"] == "system", ( "System message should be the first message in history" ) + + +# --------------------------------------------------------------------------- +# Synthetic readers — integration (PR #545 finish-up) +# --------------------------------------------------------------------------- + + +# Module-level binding the LLM will be asked to inspect via a synthetic +# reader. The reader's name in the tool list is `_known_data`. +_known_data = [10, 20, 30, 40, 50] + + +class TestSyntheticReaderIntegration: + """The LLM can read lexical context through synthetic reader tools.""" + + @requires_llm + def test_llm_reads_lexical_value(self, request): + """Template asks LLM to inspect _known_data and report its sum. + The synthetic reader for _known_data is available in the tools + array; the LLM should call it, see [10,20,30,40,50], and report + the sum (150).""" + + @Template.define + def report_sum() -> int: + """Use the `_known_data` tool to read the list of numbers, + then return their sum as an integer.""" + raise NotImplementedError + + with handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)): + result = report_sum() + + assert isinstance(result, int) + assert result == sum(_known_data) # 150 diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index d17723110..2b9dfae74 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -778,7 +778,8 @@ def f(self) -> int: assert a.random in a.f.tools.values() # f is the template itself — found via self but correctly removed (non-recursive) assert a.f not in a.f.tools.values() - assert "local_variable" in a.f.__context__ and "local_variable" not in a.f.tools + # local_variable is now exposed as a synthetic reader (PR #545 finish-up). + assert "local_variable" in a.f.__context__ and "local_variable" in a.f.tools assert any(t() == 4 for t in a.f.tools.values() if t is a.random) class B(A): @@ -795,7 +796,8 @@ def reverse(self, s: str) -> str: assert isinstance(b.f, Template) assert b.random in b.f.tools.values() assert b.reverse in b.f.tools.values() - assert "local_variable" in b.f.__context__ and "local_variable" not in a.f.tools + # local_variable is now exposed as a synthetic reader (PR #545 finish-up). + assert "local_variable" in b.f.__context__ and "local_variable" in a.f.tools def test_template_method_nested_class(): @@ -826,7 +828,8 @@ def f(self) -> int: assert "random" in a.f.tools # f is the template itself — found via self but correctly removed (non-recursive) assert "f" not in a.f.tools - assert "local_variable" in a.f.__context__ and "local_variable" not in a.f.tools + # local_variable is now exposed as a synthetic reader (PR #545 finish-up). + assert "local_variable" in a.f.__context__ and "local_variable" in a.f.tools assert a.f.tools["random"]() == 4 @@ -1531,3 +1534,303 @@ def test_tool_forward_ref(): sig = inspect.signature(_tool_forward_ref) assert sig.parameters["x"].annotation is int assert sig.return_annotation is str + + +# --------------------------------------------------------------------------- +# Synthetic readers for lexical context (PR #545 finish-up) +# --------------------------------------------------------------------------- + +import os +import re +import typing +from pathlib import Path + +import pydantic + +from effectful.handlers.llm.completions import ( + _build_synthetic_reader, + _collect_synthetic_readers, + _collect_tools, +) +from effectful.handlers.llm.template import _LEXICAL_READERS_PREFACE + + +# Helpers for the test matrix +@dataclasses.dataclass +class _SimpleDataclass: + x: int + y: str + + +class _SimpleModel(pydantic.BaseModel): + """Pydantic model used in encodable-probe matrix tests.""" + + x: int + y: str + + +class _OpaqueNoEncoder: + """Plain user class with no Pydantic encoder.""" + + pass + + +def test_synthetic_reader_value_reader_returns_live_value(): + """A value-reader returns whatever env[name] currently is. Mutation + after reader construction propagates.""" + env = {"x": [1, 2, 3]} + tool = _build_synthetic_reader([1, 2, 3], "x", env) + assert tool is not None + assert tool() == [1, 2, 3] + env["x"].append(4) + assert tool() == [1, 2, 3, 4] + + +def test_synthetic_reader_value_reader_rebind(): + """A reader returns the current binding even after rebind. The + annotation is set at .tools-access time but the body reads live.""" + env: dict = {"x": 42} + tool = _build_synthetic_reader(42, "x", env) + assert tool is not None + assert tool() == 42 + env["x"] = 99 + assert tool() == 99 + + +def test_synthetic_reader_skips_when_name_deleted(): + """If env[name] is deleted between collection and call, the reader + raises KeyError on direct invocation. call_tool's ToolCallExecutionError + wrapping is the standard tool-runtime-error contract, tested elsewhere.""" + env = {"x": 42} + tool = _build_synthetic_reader(42, "x", env) + assert tool is not None + del env["x"] + with pytest.raises(KeyError): + tool() + + +@pytest.mark.parametrize( + "name,value,should_have_tool", + [ + ("primitive_int", 42, True), + ("primitive_str", "hello", True), + ("list_of_int", [1, 2, 3], True), + ("dict", {"a": 1}, True), + ("dataclass_simple", _SimpleDataclass(x=1, y="hello"), True), + ("pydantic_model", _SimpleModel(x=1, y="hello"), True), + # The plan's matrix says re.Pattern and pathlib.PosixPath instances + # ARE exposed by Encodable. Pin them here as a regression guard. + ("re_pattern", re.compile(r"x"), True), + ("pathlib_path", Path("/tmp"), True), + # Unencodable categories: probe rejects. + ("opaque", _OpaqueNoEncoder(), False), + ("typevar", typing.TypeVar("T"), False), + ], +) +def test_synthetic_reader_probe_matches_encodable(name, value, should_have_tool): + """The §2c probe accepts iff the symbol can produce a Pydantic + schema. No false positives, no false negatives.""" + env = {name: value} + tool = _build_synthetic_reader(value, name, env) + if should_have_tool: + assert tool is not None + else: + assert tool is None + + +def test_synthetic_readers_yield_to_real_tools(): + """Real Tools/Templates collected by `_collect_tools` take precedence + over same-named synthetic readers (which are skipped via + `already_collected`).""" + + @Tool.define + def shared() -> int: + """Doc.""" + return 1 + + env = collections.ChainMap({"shared": shared, "shared_value": 42}) + real = _collect_tools(env) + synth = _collect_synthetic_readers(env, set(real)) + assert "shared" not in synth, "real tool name should not be re-wrapped" + assert "shared_value" in synth + + +def test_synthetic_reader_annotation_has_no_free_typevars(): + """Polymorphic-Template post-#668 substitutes TypeVars in Tool + annotations. Synthetic reader annotations come from nested_type, + which produces concrete types — substitution is a no-op.""" + from effectful.internals.unification import freetypevars + + env = {"x": [1, 2, 3]} + tool = _build_synthetic_reader([1, 2, 3], "x", env) + assert tool is not None + sig = inspect.signature(tool) + assert freetypevars(sig.return_annotation) == set() + + +# ---- Definition-readers ---- + + +def test_definition_reader_short_form_substring_properties(): + """Short form (pydoc.render_doc output) contains the docstring, + method signatures, and method docstrings. Substring assertions + only — pydoc output format is not a stable API.""" + + class _ReaderTarget: + """Class docstring.""" + + def method(self, x: int) -> str: + """Method docstring.""" + return str(x) + + env = {"_ReaderTarget": _ReaderTarget} + tool = _build_synthetic_reader(_ReaderTarget, "_ReaderTarget", env) + assert tool is not None + + short = tool(level="short") + assert "Class docstring." in short + assert "method(self, x: int) -> str" in short + assert "Method docstring." in short + + +def test_definition_reader_full_form_routes_to_getsource_not_pydoc(): + """level="full" must route to inspect.getsource, not pydoc. Two + independent properties distinguish them: method bodies appear only + in getsource output, and the 'Methods defined here:' header is + pydoc-specific.""" + + class _ReaderTarget: + """Doc.""" + + def method(self, x: int) -> str: + return str(x) + + env = {"_ReaderTarget": _ReaderTarget} + tool = _build_synthetic_reader(_ReaderTarget, "_ReaderTarget", env) + assert tool is not None + + full = tool(level="full") + assert "return str(x)" in full, ( + "method body must be present (only getsource shows bodies)" + ) + assert "Methods defined here:" not in full, "pydoc-specific header must be absent" + + +def test_definition_reader_lives_through_env(): + """Rebinding the class in env shows the new definition on the next + call. Matches §1 live semantics.""" + + class _Old: + """Old class.""" + + class _New: + """New class.""" + + env: dict = {"x": _Old} + tool = _build_synthetic_reader(_Old, "x", env) + assert tool is not None + assert "Old class." in tool(level="short") + + env["x"] = _New + assert "New class." in tool(level="short") + + +def test_definition_reader_function(): + """A user-defined function gets a definition-reader. Its short form + contains the function signature and docstring; full form contains + the function body.""" + + def _example(x: int) -> int: + """Add one.""" + return x + 1 + + env = {"_example": _example} + tool = _build_synthetic_reader(_example, "_example", env) + assert tool is not None + short = tool(level="short") + assert "_example(x: int) -> int" in short + assert "Add one." in short + full = tool(level="full") + assert "return x + 1" in full + + +def test_definition_reader_baseclass_uses_metaclass_dispatch(): + """Singledispatch on `type` matches Pydantic BaseModel subclasses + (whose metaclass is ModelMetaclass, a subclass of type). The plan + relies on this — verify.""" + + class BMSubclass(pydantic.BaseModel): + """BM doc.""" + + x: int + + env = {"BMSubclass": BMSubclass} + tool = _build_synthetic_reader(BMSubclass, "BMSubclass", env) + assert tool is not None + short = tool(level="short") + assert "BM doc." in short + + +def test_definition_reader_skips_when_source_unreachable(): + """Builtin C types like `int` fail inspect.getsource with TypeError; + we skip them at probe time rather than crashing at call time.""" + env = {"int": int} + assert _build_synthetic_reader(int, "int", env) is None + + +def test_typevars_are_skipped(): + """TypeVar instances are not classes or routines; they fall through + to the value-reader default branch, fail the §2c probe with + PydanticSchemaGenerationError, and are skipped.""" + T = typing.TypeVar("T") + env = {"T": T} + assert _build_synthetic_reader(T, "T", env) is None + + +def test_module_values_are_skipped(): + """Modules are never wrapped as synthetic readers — registered to + return None at dispatch time.""" + env = {"os": os} + assert _build_synthetic_reader(os, "os", env) is None + + +def test_box_value_filtered_via_probe_typeerror_chain(): + """Box-as-value is skipped via the probe TypeError branch. Chain: + + nested_type(Box(42)) → Box(42) (passthrough; unification.py:952) + .value → 42 (Box's only field) + Encodable[42] → TypeError (literal value, not a type) + §2c probe catches → _build_synthetic_reader returns None + + Named for the contract (chain-as-contract). If nested_type later + handles Box more usefully, this test breaks and we revisit.""" + from effectful.internals.unification import Box + + env = {"b": Box(42)} + assert _build_synthetic_reader(Box(42), "b", env) is None + + +def test_system_prompt_contains_preface(): + """Template.__system_prompt__ includes the lexical-readers preface + sentence unconditionally.""" + + @Template.define + def with_preface(x: int) -> int: + """Doc.""" + raise NotHandled + + assert _LEXICAL_READERS_PREFACE in with_preface.__system_prompt__ + + +def test_template_tools_includes_synthetic_readers_for_locals(): + """The Template.tools property includes synthetic readers for + plain values in lexical scope (e.g., test-local variables).""" + _test_data = [10, 20, 30] + + @Template.define + def t() -> int: + """Doc.""" + raise NotHandled + + assert "_test_data" in t.tools + assert t.tools["_test_data"]() == [10, 20, 30] From 76d86a6124be31a5e82cdfaa9cd2673f0f84da9b Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sat, 30 May 2026 08:59:11 -0400 Subject: [PATCH 02/18] Fix mypy types in synthetic-reader probe --- effectful/handlers/llm/completions.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index b839e3e46..9fb0660e7 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -227,7 +227,7 @@ def _build_definition_reader( def body(level: typing.Literal["short", "full"] = "short") -> str: obj = env[name] if level == "short": - return pydoc.render_doc(obj, renderer=pydoc.plaintext) + return pydoc.render_doc(obj, renderer=pydoc.plaintext) # type: ignore[attr-defined] return inspect.getsource(obj) body.__name__ = name @@ -262,18 +262,19 @@ def _build_synthetic_reader( `_collect_tools` or intentionally excluded). """ try: - inferred = nested_type(value).value - adapter = pydantic.TypeAdapter(Encodable[inferred]) + inferred: typing.Any = nested_type(value).value + adapter: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( + Encodable[inferred] + ) adapter.json_schema() except Exception: # The probe chains through several third-party libraries - # (nested_type → inspect.signature → typing.get_overloads → + # (nested_type, inspect.signature, typing.get_overloads, # Pydantic schema generation). Any failure means "this symbol - # cannot be exposed as a synthetic reader" — the contract is - # skip-on-probe-failure, so we catch broadly. The cost of a - # too-narrow catch is a crash mid-call_assistant; the cost of a - # too-broad catch is silently skipping a symbol that might have - # worked. + # cannot be exposed as a synthetic reader", so we catch broadly. + # The cost of a too-narrow catch is a crash mid-call_assistant; + # the cost of a too-broad catch is silently skipping a symbol + # that might have worked. return None def body(): From 368e360f42060c7ed2f357c75d556fbf01d02840 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sat, 30 May 2026 10:51:45 -0400 Subject: [PATCH 03/18] Constrain test_tool_calling prompt against reader exploration Adds an explicit instruction to generate_good_poem to ignore any read-only lexical reader tools that may appear in the tool list. With synthetic readers now exposing module-level imports/classes as inspectable tools, the LLM was exploring those instead of finishing the task, exceeding max_calls=4. --- tests/test_handlers_llm_tool_calling_poem.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_handlers_llm_tool_calling_poem.py b/tests/test_handlers_llm_tool_calling_poem.py index 9ff8587f6..c1227cf70 100644 --- a/tests/test_handlers_llm_tool_calling_poem.py +++ b/tests/test_handlers_llm_tool_calling_poem.py @@ -92,7 +92,10 @@ def generate_good_poem(topic: str) -> Poem: Keep iterating until evaluate_poem_tool returns GOOD. Return your final poem as JSON with 'content' and 'form' fields. - Do not call the 'generate_good_poem' tool. + Do not call any tool other than evaluate_poem_tool. In particular, + do not call generate_good_poem and do not call any read-only lexical + reader that may appear in the tool list. Those are not relevant to + this task. """ raise NotHandled From 704be6ff9841388910b498579bb4f3a6051effda Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sun, 7 Jun 2026 13:12:59 -0400 Subject: [PATCH 04/18] Replace synthetic-reader singledispatch with _LexicalVariableTool Address review feedback on PR #670: - Add `_LexicalVariableTool[T](Tool[[], T])` in completions.py with a classmethod `define(env, *, name)` that probes `TypeAdapter(Encodable[typ]).json_schema()` and lets schema failures propagate to the call site. - Inline reader generation into `_collect_tools`; remove `_collect_synthetic_readers`, `_build_synthetic_reader`, `_build_definition_reader`, all `@functools.singledispatch.register` handlers for `type`/`FunctionType`/`MethodType`/`BuiltinFunctionType`/ `ModuleType`/`Tool`/`Agent`, and the `Literal["short", "full"]` toggle. - Register passthrough handlers in encoding.py for `types.ModuleType`, `types.FunctionType`, `types.BuiltinFunctionType`, `types.MethodType`, `type`, and `Agent`. They preempt the broad `_pydantic_callable` fallback so Pydantic's natural schema-generation error fires for these types during the probe; the call site catches and skips. - Delete `_LEXICAL_READERS_PREFACE` and the system-prompt injection path; per-instance `tool_fn.__doc__` carries the framing the LLM sees, scoped to one specific lexical variable per reader. - Widen the probe failure catch tuple to cover the empirically observed failure modes (`PydanticInvalidForJsonSchema`, `PydanticUserError`, `TypeError`, `AttributeError`, `NameError`). - Filter `_collect_tools` to identifier-only, non-dunder names to skip the `@py_builtins`/`@py_assert*` names that pytest's assertion rewriting injects into module globals. - Move `_known_data` from module scope into the test function above `report_sum` in `test_llm_reads_lexical_value`. - Add Test A `test_template_synthesis_uses_lexical_reader` (skipped pending fixture recording with a real API key) and Test B `test_template_skips_lexical_classes` (no LLM, locks the skip-via- catch contract using the #497 `Hand`/`Finger` example). - Rewrite `test_handlers_llm_template.py` PR545 section against the new entry points. Add positive-skip coverage for modules, user classes, unannotated functions/methods, builtins, and Agents; pin that pytest's `MarkDecorator` and `__builtins__` are skipped without aborting collection; pin that annotated callables ARE exposed (the "annotated callable" caveat). --- effectful/handlers/llm/completions.py | 200 ++++++---------- effectful/handlers/llm/encoding.py | 17 +- effectful/handlers/llm/template.py | 21 +- tests/test_handlers_llm_provider.py | 74 +++++- tests/test_handlers_llm_template.py | 319 +++++++++++++------------- 5 files changed, 319 insertions(+), 312 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 9fb0660e7..44b2e6646 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -5,11 +5,9 @@ import functools import inspect import json -import pydoc import string import textwrap import traceback -import types import typing import uuid @@ -180,29 +178,95 @@ def to_feedback_message(self, include_traceback: bool) -> Message: type MessageResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] +class _LexicalVariableTool[T](Tool[[], T]): + """A Tool that returns the current value of a variable captured from + a Template's lexical context. + + Variables become available as zero-argument tools without explicit + wrapping. The body reads `env[name]` at call time, so mutation or + rebinding of the captured variable propagates to subsequent calls. + """ + + @classmethod + def define( + cls, + env: collections.abc.Mapping[str, typing.Any], + *, + name: str, + **kwargs, + ) -> "Tool[[], typing.Any]": + value = env[name] + assert name.isidentifier() + assert not isinstance(value, Tool) + typ: typing.Any = nested_type(value).value + # Probe schema generation. Raises when `Encodable[typ]` is not + # implemented for this type; the caller is responsible for + # catching probe failures and skipping the symbol. + pydantic.TypeAdapter(Encodable[typ]).json_schema() + + def tool_fn(): + return env[name] + + tool_fn.__name__ = name + tool_fn.__qualname__ = name + tool_fn.__module__ = type(value).__module__ + tool_fn.__doc__ = ( + f"Reads the value of lexical variable `{name}` from the " + f"enclosing scope where this Template was defined. Takes " + f"no arguments; returns the current value." + ) + tool_fn.__annotations__ = {"return": typ} + return super().define(tool_fn, **kwargs) + + +# Exceptions the probe (`_LexicalVariableTool.define`) can raise that +# the caller should treat as "this symbol is not exposable as a reader, +# skip it." Each entry covers an observed failure mode: +# * `PydanticSchemaGenerationError` / `PydanticInvalidForJsonSchema`: +# `Encodable[typ]` is not implemented for the inferred type. +# * `PydanticUserError`: schema construction succeeds but the type is +# not fully defined (bare ForwardRef). +# * `TypeError`: `Encodable[typ]` evaluation cannot traverse the type. +# * `AttributeError`: `nested_type` attribute lookups (`__qualname__`, +# `__module__`, `typing.get_overloads`) fail on pathological values +# such as `pytest.mark.parametrize` or method descriptors. +# * `NameError`: `nested_type` evaluates a forward-ref annotation that +# does not resolve in the current scope (e.g. an Operation wrapping a +# third-party callable with stringified annotations). +_SYNTHETIC_READER_PROBE_FAILURES: tuple[type[BaseException], ...] = ( + pydantic.errors.PydanticSchemaGenerationError, + pydantic.errors.PydanticInvalidForJsonSchema, + pydantic.errors.PydanticUserError, + TypeError, + AttributeError, + NameError, +) + + def _collect_tools( env: collections.abc.Mapping[str, typing.Any], ) -> collections.abc.Mapping[str, Tool]: - """Operations and Templates available as tools. Auto-capture from lexical context.""" - result = {} + """Operations and Templates available as tools, plus synthetic + readers for other lexical symbols. Auto-captured from lexical context.""" + result: dict[str, Tool] = {} for name, obj in env.items(): - # Collect tools directly in context if isinstance(obj, Tool | Template): result[name] = obj - - # Collect tools as methods on Agent instances in context elif isinstance(obj, Agent): for cls in type(obj).__mro__: for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) + elif name.isidentifier() and not name.startswith("__"): + try: + result[name] = _LexicalVariableTool.define(env, name=name) + except _SYNTHETIC_READER_PROBE_FAILURES: + continue - # The same Tool can appear under multiple names when it is both - # visible in the enclosing scope *and* discovered via an Agent - # instance's MRO. Since Tools are hashable Operations and - # instance-method Tools are cached per instance, we keep only - # the last name for each unique tool object. + # Same Tool can appear under multiple names when visible both in the + # enclosing scope and via an Agent instance's MRO. Keep only the + # last name for each unique tool object. tool2name = {tool: name for name, tool in sorted(result.items())} for name, tool in tuple(result.items()): if tool2name[tool] != name: @@ -211,113 +275,6 @@ def _collect_tools( return result -def _build_definition_reader( - value: typing.Any, name: str, env: collections.abc.Mapping[str, typing.Any] -) -> Tool | None: - """Build a Tool that returns the source / help() output of a class or - function. Probe is `inspect.getsource` reachability; symbols whose - source is unreachable (builtin C, REPL lambdas, etc.) are skipped.""" - try: - inspect.getsource(value) - except (OSError, TypeError): - return None - - kind = "class" if inspect.isclass(value) else "function" - - def body(level: typing.Literal["short", "full"] = "short") -> str: - obj = env[name] - if level == "short": - return pydoc.render_doc(obj, renderer=pydoc.plaintext) # type: ignore[attr-defined] - return inspect.getsource(obj) - - body.__name__ = name - body.__doc__ = ( - f"Read the definition of `{name}` (a {kind}). " - f'`level="short"` (default) returns `help({name})` output; ' - f'`level="full"` returns `inspect.getsource({name})`. ' - f"Calls must include `level` explicitly under OpenAI strict mode." - ) - body.__annotations__ = { - "level": typing.Literal["short", "full"], - "return": str, - } - return Tool.define(body) - - -@functools.singledispatch -def _build_synthetic_reader( - value: typing.Any, - name: str, - env: collections.abc.Mapping[str, typing.Any], -) -> Tool | None: - """Build a synthetic read-only Tool for a lexical symbol. - - Default branch: value-reader. Probe is the §2c Encodable schema check; - on failure (unencodable value), return None and let the symbol be - skipped silently. - - Registrations route classes and functions through the - definition-reader path (`_build_definition_reader`), and route Tool, - Agent, and module values to a no-op (already collected by - `_collect_tools` or intentionally excluded). - """ - try: - inferred: typing.Any = nested_type(value).value - adapter: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( - Encodable[inferred] - ) - adapter.json_schema() - except Exception: - # The probe chains through several third-party libraries - # (nested_type, inspect.signature, typing.get_overloads, - # Pydantic schema generation). Any failure means "this symbol - # cannot be exposed as a synthetic reader", so we catch broadly. - # The cost of a too-narrow catch is a crash mid-call_assistant; - # the cost of a too-broad catch is silently skipping a symbol - # that might have worked. - return None - - def body(): - return env[name] - - body.__name__ = name - body.__doc__ = f"Read the value of lexical variable `{name}` (type `{inferred}`)." - body.__annotations__ = {"return": inferred} - return Tool.define(body) - - -_build_synthetic_reader.register(type, _build_definition_reader) -_build_synthetic_reader.register(types.FunctionType, _build_definition_reader) -_build_synthetic_reader.register(types.BuiltinFunctionType, _build_definition_reader) -_build_synthetic_reader.register(types.MethodType, _build_definition_reader) - - -@_build_synthetic_reader.register(types.ModuleType) -@_build_synthetic_reader.register(Tool) -@_build_synthetic_reader.register(Agent) -def _no_synthetic_reader(value, name, env): - return None - - -def _collect_synthetic_readers( - env: collections.abc.Mapping[str, typing.Any], - already_collected: collections.abc.Set[str], -) -> collections.abc.Mapping[str, Tool]: - """Synthetic readers for lexical symbols not already covered by - `_collect_tools`. Skips dunder names and any name already in - `already_collected`.""" - result: dict[str, Tool] = {} - for name, obj in env.items(): - if name in already_collected: - continue - if name.startswith("__"): - continue - tool = _build_synthetic_reader(obj, name, env) - if tool is not None: - result[name] = tool - return result - - @Operation.define @functools.wraps(litellm.completion) def completion(*args, **kwargs) -> typing.Any: @@ -353,11 +310,6 @@ def call_assistant[T]( includes the raw assistant message for retry handling. """ tools = dict(_collect_tools(env)) - # Add synthetic readers for non-Tool lexical symbols. These mirror the - # bound-args layer that LiteLLMProvider._call adds to env, so the LLM - # sees readers for the Template's arguments alongside other lexical - # context. - tools.update(_collect_synthetic_readers(env, set(tools))) tool_specs = { k: typing.cast( pydantic.TypeAdapter[typing.Any], diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index cfbb08aec..79f829a3a 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -31,7 +31,7 @@ from PIL import Image import effectful.handlers.llm.evaluation as evaluation -from effectful.handlers.llm.template import Tool +from effectful.handlers.llm.template import Agent, Tool from effectful.internals.unification import GenericAlias, TypeEvaluator, nested_type from effectful.ops.types import Operation, Term @@ -684,3 +684,18 @@ def _pydantic_type_tool_call(ty: type[DecodedToolCall]): pydantic.PlainSerializer(_serialize_tool_call), pydantic.WithJsonSchema(schema), ] + + +# Handlers for types that should not appear as synthetic readers in +# `_collect_tools`. They return `ty` unchanged, preempting the broad +# `_pydantic_callable` fallback so Pydantic's natural +# PydanticSchemaGenerationError / PydanticInvalidForJsonSchema fires at +# TypeAdapter(...).json_schema() and the caller's catch skips the value. +@TypeToPydanticType.register(types.ModuleType) +@TypeToPydanticType.register(types.FunctionType) +@TypeToPydanticType.register(types.BuiltinFunctionType) +@TypeToPydanticType.register(types.MethodType) +@TypeToPydanticType.register(type) +@TypeToPydanticType.register(Agent) +def _pydantic_type_passthrough(ty): + return ty diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 8b2a96194..15f991b77 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -11,12 +11,6 @@ from effectful.ops.types import Annotation, Operation -_LEXICAL_READERS_PREFACE = ( - "You also have access to read-only tools for inspecting the lexical " - "scope where this template is defined. Their names match variable " - "names from that scope; calling one returns the current value." -) - class _IsRecursiveAnnotation(Annotation): """ @@ -223,13 +217,9 @@ def __prompt_template__(self) -> str: def tools(self) -> Mapping[str, Tool]: """Operations and Templates available as tools, plus synthetic readers for other lexical symbols. Auto-captured from lexical context.""" - from effectful.handlers.llm.completions import ( - _collect_synthetic_readers, - _collect_tools, - ) + from effectful.handlers.llm.completions import _collect_tools result = dict(_collect_tools(self.__context__)) - result.update(_collect_synthetic_readers(self.__context__, set(result))) # We remove the template itself from the tool map unless it is explicitly # marked as recursive (see test_template_method, test_template_method_nested_class). @@ -322,14 +312,7 @@ def define[**Q, V]( op = super().define(default, *args, **kwargs) op.__context__ = context # type: ignore[attr-defined] mod = inspect.getmodule(_fn) - op.__system_prompt__ = "\n\n".join( # type: ignore[attr-defined] - part - for part in ( - inspect.getdoc(mod) if mod is not None else None, - _LEXICAL_READERS_PREFACE, - ) - if part - ) + op.__system_prompt__ = inspect.getdoc(mod) or "" # type: ignore[attr-defined] # Keep validation on original define-time callables, but skip the bound wrapper path. # to avoid dropping `self` from the signature and falsely rejecting valid prompt fields like `{self.name}`. is_bound_wrapper = ( diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index a994645d2..3dee9d070 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -2165,11 +2165,6 @@ def _completion(self, model, messages=None, **kwargs): # --------------------------------------------------------------------------- -# Module-level binding the LLM will be asked to inspect via a synthetic -# reader. The reader's name in the tool list is `_known_data`. -_known_data = [10, 20, 30, 40, 50] - - class TestSyntheticReaderIntegration: """The LLM can read lexical context through synthetic reader tools.""" @@ -2179,6 +2174,7 @@ def test_llm_reads_lexical_value(self, request): The synthetic reader for _known_data is available in the tools array; the LLM should call it, see [10,20,30,40,50], and report the sum (150).""" + _known_data = [10, 20, 30, 40, 50] @Template.define def report_sum() -> int: @@ -2191,3 +2187,71 @@ def report_sum() -> int: assert isinstance(result, int) assert result == sum(_known_data) # 150 + + @pytest.mark.skip( + reason=( + "Fixture recording pending: run `REBUILD_FIXTURES=1 pytest " + "tests/test_handlers_llm_provider.py" + "::TestSyntheticReaderIntegration" + "::test_template_synthesis_uses_lexical_reader` " + "locally with an API key, commit the recorded JSON files, " + "and remove this skip." + ) + ) + @requires_llm + def test_template_synthesis_uses_lexical_reader(self, request): + """A Template that synthesizes a callable grounds its output + in a lexical value exposed as a synthetic reader. + + The Template asks the LLM to write a lambda comparing its + argument against `threshold`; the LLM must call the `threshold` + reader to inspect the value before emitting code. + """ + threshold = 0.85 + + @Template.define + def make_above_threshold() -> Callable[[float], bool]: + """Write a Python lambda that returns True iff its float + argument is strictly greater than the value of `threshold`. + Use the `threshold` reader tool to inspect its current value + before emitting the lambda.""" + raise NotImplementedError + + with ( + handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)), + handler(UnsafeEvalProvider()), + handler(LimitLLMCallsHandler(max_calls=4)), + ): + fn = make_above_threshold() + + assert fn(0.9) is True + assert fn(0.5) is False + assert fn(threshold) is False + + def test_template_skips_lexical_classes(self): + """Classes in the defining scope are NOT exposed as readers. + Locks the skip-via-catch direction for the `type` Encodable + handler: `Hand`/`Finger` produce `PydanticInvalidForJsonSchema` + at the probe and the call site catches them. + + This is the `Hand`/`Finger`/`generate_arm` motivating example + from #497 pinned to its current contract. A follow-up that + adds real `Encodable[type]` impls flips this test to a positive + assertion. + """ + + class Finger: + def wiggle(self) -> str: + return "wiggle" + + class Hand: + fingers: list[Finger] + + @Template.define + def describe_hand_action() -> str: + """Doc.""" + raise NotImplementedError + + tools = describe_hand_action.tools + assert "Finger" not in tools + assert "Hand" not in tools diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 2b9dfae74..cd89ef060 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1548,11 +1548,10 @@ def test_tool_forward_ref(): import pydantic from effectful.handlers.llm.completions import ( - _build_synthetic_reader, - _collect_synthetic_readers, + _SYNTHETIC_READER_PROBE_FAILURES, _collect_tools, + _LexicalVariableTool, ) -from effectful.handlers.llm.template import _LEXICAL_READERS_PREFACE # Helpers for the test matrix @@ -1575,23 +1574,31 @@ class _OpaqueNoEncoder: pass +def _example_unannotated(x): + """Unannotated function for the skip-via-catch tests.""" + return x + + +def _example_annotated(x: int) -> int: + """Annotated function returning x.""" + return x + + def test_synthetic_reader_value_reader_returns_live_value(): """A value-reader returns whatever env[name] currently is. Mutation after reader construction propagates.""" - env = {"x": [1, 2, 3]} - tool = _build_synthetic_reader([1, 2, 3], "x", env) - assert tool is not None + env: dict = {"x": [1, 2, 3]} + tool = _LexicalVariableTool.define(env, name="x") assert tool() == [1, 2, 3] env["x"].append(4) assert tool() == [1, 2, 3, 4] def test_synthetic_reader_value_reader_rebind(): - """A reader returns the current binding even after rebind. The - annotation is set at .tools-access time but the body reads live.""" + """The reader returns the current binding even after rebind. The + body reads `env[name]` at call time, not a snapshot.""" env: dict = {"x": 42} - tool = _build_synthetic_reader(42, "x", env) - assert tool is not None + tool = _LexicalVariableTool.define(env, name="x") assert tool() == 42 env["x"] = 99 assert tool() == 99 @@ -1599,11 +1606,9 @@ def test_synthetic_reader_value_reader_rebind(): def test_synthetic_reader_skips_when_name_deleted(): """If env[name] is deleted between collection and call, the reader - raises KeyError on direct invocation. call_tool's ToolCallExecutionError - wrapping is the standard tool-runtime-error contract, tested elsewhere.""" - env = {"x": 42} - tool = _build_synthetic_reader(42, "x", env) - assert tool is not None + raises KeyError on direct invocation.""" + env: dict = {"x": 42} + tool = _LexicalVariableTool.define(env, name="x") del env["x"] with pytest.raises(KeyError): tool() @@ -1615,11 +1620,11 @@ def test_synthetic_reader_skips_when_name_deleted(): ("primitive_int", 42, True), ("primitive_str", "hello", True), ("list_of_int", [1, 2, 3], True), - ("dict", {"a": 1}, True), + ("dict_value", {"a": 1}, True), ("dataclass_simple", _SimpleDataclass(x=1, y="hello"), True), ("pydantic_model", _SimpleModel(x=1, y="hello"), True), - # The plan's matrix says re.Pattern and pathlib.PosixPath instances - # ARE exposed by Encodable. Pin them here as a regression guard. + # `re.Pattern` and `pathlib.PosixPath` instances ARE exposed by + # Encodable. Pin them here as a regression guard. ("re_pattern", re.compile(r"x"), True), ("pathlib_path", Path("/tmp"), True), # Unencodable categories: probe rejects. @@ -1628,20 +1633,20 @@ def test_synthetic_reader_skips_when_name_deleted(): ], ) def test_synthetic_reader_probe_matches_encodable(name, value, should_have_tool): - """The §2c probe accepts iff the symbol can produce a Pydantic - schema. No false positives, no false negatives.""" + """The probe accepts iff the symbol can produce a Pydantic schema.""" env = {name: value} - tool = _build_synthetic_reader(value, name, env) if should_have_tool: - assert tool is not None + tool = _LexicalVariableTool.define(env, name=name) + assert tool() == value else: - assert tool is None + with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): + _LexicalVariableTool.define(env, name=name) def test_synthetic_readers_yield_to_real_tools(): - """Real Tools/Templates collected by `_collect_tools` take precedence - over same-named synthetic readers (which are skipped via - `already_collected`).""" + """Real Tools/Templates take precedence over same-named synthetic + readers; the `isinstance(obj, Tool | Template)` branch fires first + in `_collect_tools`.""" @Tool.define def shared() -> int: @@ -1649,177 +1654,165 @@ def shared() -> int: return 1 env = collections.ChainMap({"shared": shared, "shared_value": 42}) - real = _collect_tools(env) - synth = _collect_synthetic_readers(env, set(real)) - assert "shared" not in synth, "real tool name should not be re-wrapped" - assert "shared_value" in synth + result = _collect_tools(env) + assert result["shared"] is shared + assert isinstance(result["shared_value"], _LexicalVariableTool) + assert result["shared_value"]() == 42 def test_synthetic_reader_annotation_has_no_free_typevars(): - """Polymorphic-Template post-#668 substitutes TypeVars in Tool - annotations. Synthetic reader annotations come from nested_type, - which produces concrete types — substitution is a no-op.""" + """Synthetic reader annotations come from `nested_type`, which + produces concrete types — TypeVar substitution is a no-op.""" from effectful.internals.unification import freetypevars env = {"x": [1, 2, 3]} - tool = _build_synthetic_reader([1, 2, 3], "x", env) - assert tool is not None + tool = _LexicalVariableTool.define(env, name="x") sig = inspect.signature(tool) assert freetypevars(sig.return_annotation) == set() -# ---- Definition-readers ---- - - -def test_definition_reader_short_form_substring_properties(): - """Short form (pydoc.render_doc output) contains the docstring, - method signatures, and method docstrings. Substring assertions - only — pydoc output format is not a stable API.""" - - class _ReaderTarget: - """Class docstring.""" - - def method(self, x: int) -> str: - """Method docstring.""" - return str(x) +# ---- Skip-via-catch coverage for the types that preempt Callable ---- - env = {"_ReaderTarget": _ReaderTarget} - tool = _build_synthetic_reader(_ReaderTarget, "_ReaderTarget", env) - assert tool is not None - - short = tool(level="short") - assert "Class docstring." in short - assert "method(self, x: int) -> str" in short - assert "Method docstring." in short - - -def test_definition_reader_full_form_routes_to_getsource_not_pydoc(): - """level="full" must route to inspect.getsource, not pydoc. Two - independent properties distinguish them: method bodies appear only - in getsource output, and the 'Methods defined here:' header is - pydoc-specific.""" - - class _ReaderTarget: - """Doc.""" - - def method(self, x: int) -> str: - return str(x) - - env = {"_ReaderTarget": _ReaderTarget} - tool = _build_synthetic_reader(_ReaderTarget, "_ReaderTarget", env) - assert tool is not None - - full = tool(level="full") - assert "return str(x)" in full, ( - "method body must be present (only getsource shows bodies)" - ) - assert "Methods defined here:" not in full, "pydoc-specific header must be absent" - - -def test_definition_reader_lives_through_env(): - """Rebinding the class in env shows the new definition on the next - call. Matches §1 live semantics.""" - - class _Old: - """Old class.""" - - class _New: - """New class.""" - - env: dict = {"x": _Old} - tool = _build_synthetic_reader(_Old, "x", env) - assert tool is not None - assert "Old class." in tool(level="short") - - env["x"] = _New - assert "New class." in tool(level="short") - - -def test_definition_reader_function(): - """A user-defined function gets a definition-reader. Its short form - contains the function signature and docstring; full form contains - the function body.""" - - def _example(x: int) -> int: - """Add one.""" - return x + 1 - - env = {"_example": _example} - tool = _build_synthetic_reader(_example, "_example", env) - assert tool is not None - short = tool(level="short") - assert "_example(x: int) -> int" in short - assert "Add one." in short - full = tool(level="full") - assert "return x + 1" in full +def test_lexical_reader_skips_modules(): + """Modules are not exposed as readers; `Encodable[types.ModuleType]` + preempts the Callable fallback and Pydantic raises during the probe.""" + env = {"os": os} + with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): + _LexicalVariableTool.define(env, name="os") + assert "os" not in _collect_tools(env) -def test_definition_reader_baseclass_uses_metaclass_dispatch(): - """Singledispatch on `type` matches Pydantic BaseModel subclasses - (whose metaclass is ModelMetaclass, a subclass of type). The plan - relies on this — verify.""" - class BMSubclass(pydantic.BaseModel): - """BM doc.""" +def test_lexical_reader_skips_user_classes(): + """User-defined classes are not exposed as readers; Pydantic raises + `PydanticInvalidForJsonSchema` when handed a bare `type`.""" - x: int + class _UserClass: + """Some class.""" - env = {"BMSubclass": BMSubclass} - tool = _build_synthetic_reader(BMSubclass, "BMSubclass", env) - assert tool is not None - short = tool(level="short") - assert "BM doc." in short + env = {"_UserClass": _UserClass} + with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): + _LexicalVariableTool.define(env, name="_UserClass") + assert "_UserClass" not in _collect_tools(env) -def test_definition_reader_skips_when_source_unreachable(): - """Builtin C types like `int` fail inspect.getsource with TypeError; - we skip them at probe time rather than crashing at call time.""" - env = {"int": int} - assert _build_synthetic_reader(int, "int", env) is None +def test_lexical_reader_skips_unannotated_functions(): + """Unannotated functions are not exposed as readers.""" + env = {"_example_unannotated": _example_unannotated} + with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): + _LexicalVariableTool.define(env, name="_example_unannotated") + assert "_example_unannotated" not in _collect_tools(env) -def test_typevars_are_skipped(): - """TypeVar instances are not classes or routines; they fall through - to the value-reader default branch, fail the §2c probe with - PydanticSchemaGenerationError, and are skipped.""" - T = typing.TypeVar("T") - env = {"T": T} - assert _build_synthetic_reader(T, "T", env) is None +def test_lexical_reader_skips_unannotated_bound_methods(): + """Unannotated bound methods are not exposed as readers. + Annotated callables resolve to `Callable[...]` and follow the + `_pydantic_callable` path instead (see the + `test_lexical_reader_exposes_annotated_callables` contract below).""" + class _C: + def m(self): + return 1 -def test_module_values_are_skipped(): - """Modules are never wrapped as synthetic readers — registered to - return None at dispatch time.""" - env = {"os": os} - assert _build_synthetic_reader(os, "os", env) is None + env = {"m": _C().m} + with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): + _LexicalVariableTool.define(env, name="m") + assert "m" not in _collect_tools(env) -def test_box_value_filtered_via_probe_typeerror_chain(): - """Box-as-value is skipped via the probe TypeError branch. Chain: +def test_lexical_reader_exposes_annotated_callables(): + """Annotated callables in lexical scope ARE exposed as readers via + the `Callable[Args, Ret]` resolution path. Their schemas describe + the function signature, which is useful synthesis context.""" + env = {"_example_annotated": _example_annotated} + result = _collect_tools(env) + assert "_example_annotated" in result + assert result["_example_annotated"]() is _example_annotated - nested_type(Box(42)) → Box(42) (passthrough; unification.py:952) - .value → 42 (Box's only field) - Encodable[42] → TypeError (literal value, not a type) - §2c probe catches → _build_synthetic_reader returns None - Named for the contract (chain-as-contract). If nested_type later - handles Box more usefully, this test breaks and we revisit.""" - from effectful.internals.unification import Box +def test_lexical_reader_skips_builtin_functions(): + """Builtin functions (`len`, etc.) are not exposed as readers.""" + env = {"len": len} + with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): + _LexicalVariableTool.define(env, name="len") + assert "len" not in _collect_tools(env) - env = {"b": Box(42)} - assert _build_synthetic_reader(Box(42), "b", env) is None +def test_lexical_reader_skips_agent_instances_but_exposes_their_tools(): + """Agent instances themselves are not exposed as readers, but the + MRO walk in `_collect_tools` continues to expose their contained + Tools under `agent_name__method_name`.""" -def test_system_prompt_contains_preface(): - """Template.__system_prompt__ includes the lexical-readers preface - sentence unconditionally.""" + class _A(Agent): + @Tool.define + def t(self) -> int: + """Doc.""" + return 7 + + inst = _A() + env = {"a": inst} + result = _collect_tools(env) + assert "a" not in result + # The MRO walk picks up the contained Tool. + assert any(k.startswith("a__") for k in result) + + +def test_lexical_reader_exposes_data_values(): + """Data-shaped values are exposed as readers (positive contract).""" + env = { + "x": 1, + "s": "hello", + "lst": [1, 2, 3], + "d": {"k": 1}, + "model": _SimpleModel(x=1, y="hi"), + } + result = _collect_tools(env) + assert {"x", "s", "lst", "d", "model"} <= set(result) + assert result["x"]() == 1 + assert result["model"]() == env["model"] + + +def test_lexical_reader_skips_dunders(): + """Dunder-prefixed names like `__builtins__` are never exposed.""" + env = {"__builtins__": __builtins__, "regular": 42} + result = _collect_tools(env) + assert "__builtins__" not in result + assert "regular" in result + + +def test_lexical_reader_skips_marker_objects(): + """`pytest.mark.parametrize` (a `MarkDecorator`) in env does not + abort tool collection; the probe catches the `AttributeError` from + `nested_type`'s Callable branch (`typing.get_overloads` accesses + `__qualname__` which `MarkDecorator` lacks).""" + env = {"mark": pytest.mark.parametrize, "regular": 42} + result = _collect_tools(env) + assert "mark" not in result + assert "regular" in result + + +def test_system_prompt_has_no_lexical_readers_preface(): + """The system prompt no longer carries a global preface about + lexical readers; the per-tool docstring carries the framing.""" @Template.define - def with_preface(x: int) -> int: + def t(x: int) -> int: """Doc.""" raise NotHandled - assert _LEXICAL_READERS_PREFACE in with_preface.__system_prompt__ + # Module docstring may be present, but the old preface phrasing + # ("You also have access to read-only tools") is not. + assert "read-only tools for inspecting the lexical" not in t.__system_prompt__ + + +def test_lexical_reader_doc_describes_lexical_origin(): + """Each `_LexicalVariableTool` carries a per-instance docstring + that tells the LLM it reads a variable from the enclosing scope.""" + env = {"x": [1, 2, 3]} + tool = _LexicalVariableTool.define(env, name="x") + assert tool.__doc__ is not None + assert "lexical variable" in tool.__doc__ + assert "`x`" in tool.__doc__ def test_template_tools_includes_synthetic_readers_for_locals(): From b3e4b6f9f13ffc654773b849f86d1bd820a84798 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sun, 7 Jun 2026 13:33:28 -0400 Subject: [PATCH 05/18] Skip class values at _LexicalVariableTool.define `nested_type(SomeClass)` routes through its Callable branch and extracts `__init__`'s signature, returning `Callable[Args, Return]`. For dataclass-like classes with annotated constructors that schema generates fine via `_pydantic_callable`, bypassing the `type` passthrough in `Encodable[T]`. Add an `isinstance(value, type)` pre-check at the top of `_LexicalVariableTool.define` so the skip-via-catch direction stays consistent for both bare classes (caught via `Encodable[type]`) and dataclass-like classes (caught here). --- effectful/handlers/llm/completions.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 44b2e6646..5bdfd5b4b 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -198,6 +198,16 @@ def define( value = env[name] assert name.isidentifier() assert not isinstance(value, Tool) + # Class objects route through `nested_type`'s Callable branch, + # which extracts their `__init__` signature and returns + # `Callable[Args, Return]`. That sidesteps the `type` passthrough + # in `Encodable[T]`. Reject classes at the value level so the + # skip-via-catch direction stays consistent for both bare classes + # and dataclass-like classes with annotated constructors. + if isinstance(value, type): + raise pydantic.errors.PydanticSchemaGenerationError( + f"Class objects are not exposed as synthetic readers ({name})." + ) typ: typing.Any = nested_type(value).value # Probe schema generation. Raises when `Encodable[typ]` is not # implemented for this type; the caller is responsible for From 6ef47285c588bcba1be24dfbc19f88933348eb6f Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sun, 7 Jun 2026 16:22:41 -0400 Subject: [PATCH 06/18] Collapse skip mechanism into one _is_synthetic_reader_eligible predicate Replaces two parallel skip mechanisms (encoding.py passthrough handlers that provoke a Pydantic schema error, plus an isinstance raise in _LexicalVariableTool.define that forges PydanticSchemaGenerationError from outside Pydantic) with a single predicate in completions.py. - Add _is_synthetic_reader_eligible(value) -> bool that rejects values whose type is in _NON_READER_TYPES (type, Module, Function, Method, BuiltinFunction, Agent, Tool) and probes Encodable[T] schema generation for the rest. - Use the predicate as the third branch's gate in _collect_tools; no try/except around tool construction. - Strip the isinstance(value, type) raise and the in-define probe from _LexicalVariableTool.define; the class is now purely constructive and assumes its caller has already gated on the predicate. - Drop the six TypeToPydanticType registrations and the _pydantic_type_passthrough function from encoding.py; drop the now- unused Agent import there. - Re-add `import types` to completions.py for the predicate's isinstance tuple. Behavior change: annotated callables in lexical scope are now skipped along with unannotated ones, matching how class objects (which also resolve to Callable[Args, Return] via nested_type) are treated. The test that previously pinned "annotated callables ARE exposed" is inverted to pin "they are skipped". --- effectful/handlers/llm/completions.py | 87 +++++++++++++++------------ effectful/handlers/llm/encoding.py | 17 +----- tests/test_handlers_llm_template.py | 73 ++++++++++------------ 3 files changed, 82 insertions(+), 95 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 5bdfd5b4b..d055a110e 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -8,6 +8,7 @@ import string import textwrap import traceback +import types import typing import uuid @@ -182,9 +183,9 @@ class _LexicalVariableTool[T](Tool[[], T]): """A Tool that returns the current value of a variable captured from a Template's lexical context. - Variables become available as zero-argument tools without explicit - wrapping. The body reads `env[name]` at call time, so mutation or - rebinding of the captured variable propagates to subsequent calls. + Callers must gate construction on `_is_synthetic_reader_eligible`; + this classmethod assumes the value has already been deemed + schema-encodable and does no further checks. """ @classmethod @@ -196,23 +197,7 @@ def define( **kwargs, ) -> "Tool[[], typing.Any]": value = env[name] - assert name.isidentifier() - assert not isinstance(value, Tool) - # Class objects route through `nested_type`'s Callable branch, - # which extracts their `__init__` signature and returns - # `Callable[Args, Return]`. That sidesteps the `type` passthrough - # in `Encodable[T]`. Reject classes at the value level so the - # skip-via-catch direction stays consistent for both bare classes - # and dataclass-like classes with annotated constructors. - if isinstance(value, type): - raise pydantic.errors.PydanticSchemaGenerationError( - f"Class objects are not exposed as synthetic readers ({name})." - ) typ: typing.Any = nested_type(value).value - # Probe schema generation. Raises when `Encodable[typ]` is not - # implemented for this type; the caller is responsible for - # catching probe failures and skipping the symbol. - pydantic.TypeAdapter(Encodable[typ]).json_schema() def tool_fn(): return env[name] @@ -229,20 +214,11 @@ def tool_fn(): return super().define(tool_fn, **kwargs) -# Exceptions the probe (`_LexicalVariableTool.define`) can raise that -# the caller should treat as "this symbol is not exposable as a reader, -# skip it." Each entry covers an observed failure mode: -# * `PydanticSchemaGenerationError` / `PydanticInvalidForJsonSchema`: -# `Encodable[typ]` is not implemented for the inferred type. -# * `PydanticUserError`: schema construction succeeds but the type is -# not fully defined (bare ForwardRef). -# * `TypeError`: `Encodable[typ]` evaluation cannot traverse the type. -# * `AttributeError`: `nested_type` attribute lookups (`__qualname__`, -# `__module__`, `typing.get_overloads`) fail on pathological values -# such as `pytest.mark.parametrize` or method descriptors. -# * `NameError`: `nested_type` evaluates a forward-ref annotation that -# does not resolve in the current scope (e.g. an Operation wrapping a -# third-party callable with stringified annotations). +# Exceptions raised by the eligibility probe in +# `_is_synthetic_reader_eligible`. `TypeError` / `AttributeError` / +# `NameError` cover empirically-observed `nested_type` failures on +# pathological values (`pytest.mark.parametrize`, method descriptors, +# Operations with unresolved forward-refs); see #673. _SYNTHETIC_READER_PROBE_FAILURES: tuple[type[BaseException], ...] = ( pydantic.errors.PydanticSchemaGenerationError, pydantic.errors.PydanticInvalidForJsonSchema, @@ -253,6 +229,40 @@ def tool_fn(): ) +# Concrete types that are never wrapped as synthetic readers regardless +# of whether `Encodable[T]` would happen to schematise them. Class +# objects with annotated `__init__` route through `nested_type`'s +# Callable branch and slip past `Encodable[type]`; modules / functions +# / methods / builtins / Agent instances would otherwise be wrapped as +# `Callable[Args, Return]` synthesis schemas. +_NON_READER_TYPES: tuple[type, ...] = ( + type, + types.ModuleType, + types.FunctionType, + types.MethodType, + types.BuiltinFunctionType, + Agent, + Tool, +) + + +def _is_synthetic_reader_eligible(value: typing.Any) -> bool: + """Decide whether `value` should be exposed as a synthetic reader. + + A value is eligible iff it is not an instance of any + `_NON_READER_TYPES` class AND its `Encodable[nested_type(value)]` + schema can be generated. + """ + if isinstance(value, _NON_READER_TYPES): + return False + try: + typ = nested_type(value).value + pydantic.TypeAdapter(Encodable[typ]).json_schema() + except _SYNTHETIC_READER_PROBE_FAILURES: + return False + return True + + def _collect_tools( env: collections.abc.Mapping[str, typing.Any], ) -> collections.abc.Mapping[str, Tool]: @@ -268,11 +278,12 @@ def _collect_tools( for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - elif name.isidentifier() and not name.startswith("__"): - try: - result[name] = _LexicalVariableTool.define(env, name=name) - except _SYNTHETIC_READER_PROBE_FAILURES: - continue + elif ( + name.isidentifier() + and not name.startswith("__") + and _is_synthetic_reader_eligible(obj) + ): + result[name] = _LexicalVariableTool.define(env, name=name) # Same Tool can appear under multiple names when visible both in the # enclosing scope and via an Agent instance's MRO. Keep only the diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 79f829a3a..cfbb08aec 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -31,7 +31,7 @@ from PIL import Image import effectful.handlers.llm.evaluation as evaluation -from effectful.handlers.llm.template import Agent, Tool +from effectful.handlers.llm.template import Tool from effectful.internals.unification import GenericAlias, TypeEvaluator, nested_type from effectful.ops.types import Operation, Term @@ -684,18 +684,3 @@ def _pydantic_type_tool_call(ty: type[DecodedToolCall]): pydantic.PlainSerializer(_serialize_tool_call), pydantic.WithJsonSchema(schema), ] - - -# Handlers for types that should not appear as synthetic readers in -# `_collect_tools`. They return `ty` unchanged, preempting the broad -# `_pydantic_callable` fallback so Pydantic's natural -# PydanticSchemaGenerationError / PydanticInvalidForJsonSchema fires at -# TypeAdapter(...).json_schema() and the caller's catch skips the value. -@TypeToPydanticType.register(types.ModuleType) -@TypeToPydanticType.register(types.FunctionType) -@TypeToPydanticType.register(types.BuiltinFunctionType) -@TypeToPydanticType.register(types.MethodType) -@TypeToPydanticType.register(type) -@TypeToPydanticType.register(Agent) -def _pydantic_type_passthrough(ty): - return ty diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index cd89ef060..b09e27cb8 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1548,9 +1548,9 @@ def test_tool_forward_ref(): import pydantic from effectful.handlers.llm.completions import ( - _SYNTHETIC_READER_PROBE_FAILURES, - _collect_tools, _LexicalVariableTool, + _collect_tools, + _is_synthetic_reader_eligible, ) @@ -1633,14 +1633,9 @@ def test_synthetic_reader_skips_when_name_deleted(): ], ) def test_synthetic_reader_probe_matches_encodable(name, value, should_have_tool): - """The probe accepts iff the symbol can produce a Pydantic schema.""" - env = {name: value} - if should_have_tool: - tool = _LexicalVariableTool.define(env, name=name) - assert tool() == value - else: - with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): - _LexicalVariableTool.define(env, name=name) + """The eligibility predicate accepts iff the symbol produces a + Pydantic schema and is not a class/module/callable/Agent/Tool.""" + assert _is_synthetic_reader_eligible(value) is should_have_tool def test_synthetic_readers_yield_to_real_tools(): @@ -1675,66 +1670,62 @@ def test_synthetic_reader_annotation_has_no_free_typevars(): def test_lexical_reader_skips_modules(): - """Modules are not exposed as readers; `Encodable[types.ModuleType]` - preempts the Callable fallback and Pydantic raises during the probe.""" + """Modules are not exposed as readers.""" env = {"os": os} - with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): - _LexicalVariableTool.define(env, name="os") + assert not _is_synthetic_reader_eligible(os) assert "os" not in _collect_tools(env) def test_lexical_reader_skips_user_classes(): - """User-defined classes are not exposed as readers; Pydantic raises - `PydanticInvalidForJsonSchema` when handed a bare `type`.""" + """User-defined classes are not exposed as readers.""" class _UserClass: """Some class.""" env = {"_UserClass": _UserClass} - with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): - _LexicalVariableTool.define(env, name="_UserClass") + assert not _is_synthetic_reader_eligible(_UserClass) assert "_UserClass" not in _collect_tools(env) def test_lexical_reader_skips_unannotated_functions(): """Unannotated functions are not exposed as readers.""" env = {"_example_unannotated": _example_unannotated} - with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): - _LexicalVariableTool.define(env, name="_example_unannotated") + assert not _is_synthetic_reader_eligible(_example_unannotated) assert "_example_unannotated" not in _collect_tools(env) -def test_lexical_reader_skips_unannotated_bound_methods(): - """Unannotated bound methods are not exposed as readers. - Annotated callables resolve to `Callable[...]` and follow the - `_pydantic_callable` path instead (see the - `test_lexical_reader_exposes_annotated_callables` contract below).""" +def test_lexical_reader_skips_annotated_functions(): + """Annotated functions in lexical scope are skipped by the predicate + just like unannotated ones — the eligibility check is by value type, + not by signature shape, so the synthetic-reader machinery never + duplicates the Callable-synthesis schema that real Tool definitions + rely on.""" + env = {"_example_annotated": _example_annotated} + assert not _is_synthetic_reader_eligible(_example_annotated) + assert "_example_annotated" not in _collect_tools(env) + + +def test_lexical_reader_skips_bound_methods(): + """Bound methods (annotated or not) are not exposed as readers.""" class _C: - def m(self): + def annotated(self) -> int: return 1 - env = {"m": _C().m} - with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): - _LexicalVariableTool.define(env, name="m") - assert "m" not in _collect_tools(env) - + def unannotated(self): + return 2 -def test_lexical_reader_exposes_annotated_callables(): - """Annotated callables in lexical scope ARE exposed as readers via - the `Callable[Args, Ret]` resolution path. Their schemas describe - the function signature, which is useful synthesis context.""" - env = {"_example_annotated": _example_annotated} - result = _collect_tools(env) - assert "_example_annotated" in result - assert result["_example_annotated"]() is _example_annotated + inst = _C() + for name, value in [("annotated", inst.annotated), ("unannotated", inst.unannotated)]: + env = {name: value} + assert not _is_synthetic_reader_eligible(value) + assert name not in _collect_tools(env) def test_lexical_reader_skips_builtin_functions(): """Builtin functions (`len`, etc.) are not exposed as readers.""" env = {"len": len} - with pytest.raises(_SYNTHETIC_READER_PROBE_FAILURES): - _LexicalVariableTool.define(env, name="len") + assert not _is_synthetic_reader_eligible(len) assert "len" not in _collect_tools(env) From 7f313cb583d53af96e44311106a2f5269d3a1e36 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sun, 7 Jun 2026 17:14:09 -0400 Subject: [PATCH 07/18] Pure-Encodable reader collection: drop isinstance pre-filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the class+predicate pair (_LexicalVariableTool + _is_synthetic_reader_eligible) with a single free function _define_lexical_reader that probes Encodable[T].json_schema() and lets failures propagate. _collect_tools catches the probe failures inline (six-exception tuple no longer named — the catch is the only consumer). Behavior consequence: classes (plain and dataclass-shaped), unannotated functions, lambdas, bound methods, and builtins are now exposed as readers because nested_type resolves them to a Callable shape that _pydantic_callable schematises. Modules and Agent-subclass instances still naturally fail the probe. The skip-set is whatever Encodable rejects — no per-type code path remains. Tests: - Replace _LexicalVariableTool.define call sites with _define_lexical_reader. - Replace _is_synthetic_reader_eligible checks with collect-and-check or pytest.raises. - Flip Test B (provider) from "classes skipped" to "classes exposed". - Flip the skips_user_classes / skips_*_functions / skips_*_methods / skips_builtin_functions tests into exposure tests parametrised over the Callable-shaped categories. - Keep skips for modules and Agent instances (still genuinely fail Encodable). - Drop the substring assertion on "lexical variable" from the doc test; assert the variable name appears instead. --- effectful/handlers/llm/completions.py | 127 +++++----------- tests/test_handlers_llm_provider.py | 18 +-- tests/test_handlers_llm_template.py | 210 +++++++++++++------------- 3 files changed, 157 insertions(+), 198 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index d055a110e..94c80338b 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -8,7 +8,6 @@ import string import textwrap import traceback -import types import typing import uuid @@ -179,88 +178,33 @@ def to_feedback_message(self, include_traceback: bool) -> Message: type MessageResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] -class _LexicalVariableTool[T](Tool[[], T]): - """A Tool that returns the current value of a variable captured from - a Template's lexical context. +def _define_lexical_reader( + env: collections.abc.Mapping[str, typing.Any], *, name: str +) -> "Tool[[], typing.Any]": + """Construct a synthetic reader Tool for `env[name]`. - Callers must gate construction on `_is_synthetic_reader_eligible`; - this classmethod assumes the value has already been deemed - schema-encodable and does no further checks. + Raises if the value's `Encodable[nested_type(value)]` schema cannot + be generated. The caller is responsible for catching the failure + and deciding whether to skip the symbol. """ - - @classmethod - def define( - cls, - env: collections.abc.Mapping[str, typing.Any], - *, - name: str, - **kwargs, - ) -> "Tool[[], typing.Any]": - value = env[name] - typ: typing.Any = nested_type(value).value - - def tool_fn(): - return env[name] - - tool_fn.__name__ = name - tool_fn.__qualname__ = name - tool_fn.__module__ = type(value).__module__ - tool_fn.__doc__ = ( - f"Reads the value of lexical variable `{name}` from the " - f"enclosing scope where this Template was defined. Takes " - f"no arguments; returns the current value." - ) - tool_fn.__annotations__ = {"return": typ} - return super().define(tool_fn, **kwargs) - - -# Exceptions raised by the eligibility probe in -# `_is_synthetic_reader_eligible`. `TypeError` / `AttributeError` / -# `NameError` cover empirically-observed `nested_type` failures on -# pathological values (`pytest.mark.parametrize`, method descriptors, -# Operations with unresolved forward-refs); see #673. -_SYNTHETIC_READER_PROBE_FAILURES: tuple[type[BaseException], ...] = ( - pydantic.errors.PydanticSchemaGenerationError, - pydantic.errors.PydanticInvalidForJsonSchema, - pydantic.errors.PydanticUserError, - TypeError, - AttributeError, - NameError, -) - - -# Concrete types that are never wrapped as synthetic readers regardless -# of whether `Encodable[T]` would happen to schematise them. Class -# objects with annotated `__init__` route through `nested_type`'s -# Callable branch and slip past `Encodable[type]`; modules / functions -# / methods / builtins / Agent instances would otherwise be wrapped as -# `Callable[Args, Return]` synthesis schemas. -_NON_READER_TYPES: tuple[type, ...] = ( - type, - types.ModuleType, - types.FunctionType, - types.MethodType, - types.BuiltinFunctionType, - Agent, - Tool, -) - - -def _is_synthetic_reader_eligible(value: typing.Any) -> bool: - """Decide whether `value` should be exposed as a synthetic reader. - - A value is eligible iff it is not an instance of any - `_NON_READER_TYPES` class AND its `Encodable[nested_type(value)]` - schema can be generated. - """ - if isinstance(value, _NON_READER_TYPES): - return False - try: - typ = nested_type(value).value - pydantic.TypeAdapter(Encodable[typ]).json_schema() - except _SYNTHETIC_READER_PROBE_FAILURES: - return False - return True + value = env[name] + typ: typing.Any = nested_type(value).value + # Probe schema generation; raises if `Encodable[typ]` is not implemented. + pydantic.TypeAdapter(Encodable[typ]).json_schema() + + def tool_fn(): + return env[name] + + tool_fn.__name__ = name + tool_fn.__qualname__ = name + tool_fn.__module__ = type(value).__module__ + tool_fn.__doc__ = ( + f"Reads the value of lexical variable `{name}` from the " + f"enclosing scope where this Template was defined. Takes " + f"no arguments; returns the current value." + ) + tool_fn.__annotations__ = {"return": typ} + return Tool.define(tool_fn) def _collect_tools( @@ -278,12 +222,21 @@ def _collect_tools( for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - elif ( - name.isidentifier() - and not name.startswith("__") - and _is_synthetic_reader_eligible(obj) - ): - result[name] = _LexicalVariableTool.define(env, name=name) + elif name.isidentifier() and not name.startswith("__"): + try: + result[name] = _define_lexical_reader(env, name=name) + # `TypeError`/`AttributeError`/`NameError` absorb `nested_type` + # bugs on pathological values (MarkDecorator, method descriptors, + # Operations with unresolved forward-refs); see #673. + except ( + pydantic.errors.PydanticSchemaGenerationError, + pydantic.errors.PydanticInvalidForJsonSchema, + pydantic.errors.PydanticUserError, + TypeError, + AttributeError, + NameError, + ): + continue # Same Tool can appear under multiple names when visible both in the # enclosing scope and via an Agent instance's MRO. Keep only the diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index 3dee9d070..5edd62d1c 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -2228,16 +2228,14 @@ def make_above_threshold() -> Callable[[float], bool]: assert fn(0.5) is False assert fn(threshold) is False - def test_template_skips_lexical_classes(self): - """Classes in the defining scope are NOT exposed as readers. - Locks the skip-via-catch direction for the `type` Encodable - handler: `Hand`/`Finger` produce `PydanticInvalidForJsonSchema` - at the probe and the call site catches them. + def test_template_exposes_lexical_classes(self): + """Classes in the defining scope are exposed as readers via the + broad `Encodable[Callable]` handler. This is the `Hand`/`Finger`/`generate_arm` motivating example - from #497 pinned to its current contract. A follow-up that - adds real `Encodable[type]` impls flips this test to a positive - assertion. + from #497. The pure-Encodable pivot means classes flow through + as Callable-synthesis tools (their `__init__` signature becomes + the schema), so the LLM at least sees that they exist in scope. """ class Finger: @@ -2253,5 +2251,5 @@ def describe_hand_action() -> str: raise NotImplementedError tools = describe_hand_action.tools - assert "Finger" not in tools - assert "Hand" not in tools + assert "Finger" in tools + assert "Hand" in tools diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index b09e27cb8..7d266b161 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1548,9 +1548,8 @@ def test_tool_forward_ref(): import pydantic from effectful.handlers.llm.completions import ( - _LexicalVariableTool, _collect_tools, - _is_synthetic_reader_eligible, + _define_lexical_reader, ) @@ -1584,64 +1583,83 @@ def _example_annotated(x: int) -> int: return x -def test_synthetic_reader_value_reader_returns_live_value(): - """A value-reader returns whatever env[name] currently is. Mutation +def test_synthetic_reader_returns_live_value(): + """A reader returns whatever env[name] currently is. Mutation after reader construction propagates.""" env: dict = {"x": [1, 2, 3]} - tool = _LexicalVariableTool.define(env, name="x") + tool = _define_lexical_reader(env, name="x") assert tool() == [1, 2, 3] env["x"].append(4) assert tool() == [1, 2, 3, 4] -def test_synthetic_reader_value_reader_rebind(): +def test_synthetic_reader_rebind(): """The reader returns the current binding even after rebind. The body reads `env[name]` at call time, not a snapshot.""" env: dict = {"x": 42} - tool = _LexicalVariableTool.define(env, name="x") + tool = _define_lexical_reader(env, name="x") assert tool() == 42 env["x"] = 99 assert tool() == 99 -def test_synthetic_reader_skips_when_name_deleted(): +def test_synthetic_reader_raises_when_name_deleted(): """If env[name] is deleted between collection and call, the reader raises KeyError on direct invocation.""" env: dict = {"x": 42} - tool = _LexicalVariableTool.define(env, name="x") + tool = _define_lexical_reader(env, name="x") del env["x"] with pytest.raises(KeyError): tool() -@pytest.mark.parametrize( - "name,value,should_have_tool", - [ - ("primitive_int", 42, True), - ("primitive_str", "hello", True), - ("list_of_int", [1, 2, 3], True), - ("dict_value", {"a": 1}, True), - ("dataclass_simple", _SimpleDataclass(x=1, y="hello"), True), - ("pydantic_model", _SimpleModel(x=1, y="hello"), True), - # `re.Pattern` and `pathlib.PosixPath` instances ARE exposed by - # Encodable. Pin them here as a regression guard. - ("re_pattern", re.compile(r"x"), True), - ("pathlib_path", Path("/tmp"), True), - # Unencodable categories: probe rejects. - ("opaque", _OpaqueNoEncoder(), False), - ("typevar", typing.TypeVar("T"), False), - ], -) -def test_synthetic_reader_probe_matches_encodable(name, value, should_have_tool): - """The eligibility predicate accepts iff the symbol produces a - Pydantic schema and is not a class/module/callable/Agent/Tool.""" - assert _is_synthetic_reader_eligible(value) is should_have_tool +_PROBE_OK_CASES: list[tuple[str, typing.Any]] = [ + ("primitive_int", 42), + ("primitive_str", "hello"), + ("list_of_int", [1, 2, 3]), + ("dict_value", {"a": 1}), + ("dataclass_simple", _SimpleDataclass(x=1, y="hello")), + ("pydantic_model", _SimpleModel(x=1, y="hello")), + # `re.Pattern` and `pathlib.PosixPath` are encodable in the matrix. + ("re_pattern", re.compile(r"x")), + ("pathlib_path", Path("/tmp")), +] + + +_PROBE_FAIL_CASES: list[tuple[str, typing.Any]] = [ + ("opaque", _OpaqueNoEncoder()), + ("typevar", typing.TypeVar("T")), + ("module", os), +] + + +@pytest.mark.parametrize("name,value", _PROBE_OK_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None) +def test_define_lexical_reader_returns_value(name, value): + """`_define_lexical_reader` builds a Tool when `Encodable[T]` + schema generates; calling it returns the live value.""" + env = {name: value} + tool = _define_lexical_reader(env, name=name) + assert tool() == value + + +@pytest.mark.parametrize("name,value", _PROBE_FAIL_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None) +def test_define_lexical_reader_raises_for_unencodable(name, value): + """Schema-generation failures propagate from `_define_lexical_reader`; + the call site decides whether to skip.""" + env = {name: value} + with pytest.raises(Exception): + _define_lexical_reader(env, name=name) + +def test_collect_tools_skips_unencodable_values(): + """`_collect_tools` catches probe failures and omits the symbol.""" + for name, value in _PROBE_FAIL_CASES: + env = {name: value} + assert name not in _collect_tools(env) -def test_synthetic_readers_yield_to_real_tools(): - """Real Tools/Templates take precedence over same-named synthetic - readers; the `isinstance(obj, Tool | Template)` branch fires first - in `_collect_tools`.""" + +def test_collect_tools_real_tools_take_precedence_over_value_readers(): + """`isinstance(obj, Tool | Template)` fires before the reader branch.""" @Tool.define def shared() -> int: @@ -1651,7 +1669,6 @@ def shared() -> int: env = collections.ChainMap({"shared": shared, "shared_value": 42}) result = _collect_tools(env) assert result["shared"] is shared - assert isinstance(result["shared_value"], _LexicalVariableTool) assert result["shared_value"]() == 42 @@ -1661,78 +1678,70 @@ def test_synthetic_reader_annotation_has_no_free_typevars(): from effectful.internals.unification import freetypevars env = {"x": [1, 2, 3]} - tool = _LexicalVariableTool.define(env, name="x") + tool = _define_lexical_reader(env, name="x") sig = inspect.signature(tool) assert freetypevars(sig.return_annotation) == set() -# ---- Skip-via-catch coverage for the types that preempt Callable ---- - - -def test_lexical_reader_skips_modules(): - """Modules are not exposed as readers.""" - env = {"os": os} - assert not _is_synthetic_reader_eligible(os) - assert "os" not in _collect_tools(env) - - -def test_lexical_reader_skips_user_classes(): - """User-defined classes are not exposed as readers.""" - - class _UserClass: - """Some class.""" - - env = {"_UserClass": _UserClass} - assert not _is_synthetic_reader_eligible(_UserClass) - assert "_UserClass" not in _collect_tools(env) - - -def test_lexical_reader_skips_unannotated_functions(): - """Unannotated functions are not exposed as readers.""" - env = {"_example_unannotated": _example_unannotated} - assert not _is_synthetic_reader_eligible(_example_unannotated) - assert "_example_unannotated" not in _collect_tools(env) - - -def test_lexical_reader_skips_annotated_functions(): - """Annotated functions in lexical scope are skipped by the predicate - just like unannotated ones — the eligibility check is by value type, - not by signature shape, so the synthetic-reader machinery never - duplicates the Callable-synthesis schema that real Tool definitions - rely on.""" - env = {"_example_annotated": _example_annotated} - assert not _is_synthetic_reader_eligible(_example_annotated) - assert "_example_annotated" not in _collect_tools(env) - +# ---- Encodable-passthrough exposure (annotated callables, classes, +# builtins, methods) ---- -def test_lexical_reader_skips_bound_methods(): - """Bound methods (annotated or not) are not exposed as readers.""" +def _example_method_owner_unannotated(): class _C: - def annotated(self) -> int: + def m(self): return 1 + return _C().m - def unannotated(self): - return 2 - inst = _C() - for name, value in [("annotated", inst.annotated), ("unannotated", inst.unannotated)]: - env = {name: value} - assert not _is_synthetic_reader_eligible(value) - assert name not in _collect_tools(env) +def _example_method_owner_annotated(): + class _C: + def m(self) -> int: + return 1 + return _C().m + + +_EXPOSED_THROUGH_ENCODABLE: list[tuple[str, typing.Callable[[], typing.Any]]] = [ + # Annotated function: nested_type → Callable[[int], int]; _pydantic_callable schema. + ("annotated_fn", lambda: _example_annotated), + # Unannotated function: nested_type → function; _pydantic_callable schema. + ("unannotated_fn", lambda: _example_unannotated), + # Plain class: nested_type → type; _pydantic_callable schema. + ("plain_class", lambda: type("Plain", (), {})), + # Builtin function. + ("builtin_fn", lambda: len), + # Bound method, annotated and unannotated. + ("annotated_method", _example_method_owner_annotated), + ("unannotated_method", _example_method_owner_unannotated), +] + + +@pytest.mark.parametrize("name,make_value", _EXPOSED_THROUGH_ENCODABLE, ids=lambda x: x[0] if isinstance(x, tuple) else None) +def test_collect_tools_exposes_callable_shaped_values(name, make_value): + """Annotated callables, classes, builtins, and methods flow through + Encodable's broad Callable handler and become synthesis-shaped tools. + + Templates defined in scopes that don't want this exposure should + explicitly avoid binding these names or filter them at the handler + layer (e.g., `Template.tools` is not the place to second-guess what + Encodable accepts). + """ + value = make_value() + env = {name: value} + assert name in _collect_tools(env) -def test_lexical_reader_skips_builtin_functions(): - """Builtin functions (`len`, etc.) are not exposed as readers.""" - env = {"len": len} - assert not _is_synthetic_reader_eligible(len) - assert "len" not in _collect_tools(env) +def test_collect_tools_skips_modules(): + """Modules naturally fail `Encodable[types.ModuleType]` and get + filtered by the probe-and-catch.""" + env = {"os": os} + assert "os" not in _collect_tools(env) -def test_lexical_reader_skips_agent_instances_but_exposes_their_tools(): - """Agent instances themselves are not exposed as readers, but the - MRO walk in `_collect_tools` continues to expose their contained - Tools under `agent_name__method_name`.""" +def test_collect_tools_skips_agent_instances_but_exposes_their_tools(): + """Agent instances themselves naturally fail the Encodable probe, + but the MRO walk in `_collect_tools` continues to expose their + contained Tools under `agent_name__method_name`.""" class _A(Agent): @Tool.define @@ -1796,14 +1805,13 @@ def t(x: int) -> int: assert "read-only tools for inspecting the lexical" not in t.__system_prompt__ -def test_lexical_reader_doc_describes_lexical_origin(): - """Each `_LexicalVariableTool` carries a per-instance docstring - that tells the LLM it reads a variable from the enclosing scope.""" - env = {"x": [1, 2, 3]} - tool = _LexicalVariableTool.define(env, name="x") +def test_lexical_reader_doc_mentions_name(): + """Each synthetic reader carries a per-instance docstring that + names the captured variable so the LLM can tell readers apart.""" + env = {"my_var": [1, 2, 3]} + tool = _define_lexical_reader(env, name="my_var") assert tool.__doc__ is not None - assert "lexical variable" in tool.__doc__ - assert "`x`" in tool.__doc__ + assert "my_var" in tool.__doc__ def test_template_tools_includes_synthetic_readers_for_locals(): From b4809abb06e002044d8f6d6b2b042dc7c6821ab4 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sun, 7 Jun 2026 17:31:21 -0400 Subject: [PATCH 08/18] Black-format the test parametrize call --- tests/test_handlers_llm_template.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 7d266b161..56d0d6e0c 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1633,7 +1633,9 @@ def test_synthetic_reader_raises_when_name_deleted(): ] -@pytest.mark.parametrize("name,value", _PROBE_OK_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None) +@pytest.mark.parametrize( + "name,value", _PROBE_OK_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None +) def test_define_lexical_reader_returns_value(name, value): """`_define_lexical_reader` builds a Tool when `Encodable[T]` schema generates; calling it returns the live value.""" @@ -1642,7 +1644,11 @@ def test_define_lexical_reader_returns_value(name, value): assert tool() == value -@pytest.mark.parametrize("name,value", _PROBE_FAIL_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None) +@pytest.mark.parametrize( + "name,value", + _PROBE_FAIL_CASES, + ids=lambda x: x[0] if isinstance(x, tuple) else None, +) def test_define_lexical_reader_raises_for_unencodable(name, value): """Schema-generation failures propagate from `_define_lexical_reader`; the call site decides whether to skip.""" @@ -1691,6 +1697,7 @@ def _example_method_owner_unannotated(): class _C: def m(self): return 1 + return _C().m @@ -1698,6 +1705,7 @@ def _example_method_owner_annotated(): class _C: def m(self) -> int: return 1 + return _C().m @@ -1716,7 +1724,11 @@ def m(self) -> int: ] -@pytest.mark.parametrize("name,make_value", _EXPOSED_THROUGH_ENCODABLE, ids=lambda x: x[0] if isinstance(x, tuple) else None) +@pytest.mark.parametrize( + "name,make_value", + _EXPOSED_THROUGH_ENCODABLE, + ids=lambda x: x[0] if isinstance(x, tuple) else None, +) def test_collect_tools_exposes_callable_shaped_values(name, make_value): """Annotated callables, classes, builtins, and methods flow through Encodable's broad Callable handler and become synthesis-shaped tools. From 5b569ab2adcd71015a84c5712f7c5c657c6da61c Mon Sep 17 00:00:00 2001 From: datvo06 Date: Sun, 7 Jun 2026 22:05:40 -0400 Subject: [PATCH 09/18] Drop dunder filter from synthetic-reader gate; link Test A to #674 Two minor adjustments: 1. `_collect_tools` only filters by `name.isidentifier()` now; the `not name.startswith("__")` clause was hygienic, not load-bearing. Module dunders that happen to encode (e.g. `__name__: str`) flow through as readers; module dunders that don't (`__builtins__`, `__class__` of an Agent) still naturally fail the Encodable probe. 2. `test_template_synthesis_uses_lexical_reader` (Test A) is re-skipped pending #674. Initial recording attempt revealed a pre-existing synthesizer bug: `collect_imports` drops `_`-prefixed module imports even when referenced by the emitted variable stubs, so the pytest `request` fixture's `_pytest.fixtures.TopRequest` type crashes mypy_type_check. Skip reason references the issue. --- effectful/handlers/llm/completions.py | 2 +- tests/test_handlers_llm_provider.py | 26 ++++++++++++++++---------- tests/test_handlers_llm_template.py | 8 -------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 94c80338b..97c5e294c 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -222,7 +222,7 @@ def _collect_tools( for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - elif name.isidentifier() and not name.startswith("__"): + elif name.isidentifier(): try: result[name] = _define_lexical_reader(env, name=name) # `TypeError`/`AttributeError`/`NameError` absorb `nested_type` diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index 5edd62d1c..1c9d1d6be 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -2190,12 +2190,13 @@ def report_sum() -> int: @pytest.mark.skip( reason=( - "Fixture recording pending: run `REBUILD_FIXTURES=1 pytest " - "tests/test_handlers_llm_provider.py" - "::TestSyntheticReaderIntegration" - "::test_template_synthesis_uses_lexical_reader` " - "locally with an API key, commit the recorded JSON files, " - "and remove this skip." + "Blocked on synthesizer issue #674: pytest's `request` fixture " + "has type `_pytest.fixtures.TopRequest`, which leaks into the " + "Template's lexical context. `mypy_type_check`'s " + "`collect_imports` drops `_`-prefixed modules even when " + "referenced by emitted stubs, so mypy fails on " + "`Name '_pytest' is not defined`. Unskip and record fixtures " + "with `REBUILD_FIXTURES=true` once #674 lands." ) ) @requires_llm @@ -2211,10 +2212,15 @@ def test_template_synthesis_uses_lexical_reader(self, request): @Template.define def make_above_threshold() -> Callable[[float], bool]: - """Write a Python lambda that returns True iff its float - argument is strictly greater than the value of `threshold`. - Use the `threshold` reader tool to inspect its current value - before emitting the lambda.""" + """Use the `threshold` reader tool to inspect its current + float value, then emit a single Python function definition: + + def above(x: float) -> bool: + return x > + + The function definition MUST be the last and only statement. + Do not emit any other code, no trailing assignment, no + imports, no comments after the function.""" raise NotImplementedError with ( diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 56d0d6e0c..79d3711b7 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1784,14 +1784,6 @@ def test_lexical_reader_exposes_data_values(): assert result["model"]() == env["model"] -def test_lexical_reader_skips_dunders(): - """Dunder-prefixed names like `__builtins__` are never exposed.""" - env = {"__builtins__": __builtins__, "regular": 42} - result = _collect_tools(env) - assert "__builtins__" not in result - assert "regular" in result - - def test_lexical_reader_skips_marker_objects(): """`pytest.mark.parametrize` (a `MarkDecorator`) in env does not abort tool collection; the probe catches the `AttributeError` from From b69af3b5ae2b3fd44acc1ab390b387192882f47f Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 13:19:41 -0400 Subject: [PATCH 10/18] Post-#675/#676 fixup: subclass + snapshot semantics, record Test A After merging master (which now contains the #675 collect_imports fix and the #676 nested_type widening), several touch-ups: - `_define_lexical_reader` is now a `_LexicalVariableTool.define` classmethod on a Tool subclass. The reader closes over the value snapshot rather than `env[name]` because tools are reconstructed fresh each `call_assistant` invocation. - Add a runtime assertion in `_LexicalVariableTool.define` that Tool/Template values must not be re-wrapped as lexical readers. - Narrow `_collect_tools`'s catch tuple to the three Pydantic schema errors plus `TypeError`. `TypeError` stays because the `Encodable[T]` registry raises it from `_pydantic_type_operation`, `_pydantic_type_term`, and `_pydantic_callable`'s incomplete- signature path. `AttributeError`/`NameError` are gone now that #673 widens at the source. - `test_template_synthesis_uses_lexical_reader` (Test A) unskipped; fixtures recorded against gpt-4o-mini and committed. - `test_lexical_reader_skips_marker_objects` removed: with #673, `pytest.mark.parametrize` no longer raises in `nested_type` and the `MarkDecorator` class itself is now schema-encodable through Pydantic's dataclass detection. The test pinned the old catch path; the new contract is "the value flows through". - `test_collect_tools_real_tools_take_precedence_over_value_readers` removed: the invariant (Tools never get wrapped as lexical readers) is now enforced by the runtime assertion inside `_LexicalVariableTool.define`. - `test_collect_tools_skips_agent_instances_but_exposes_their_tools` asserts on values (`inst.t in result.values()`) instead of the internal `agent__method_name` naming convention. - Live-read semantics tests (`returns_live_value`, `rebind`, `raises_when_name_deleted`) replaced with snapshot-semantics counterparts (`returns_captured_value`, `snapshot_survives_rebind`, `snapshot_survives_deletion`). --- effectful/handlers/llm/completions.py | 77 +++++++----- ...emplate_synthesis_uses_lexical_reader.json | 55 +++++++++ ...plate_synthesis_uses_lexical_reader_1.json | 46 +++++++ tests/test_handlers_llm_provider.py | 11 -- tests/test_handlers_llm_template.py | 114 +++++------------- 5 files changed, 175 insertions(+), 128 deletions(-) create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json create mode 100644 tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 97c5e294c..a976efaca 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -178,33 +178,45 @@ def to_feedback_message(self, include_traceback: bool) -> Message: type MessageResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] -def _define_lexical_reader( - env: collections.abc.Mapping[str, typing.Any], *, name: str -) -> "Tool[[], typing.Any]": - """Construct a synthetic reader Tool for `env[name]`. - - Raises if the value's `Encodable[nested_type(value)]` schema cannot - be generated. The caller is responsible for catching the failure - and deciding whether to skip the symbol. +class _LexicalVariableTool[T](Tool[[], T]): + """A zero-arg `Tool` that returns the captured value of a variable + from a `Template`'s lexical context. + + Tools are constructed fresh each `call_assistant` invocation, so + the reader closes over the snapshot `value` rather than the + surrounding `env` — in-place mutation of a mutable value is still + visible (same object reference), but rebinding the source name is + not. """ - value = env[name] - typ: typing.Any = nested_type(value).value - # Probe schema generation; raises if `Encodable[typ]` is not implemented. - pydantic.TypeAdapter(Encodable[typ]).json_schema() - - def tool_fn(): - return env[name] - - tool_fn.__name__ = name - tool_fn.__qualname__ = name - tool_fn.__module__ = type(value).__module__ - tool_fn.__doc__ = ( - f"Reads the value of lexical variable `{name}` from the " - f"enclosing scope where this Template was defined. Takes " - f"no arguments; returns the current value." - ) - tool_fn.__annotations__ = {"return": typ} - return Tool.define(tool_fn) + + @classmethod + def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": # type: ignore[override] + """Construct a synthetic reader Tool that returns `value`. + + Raises if `Encodable[nested_type(value)]` cannot be generated. + The caller is responsible for catching the failure and deciding + whether to skip the symbol. + """ + assert not isinstance(value, Tool), ( + "Tools are real tools and must not be re-wrapped as lexical readers." + ) + typ: typing.Any = nested_type(value).value + # Probe schema generation; raises if `Encodable[typ]` is not implemented. + pydantic.TypeAdapter(Encodable[typ]).json_schema() + + def tool_fn(): + return value + + tool_fn.__name__ = name + tool_fn.__qualname__ = name + tool_fn.__module__ = type(value).__module__ + tool_fn.__doc__ = ( + f"Reads the value of lexical variable `{name}` from the " + f"enclosing scope where this Template was defined. Takes " + f"no arguments; returns the current value." + ) + tool_fn.__annotations__ = {"return": typ} + return super().define(tool_fn) def _collect_tools( @@ -224,17 +236,18 @@ def _collect_tools( result[f"{name}__{attr_name}"] = getattr(obj, attr_name) elif name.isidentifier(): try: - result[name] = _define_lexical_reader(env, name=name) - # `TypeError`/`AttributeError`/`NameError` absorb `nested_type` - # bugs on pathological values (MarkDecorator, method descriptors, - # Operations with unresolved forward-refs); see #673. + result[name] = _LexicalVariableTool.define(obj, name=name) + # `TypeError` joins the three Pydantic errors because the + # `Encodable[T]` registry raises `TypeError` to signal + # "no schema possible" — e.g. `_pydantic_type_operation`, + # `_pydantic_type_term`, and `_pydantic_callable`'s + # incomplete-signature path. Same intent as the Pydantic + # cases, different exception class. except ( pydantic.errors.PydanticSchemaGenerationError, pydantic.errors.PydanticInvalidForJsonSchema, pydantic.errors.PydanticUserError, TypeError, - AttributeError, - NameError, ): continue diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json new file mode 100644 index 000000000..f6a6ce00b --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoXmRMT0faR6nMFTGqu3wIgzXKSki", + "created": 1780938623, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{}", + "name": "threshold" + }, + "id": "call_ypeSxtaJp5QNiymevzksgMSO", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 3897, + "total_tokens": 3906, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json new file mode 100644 index 000000000..ab1f54d01 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json @@ -0,0 +1,46 @@ +{ + "id": "chatcmpl-DoXmTBNxEKjwPzBRAJJUzwj3sKj5r", + "created": 1780938625, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"value\":{\"module_code\":\"def above(x: float) -> bool:\\n return x > 0.85\"}}", + "role": "assistant", + "tool_calls": null, + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 31, + "prompt_tokens": 3916, + "total_tokens": 3947, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 3840, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index 1c9d1d6be..510256dc7 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -2188,17 +2188,6 @@ def report_sum() -> int: assert isinstance(result, int) assert result == sum(_known_data) # 150 - @pytest.mark.skip( - reason=( - "Blocked on synthesizer issue #674: pytest's `request` fixture " - "has type `_pytest.fixtures.TopRequest`, which leaks into the " - "Template's lexical context. `mypy_type_check`'s " - "`collect_imports` drops `_`-prefixed modules even when " - "referenced by emitted stubs, so mypy fails on " - "`Name '_pytest' is not defined`. Unskip and record fixtures " - "with `REBUILD_FIXTURES=true` once #674 lands." - ) - ) @requires_llm def test_template_synthesis_uses_lexical_reader(self, request): """A Template that synthesizes a callable grounds its output diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 79d3711b7..2cffa091e 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1549,7 +1549,7 @@ def test_tool_forward_ref(): from effectful.handlers.llm.completions import ( _collect_tools, - _define_lexical_reader, + _LexicalVariableTool, ) @@ -1583,34 +1583,34 @@ def _example_annotated(x: int) -> int: return x -def test_synthetic_reader_returns_live_value(): - """A reader returns whatever env[name] currently is. Mutation - after reader construction propagates.""" - env: dict = {"x": [1, 2, 3]} - tool = _define_lexical_reader(env, name="x") +def test_synthetic_reader_returns_captured_value(): + """The reader closes over the value snapshot taken at construction + time. In-place mutation of a mutable captured value is visible + (same object reference); rebinding the source name is not.""" + captured: list[int] = [1, 2, 3] + tool = _LexicalVariableTool.define(captured, name="x") assert tool() == [1, 2, 3] - env["x"].append(4) + captured.append(4) assert tool() == [1, 2, 3, 4] -def test_synthetic_reader_rebind(): - """The reader returns the current binding even after rebind. The - body reads `env[name]` at call time, not a snapshot.""" +def test_synthetic_reader_snapshot_survives_rebind(): + """Tools are constructed fresh each `call_assistant` invocation, + so rebinding the source name between construction and invocation + has no effect on the captured value.""" env: dict = {"x": 42} - tool = _define_lexical_reader(env, name="x") - assert tool() == 42 + tool = _LexicalVariableTool.define(env["x"], name="x") env["x"] = 99 - assert tool() == 99 + assert tool() == 42 -def test_synthetic_reader_raises_when_name_deleted(): - """If env[name] is deleted between collection and call, the reader - raises KeyError on direct invocation.""" +def test_synthetic_reader_snapshot_survives_deletion(): + """The closure holds the value directly, so deleting the source + name does not invalidate the reader.""" env: dict = {"x": 42} - tool = _define_lexical_reader(env, name="x") + tool = _LexicalVariableTool.define(env["x"], name="x") del env["x"] - with pytest.raises(KeyError): - tool() + assert tool() == 42 _PROBE_OK_CASES: list[tuple[str, typing.Any]] = [ @@ -1636,11 +1636,10 @@ def test_synthetic_reader_raises_when_name_deleted(): @pytest.mark.parametrize( "name,value", _PROBE_OK_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None ) -def test_define_lexical_reader_returns_value(name, value): - """`_define_lexical_reader` builds a Tool when `Encodable[T]` - schema generates; calling it returns the live value.""" - env = {name: value} - tool = _define_lexical_reader(env, name=name) +def test_lexical_variable_tool_returns_value(name, value): + """`_LexicalVariableTool` builds a Tool when `Encodable[T]` + schema generates; calling it returns the captured value.""" + tool = _LexicalVariableTool.define(value, name=name) assert tool() == value @@ -1649,33 +1648,11 @@ def test_define_lexical_reader_returns_value(name, value): _PROBE_FAIL_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None, ) -def test_define_lexical_reader_raises_for_unencodable(name, value): - """Schema-generation failures propagate from `_define_lexical_reader`; +def test_lexical_variable_tool_raises_for_unencodable(name, value): + """Schema-generation failures propagate from `_LexicalVariableTool`; the call site decides whether to skip.""" - env = {name: value} with pytest.raises(Exception): - _define_lexical_reader(env, name=name) - - -def test_collect_tools_skips_unencodable_values(): - """`_collect_tools` catches probe failures and omits the symbol.""" - for name, value in _PROBE_FAIL_CASES: - env = {name: value} - assert name not in _collect_tools(env) - - -def test_collect_tools_real_tools_take_precedence_over_value_readers(): - """`isinstance(obj, Tool | Template)` fires before the reader branch.""" - - @Tool.define - def shared() -> int: - """Doc.""" - return 1 - - env = collections.ChainMap({"shared": shared, "shared_value": 42}) - result = _collect_tools(env) - assert result["shared"] is shared - assert result["shared_value"]() == 42 + _LexicalVariableTool.define(value, name=name) def test_synthetic_reader_annotation_has_no_free_typevars(): @@ -1683,8 +1660,7 @@ def test_synthetic_reader_annotation_has_no_free_typevars(): produces concrete types — TypeVar substitution is a no-op.""" from effectful.internals.unification import freetypevars - env = {"x": [1, 2, 3]} - tool = _define_lexical_reader(env, name="x") + tool = _LexicalVariableTool.define([1, 2, 3], name="x") sig = inspect.signature(tool) assert freetypevars(sig.return_annotation) == set() @@ -1743,13 +1719,6 @@ def test_collect_tools_exposes_callable_shaped_values(name, make_value): assert name in _collect_tools(env) -def test_collect_tools_skips_modules(): - """Modules naturally fail `Encodable[types.ModuleType]` and get - filtered by the probe-and-catch.""" - env = {"os": os} - assert "os" not in _collect_tools(env) - - def test_collect_tools_skips_agent_instances_but_exposes_their_tools(): """Agent instances themselves naturally fail the Encodable probe, but the MRO walk in `_collect_tools` continues to expose their @@ -1766,7 +1735,7 @@ def t(self) -> int: result = _collect_tools(env) assert "a" not in result # The MRO walk picks up the contained Tool. - assert any(k.startswith("a__") for k in result) + assert inst.t in result.values() def test_lexical_reader_exposes_data_values(): @@ -1784,36 +1753,11 @@ def test_lexical_reader_exposes_data_values(): assert result["model"]() == env["model"] -def test_lexical_reader_skips_marker_objects(): - """`pytest.mark.parametrize` (a `MarkDecorator`) in env does not - abort tool collection; the probe catches the `AttributeError` from - `nested_type`'s Callable branch (`typing.get_overloads` accesses - `__qualname__` which `MarkDecorator` lacks).""" - env = {"mark": pytest.mark.parametrize, "regular": 42} - result = _collect_tools(env) - assert "mark" not in result - assert "regular" in result - - -def test_system_prompt_has_no_lexical_readers_preface(): - """The system prompt no longer carries a global preface about - lexical readers; the per-tool docstring carries the framing.""" - - @Template.define - def t(x: int) -> int: - """Doc.""" - raise NotHandled - - # Module docstring may be present, but the old preface phrasing - # ("You also have access to read-only tools") is not. - assert "read-only tools for inspecting the lexical" not in t.__system_prompt__ - - def test_lexical_reader_doc_mentions_name(): """Each synthetic reader carries a per-instance docstring that names the captured variable so the LLM can tell readers apart.""" env = {"my_var": [1, 2, 3]} - tool = _define_lexical_reader(env, name="my_var") + tool = _LexicalVariableTool.define(env, name="my_var") assert tool.__doc__ is not None assert "my_var" in tool.__doc__ From 324689efa9878dfd611f4713e2a2dfd80fcfb54e Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 13:20:36 -0400 Subject: [PATCH 11/18] stronger test --- tests/test_handlers_llm_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 2cffa091e..b77659918 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1640,7 +1640,7 @@ def test_lexical_variable_tool_returns_value(name, value): """`_LexicalVariableTool` builds a Tool when `Encodable[T]` schema generates; calling it returns the captured value.""" tool = _LexicalVariableTool.define(value, name=name) - assert tool() == value + assert tool() is value @pytest.mark.parametrize( From 8226eca616545339e1ecbd97eef6b0ef6e9dc904 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 13:23:24 -0400 Subject: [PATCH 12/18] Identity-check returned values in lexical-reader exposure test --- tests/test_handlers_llm_template.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index b77659918..27fd9c524 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1655,16 +1655,6 @@ def test_lexical_variable_tool_raises_for_unencodable(name, value): _LexicalVariableTool.define(value, name=name) -def test_synthetic_reader_annotation_has_no_free_typevars(): - """Synthetic reader annotations come from `nested_type`, which - produces concrete types — TypeVar substitution is a no-op.""" - from effectful.internals.unification import freetypevars - - tool = _LexicalVariableTool.define([1, 2, 3], name="x") - sig = inspect.signature(tool) - assert freetypevars(sig.return_annotation) == set() - - # ---- Encodable-passthrough exposure (annotated callables, classes, # builtins, methods) ---- @@ -1739,7 +1729,9 @@ def t(self) -> int: def test_lexical_reader_exposes_data_values(): - """Data-shaped values are exposed as readers (positive contract).""" + """Data-shaped values are exposed as readers (positive contract). + Readers snapshot the value, so calling one returns the *same* + object that was in env at construction time.""" env = { "x": 1, "s": "hello", @@ -1749,8 +1741,8 @@ def test_lexical_reader_exposes_data_values(): } result = _collect_tools(env) assert {"x", "s", "lst", "d", "model"} <= set(result) - assert result["x"]() == 1 - assert result["model"]() == env["model"] + for k, v in env.items(): + assert result[k]() is v def test_lexical_reader_doc_mentions_name(): From d44cc6c3d16d9ff457a02441d6e5e42737017936 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 13:25:12 -0400 Subject: [PATCH 13/18] Drop docstring-substring test; remove now-unused type-ignore --- effectful/handlers/llm/completions.py | 2 +- tests/test_handlers_llm_template.py | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index a976efaca..154c61852 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -190,7 +190,7 @@ class _LexicalVariableTool[T](Tool[[], T]): """ @classmethod - def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": # type: ignore[override] + def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": """Construct a synthetic reader Tool that returns `value`. Raises if `Encodable[nested_type(value)]` cannot be generated. diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 27fd9c524..3833c6ba9 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1745,15 +1745,6 @@ def test_lexical_reader_exposes_data_values(): assert result[k]() is v -def test_lexical_reader_doc_mentions_name(): - """Each synthetic reader carries a per-instance docstring that - names the captured variable so the LLM can tell readers apart.""" - env = {"my_var": [1, 2, 3]} - tool = _LexicalVariableTool.define(env, name="my_var") - assert tool.__doc__ is not None - assert "my_var" in tool.__doc__ - - def test_template_tools_includes_synthetic_readers_for_locals(): """The Template.tools property includes synthetic readers for plain values in lexical scope (e.g., test-local variables).""" From 4631e128b1376d83c5ae4c55c0e45b0804f5d5a3 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 16:06:30 -0400 Subject: [PATCH 14/18] Gate synthetic lexical-reader generation behind LexicalReaders handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per eb8680: needing to coach the LLM around noisy lexical-reader tools in `test_handlers_llm_tool_calling_poem.py` was a smell. Make the generation opt-in instead. - New `expose_lexical_readers()` Operation in `completions.py`, default return `False`. `_collect_tools` gates the reader branch on it. - New `LexicalReaders` ObjectInterpretation overrides the Operation to return `True`; users install it for the call sites where the LLM should see closure state. - Revert the poem prompt-hack: the docstring no longer has to ask the LLM to ignore read-only lexical readers, because they are now off by default. - Test A (`test_template_synthesis_uses_lexical_reader`) and the reader-integration test (`test_llm_reads_lexical_value`) install `LexicalReaders` in their handler stack. Both fixtures re-recorded against gpt-4o-mini. - Template tests that exercise reader generation install the handler. New `test_lexical_readers_off_by_default` and `test_lexical_readers_handler_enables_collection` pin both sides of the gate. - `test_template_method` / `test_template_method_nested_class`: drop the side-note `"local_variable" in tools` assertions; those pinned implicit reader generation. The core method-template tool collection (random, reverse, etc.) is the actual point and still passes. - `test_template_exposes_lexical_classes` now pins both off (default) and on (handler installed) — the #497 motivating example with explicit gate semantics. --- effectful/handlers/llm/completions.py | 34 +++++++++++-- ...gration__test_llm_reads_lexical_value.json | 14 ++--- ...ation__test_llm_reads_lexical_value_1.json | 14 ++--- ...ation__test_llm_reads_lexical_value_2.json | 14 ++--- ...ation__test_llm_reads_lexical_value_3.json | 14 ++--- ...ation__test_llm_reads_lexical_value_4.json | 14 ++--- ...ation__test_llm_reads_lexical_value_5.json | 12 ++--- ...emplate_synthesis_uses_lexical_reader.json | 10 ++-- ...plate_synthesis_uses_lexical_reader_1.json | 10 ++-- tests/test_handlers_llm_provider.py | 31 +++++++---- tests/test_handlers_llm_template.py | 51 ++++++++++++------- tests/test_handlers_llm_tool_calling_poem.py | 5 +- 12 files changed, 135 insertions(+), 88 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 154c61852..8e354a076 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -219,12 +219,40 @@ def tool_fn(): return super().define(tool_fn) +@Operation.define +def expose_lexical_readers() -> bool: + """Effect controlling whether `_collect_tools` builds synthetic + read-only Tools for non-Tool/Template values in a Template's + lexical scope. + + Default behaviour is *off*: only real Tools/Templates/Agents reach + the LLM, and the lexical context is invisible. Install + `LexicalReaders` to flip it on for the call-site where the LLM + should be able to inspect closure state. + """ + return False + + +class LexicalReaders(ObjectInterpretation): + """Handler that enables synthetic lexical-reader generation in + `_collect_tools`. Each plain value in a Template's lexical context + becomes a zero-argument Tool that returns the captured value. + """ + + @implements(expose_lexical_readers) + def _enabled(self) -> bool: + return True + + def _collect_tools( env: collections.abc.Mapping[str, typing.Any], ) -> collections.abc.Mapping[str, Tool]: - """Operations and Templates available as tools, plus synthetic - readers for other lexical symbols. Auto-captured from lexical context.""" + """Operations and Templates available as tools. When + `expose_lexical_readers` is on (see :class:`LexicalReaders`), + plain values in the lexical context are also wrapped as synthetic + read-only tools.""" result: dict[str, Tool] = {} + readers_on = expose_lexical_readers() for name, obj in env.items(): if isinstance(obj, Tool | Template): @@ -234,7 +262,7 @@ def _collect_tools( for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - elif name.isidentifier(): + elif readers_on and name.isidentifier(): try: result[name] = _LexicalVariableTool.define(obj, name=name) # `TypeError` joins the three Pydantic errors because the diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json index 403d200f5..9074bed8f 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json @@ -1,9 +1,9 @@ { - "id": "chatcmpl-Dl3k0fEKWeTJpZDy3rcqAKPKE46Cu", - "created": 1780108168, + "id": "chatcmpl-DoaWCNbxu6WP3Lx3l3Dmkmr5CBxwg", + "created": 1780949148, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", - "system_fingerprint": "fp_df8c8d3b43", + "system_fingerprint": "fp_6c2953e649", "choices": [ { "finish_reason": "tool_calls", @@ -17,7 +17,7 @@ "arguments": "{}", "name": "_known_data" }, - "id": "call_DoVBVCnKiU5k7FTzWRo5nnzm", + "id": "call_nEkf11zJbFZRFcBclPCAvTao", "type": "function" } ], @@ -32,8 +32,8 @@ ], "usage": { "completion_tokens": 11, - "prompt_tokens": 6043, - "total_tokens": 6054, + "prompt_tokens": 3753, + "total_tokens": 3764, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -45,7 +45,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 0, + "cached_tokens": 2816, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json index b6e7666a3..1612450f2 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json @@ -1,9 +1,9 @@ { - "id": "chatcmpl-Dl3k2MUxG3A1S3HrmgwMXFmDNScCt", - "created": 1780108170, + "id": "chatcmpl-DoaWFqScxo1oUrffPxQIyJYv8ENNr", + "created": 1780949151, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", - "system_fingerprint": "fp_df8c8d3b43", + "system_fingerprint": "fp_6c2953e649", "choices": [ { "finish_reason": "tool_calls", @@ -17,7 +17,7 @@ "arguments": "{\"a\":10,\"b\":20}", "name": "add_numbers" }, - "id": "call_vypFiLTjH5MEpMKSL3VLoD4z", + "id": "call_TABbctvXDmYyjourZxIgLFo1", "type": "function" } ], @@ -32,8 +32,8 @@ ], "usage": { "completion_tokens": 18, - "prompt_tokens": 6077, - "total_tokens": 6095, + "prompt_tokens": 3787, + "total_tokens": 3805, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -45,7 +45,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 6016, + "cached_tokens": 3712, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json index 5683bba8f..d3d91c23b 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json @@ -1,9 +1,9 @@ { - "id": "chatcmpl-Dl3k5bfkN00lZh8L5D82Cr9YhOJsA", - "created": 1780108173, + "id": "chatcmpl-DoaWHTP5RK0QyZdDhiHbaUNkv3IfS", + "created": 1780949153, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", - "system_fingerprint": "fp_df8c8d3b43", + "system_fingerprint": "fp_6c2953e649", "choices": [ { "finish_reason": "tool_calls", @@ -17,7 +17,7 @@ "arguments": "{\"a\":30,\"b\":30}", "name": "add_numbers" }, - "id": "call_EmO3tVtqZBqdL6K4dUqLmfgn", + "id": "call_SpMwdgqjznHPUWgE19dxgSnc", "type": "function" } ], @@ -32,8 +32,8 @@ ], "usage": { "completion_tokens": 18, - "prompt_tokens": 6104, - "total_tokens": 6122, + "prompt_tokens": 3814, + "total_tokens": 3832, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -45,7 +45,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 6016, + "cached_tokens": 3712, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json index a1c38794e..8a8408247 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json @@ -1,9 +1,9 @@ { - "id": "chatcmpl-Dl3k7gKpQZx4an1oAXAhdYViG1NjR", - "created": 1780108175, + "id": "chatcmpl-DoaWJQAi5ihtJlJL7algRwDhzcgcx", + "created": 1780949155, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", - "system_fingerprint": "fp_df8c8d3b43", + "system_fingerprint": "fp_6c2953e649", "choices": [ { "finish_reason": "tool_calls", @@ -17,7 +17,7 @@ "arguments": "{\"a\":60,\"b\":40}", "name": "add_numbers" }, - "id": "call_WhTflV4rMdUJUJOY2qqOHv7x", + "id": "call_IZ66N2KVq8wyYt3mAOIqfxW0", "type": "function" } ], @@ -32,8 +32,8 @@ ], "usage": { "completion_tokens": 18, - "prompt_tokens": 6131, - "total_tokens": 6149, + "prompt_tokens": 3841, + "total_tokens": 3859, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -45,7 +45,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 6016, + "cached_tokens": 3712, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json index 08c5bc100..3da2320ba 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json @@ -1,9 +1,9 @@ { - "id": "chatcmpl-Dl3k86CG3iPs5P7110XgQ3Eso2CfZ", - "created": 1780108176, + "id": "chatcmpl-DoaWLlSqQM5WHmGa4dcjSRypJ0EXu", + "created": 1780949157, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", - "system_fingerprint": "fp_df8c8d3b43", + "system_fingerprint": "fp_6c2953e649", "choices": [ { "finish_reason": "tool_calls", @@ -17,7 +17,7 @@ "arguments": "{\"a\":100,\"b\":50}", "name": "add_numbers" }, - "id": "call_LTy1WHCfWTFnwZ5GSAUElfDA", + "id": "call_0unsPGaZPBHhTLObCfc5zAgf", "type": "function" } ], @@ -32,8 +32,8 @@ ], "usage": { "completion_tokens": 18, - "prompt_tokens": 6158, - "total_tokens": 6176, + "prompt_tokens": 3868, + "total_tokens": 3886, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -45,7 +45,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 6016, + "cached_tokens": 3840, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json index e1fe0b3c4..35aa33d94 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json @@ -1,9 +1,9 @@ { - "id": "chatcmpl-Dl3k9lFSuyqAQbfM8J1H3BQmgYopC", - "created": 1780108177, + "id": "chatcmpl-DoaWNFKpBO8KO9IuPrwpMf2aDayBQ", + "created": 1780949159, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", - "system_fingerprint": "fp_df8c8d3b43", + "system_fingerprint": "fp_6c2953e649", "choices": [ { "finish_reason": "stop", @@ -23,8 +23,8 @@ ], "usage": { "completion_tokens": 11, - "prompt_tokens": 6185, - "total_tokens": 6196, + "prompt_tokens": 3895, + "total_tokens": 3906, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -36,7 +36,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 6144, + "cached_tokens": 3840, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json index f6a6ce00b..0d50b8ef2 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json @@ -1,6 +1,6 @@ { - "id": "chatcmpl-DoXmRMT0faR6nMFTGqu3wIgzXKSki", - "created": 1780938623, + "id": "chatcmpl-DoaVmdWxIVHC9IAfb8gM7TohRDtBb", + "created": 1780949122, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "system_fingerprint": "fp_6c2953e649", @@ -17,7 +17,7 @@ "arguments": "{}", "name": "threshold" }, - "id": "call_ypeSxtaJp5QNiymevzksgMSO", + "id": "call_LZbU6rs5AtcMphaKxdwuERBs", "type": "function" } ], @@ -32,8 +32,8 @@ ], "usage": { "completion_tokens": 9, - "prompt_tokens": 3897, - "total_tokens": 3906, + "prompt_tokens": 3937, + "total_tokens": 3946, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json index ab1f54d01..571902fa8 100644 --- a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json @@ -1,6 +1,6 @@ { - "id": "chatcmpl-DoXmTBNxEKjwPzBRAJJUzwj3sKj5r", - "created": 1780938625, + "id": "chatcmpl-DoaVqPg5WV0Vjsp6Mklax4LPZSAD1", + "created": 1780949126, "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "system_fingerprint": "fp_6c2953e649", @@ -23,8 +23,8 @@ ], "usage": { "completion_tokens": 31, - "prompt_tokens": 3916, - "total_tokens": 3947, + "prompt_tokens": 3956, + "total_tokens": 3987, "completion_tokens_details": { "accepted_prediction_tokens": 0, "audio_tokens": 0, @@ -36,7 +36,7 @@ }, "prompt_tokens_details": { "audio_tokens": 0, - "cached_tokens": 3840, + "cached_tokens": 0, "text_tokens": null, "image_tokens": null, "video_tokens": null diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index 510256dc7..198b4f106 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -28,6 +28,7 @@ from effectful.handlers.llm import Agent, Template from effectful.handlers.llm.completions import ( DecodedToolCall, + LexicalReaders, LiteLLMProvider, ResultDecodingError, RetryLLMHandler, @@ -2182,7 +2183,10 @@ def report_sum() -> int: then return their sum as an integer.""" raise NotImplementedError - with handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)): + with ( + handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)), + handler(LexicalReaders()), + ): result = report_sum() assert isinstance(result, int) @@ -2216,6 +2220,7 @@ def above(x: float) -> bool: handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)), handler(UnsafeEvalProvider()), handler(LimitLLMCallsHandler(max_calls=4)), + handler(LexicalReaders()), ): fn = make_above_threshold() @@ -2224,13 +2229,11 @@ def above(x: float) -> bool: assert fn(threshold) is False def test_template_exposes_lexical_classes(self): - """Classes in the defining scope are exposed as readers via the - broad `Encodable[Callable]` handler. - - This is the `Hand`/`Finger`/`generate_arm` motivating example - from #497. The pure-Encodable pivot means classes flow through - as Callable-synthesis tools (their `__init__` signature becomes - the schema), so the LLM at least sees that they exist in scope. + """When `LexicalReaders` is installed, classes in the defining + scope are exposed as readers via the broad `Encodable[Callable]` + handler — the `Hand`/`Finger`/`generate_arm` motivating example + from #497. Without the handler the readers are gated off; this + test pins both contracts. """ class Finger: @@ -2245,6 +2248,12 @@ def describe_hand_action() -> str: """Doc.""" raise NotImplementedError - tools = describe_hand_action.tools - assert "Finger" in tools - assert "Hand" in tools + # Off by default. + assert "Finger" not in describe_hand_action.tools + assert "Hand" not in describe_hand_action.tools + + # On under the handler. + with handler(LexicalReaders()): + tools = describe_hand_action.tools + assert "Finger" in tools + assert "Hand" in tools diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 3833c6ba9..26b43f9d9 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -778,8 +778,6 @@ def f(self) -> int: assert a.random in a.f.tools.values() # f is the template itself — found via self but correctly removed (non-recursive) assert a.f not in a.f.tools.values() - # local_variable is now exposed as a synthetic reader (PR #545 finish-up). - assert "local_variable" in a.f.__context__ and "local_variable" in a.f.tools assert any(t() == 4 for t in a.f.tools.values() if t is a.random) class B(A): @@ -796,8 +794,6 @@ def reverse(self, s: str) -> str: assert isinstance(b.f, Template) assert b.random in b.f.tools.values() assert b.reverse in b.f.tools.values() - # local_variable is now exposed as a synthetic reader (PR #545 finish-up). - assert "local_variable" in b.f.__context__ and "local_variable" in a.f.tools def test_template_method_nested_class(): @@ -828,8 +824,6 @@ def f(self) -> int: assert "random" in a.f.tools # f is the template itself — found via self but correctly removed (non-recursive) assert "f" not in a.f.tools - # local_variable is now exposed as a synthetic reader (PR #545 finish-up). - assert "local_variable" in a.f.__context__ and "local_variable" in a.f.tools assert a.f.tools["random"]() == 4 @@ -1548,6 +1542,7 @@ def test_tool_forward_ref(): import pydantic from effectful.handlers.llm.completions import ( + LexicalReaders, _collect_tools, _LexicalVariableTool, ) @@ -1696,17 +1691,13 @@ def m(self) -> int: ids=lambda x: x[0] if isinstance(x, tuple) else None, ) def test_collect_tools_exposes_callable_shaped_values(name, make_value): - """Annotated callables, classes, builtins, and methods flow through - Encodable's broad Callable handler and become synthesis-shaped tools. - - Templates defined in scopes that don't want this exposure should - explicitly avoid binding these names or filter them at the handler - layer (e.g., `Template.tools` is not the place to second-guess what - Encodable accepts). - """ + """With `LexicalReaders` installed, annotated callables, classes, + builtins, and methods flow through Encodable's broad Callable + handler and become synthesis-shaped tools.""" value = make_value() env = {name: value} - assert name in _collect_tools(env) + with handler(LexicalReaders()): + assert name in _collect_tools(env) def test_collect_tools_skips_agent_instances_but_exposes_their_tools(): @@ -1739,7 +1730,8 @@ def test_lexical_reader_exposes_data_values(): "d": {"k": 1}, "model": _SimpleModel(x=1, y="hi"), } - result = _collect_tools(env) + with handler(LexicalReaders()): + result = _collect_tools(env) assert {"x", "s", "lst", "d", "model"} <= set(result) for k, v in env.items(): assert result[k]() is v @@ -1747,7 +1739,7 @@ def test_lexical_reader_exposes_data_values(): def test_template_tools_includes_synthetic_readers_for_locals(): """The Template.tools property includes synthetic readers for - plain values in lexical scope (e.g., test-local variables).""" + plain values in lexical scope when `LexicalReaders` is installed.""" _test_data = [10, 20, 30] @Template.define @@ -1755,5 +1747,26 @@ def t() -> int: """Doc.""" raise NotHandled - assert "_test_data" in t.tools - assert t.tools["_test_data"]() == [10, 20, 30] + with handler(LexicalReaders()): + tools = t.tools + assert "_test_data" in tools + assert tools["_test_data"]() == [10, 20, 30] + + +def test_lexical_readers_off_by_default(): + """Without `LexicalReaders` installed, `_collect_tools` does not + wrap plain values as synthetic readers.""" + env = {"x": 42, "s": "hello"} + result = _collect_tools(env) + assert "x" not in result + assert "s" not in result + + +def test_lexical_readers_handler_enables_collection(): + """Installing `LexicalReaders` flips the gate; the same values are + exposed as zero-arg reader tools.""" + env = {"x": 42, "s": "hello"} + with handler(LexicalReaders()): + result = _collect_tools(env) + assert result["x"]() == 42 + assert result["s"]() == "hello" diff --git a/tests/test_handlers_llm_tool_calling_poem.py b/tests/test_handlers_llm_tool_calling_poem.py index c1227cf70..9ff8587f6 100644 --- a/tests/test_handlers_llm_tool_calling_poem.py +++ b/tests/test_handlers_llm_tool_calling_poem.py @@ -92,10 +92,7 @@ def generate_good_poem(topic: str) -> Poem: Keep iterating until evaluate_poem_tool returns GOOD. Return your final poem as JSON with 'content' and 'form' fields. - Do not call any tool other than evaluate_poem_tool. In particular, - do not call generate_good_poem and do not call any read-only lexical - reader that may appear in the tool list. Those are not relevant to - this task. + Do not call the 'generate_good_poem' tool. """ raise NotHandled From bbe3c33cc1821d3fd161bbfc1e2e085caf46ba7e Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 16:18:50 -0400 Subject: [PATCH 15/18] remove weak test --- tests/test_handlers_llm_template.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 26b43f9d9..c95f8377d 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1621,13 +1621,6 @@ def test_synthetic_reader_snapshot_survives_deletion(): ] -_PROBE_FAIL_CASES: list[tuple[str, typing.Any]] = [ - ("opaque", _OpaqueNoEncoder()), - ("typevar", typing.TypeVar("T")), - ("module", os), -] - - @pytest.mark.parametrize( "name,value", _PROBE_OK_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None ) @@ -1638,18 +1631,6 @@ def test_lexical_variable_tool_returns_value(name, value): assert tool() is value -@pytest.mark.parametrize( - "name,value", - _PROBE_FAIL_CASES, - ids=lambda x: x[0] if isinstance(x, tuple) else None, -) -def test_lexical_variable_tool_raises_for_unencodable(name, value): - """Schema-generation failures propagate from `_LexicalVariableTool`; - the call site decides whether to skip.""" - with pytest.raises(Exception): - _LexicalVariableTool.define(value, name=name) - - # ---- Encodable-passthrough exposure (annotated callables, classes, # builtins, methods) ---- From ea0130844beee672ab5c832fd82976f0a35b2c35 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Mon, 8 Jun 2026 16:19:05 -0400 Subject: [PATCH 16/18] Format --- tests/test_handlers_llm_template.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index c95f8377d..5cfa49708 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1534,7 +1534,6 @@ def test_tool_forward_ref(): # Synthetic readers for lexical context (PR #545 finish-up) # --------------------------------------------------------------------------- -import os import re import typing from pathlib import Path From 74c1ae235df56ec14c4cb1e507a2b349dba7055c Mon Sep 17 00:00:00 2001 From: datvo06 Date: Tue, 9 Jun 2026 10:55:00 -0400 Subject: [PATCH 17/18] Promote collect_tools to a public Operation; LexicalReaders overrides it Replaces the narrow `expose_lexical_readers` bool gate with a general extension point: `collect_tools` is now an `Operation` whose default rule does the minimal Tool/Template/Agent collection (including the same-Tool-different-name dedup). Handlers override it to customise what gets exposed to the LLM. `LexicalReaders` becomes an `ObjectInterpretation` that `@implements(collect_tools)`: call `fwd()` to get the base set, then add a synthetic `_LexicalVariableTool` for each plain value in env whose `Encodable[T]` accepts it. Renames the internal `_collect_tools` to public `collect_tools` everywhere: `Template.tools`, `call_assistant`, and the template tests. No behaviour change beyond the design promotion. --- effectful/handlers/llm/completions.py | 83 ++++++++++++++------------- effectful/handlers/llm/template.py | 4 +- tests/test_handlers_llm_template.py | 36 ++---------- 3 files changed, 48 insertions(+), 75 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 8e354a076..87ab9874f 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -220,39 +220,20 @@ def tool_fn(): @Operation.define -def expose_lexical_readers() -> bool: - """Effect controlling whether `_collect_tools` builds synthetic - read-only Tools for non-Tool/Template values in a Template's - lexical scope. - - Default behaviour is *off*: only real Tools/Templates/Agents reach - the LLM, and the lexical context is invisible. Install - `LexicalReaders` to flip it on for the call-site where the LLM - should be able to inspect closure state. - """ - return False +def collect_tools( + env: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.Mapping[str, Tool]: + """Return the tools available to a Template given its lexical context. + Default rule: real `Tool` and `Template` values bound directly in + `env`, plus `Tool` methods discovered through the MRO of any + `Agent` instance in `env`. Same-Tool-under-different-names is + deduped so each Tool appears exactly once. -class LexicalReaders(ObjectInterpretation): - """Handler that enables synthetic lexical-reader generation in - `_collect_tools`. Each plain value in a Template's lexical context - becomes a zero-argument Tool that returns the captured value. + Handlers (see :class:`LexicalReaders`) may override this to add + synthetic readers, hide tools, etc. """ - - @implements(expose_lexical_readers) - def _enabled(self) -> bool: - return True - - -def _collect_tools( - env: collections.abc.Mapping[str, typing.Any], -) -> collections.abc.Mapping[str, Tool]: - """Operations and Templates available as tools. When - `expose_lexical_readers` is on (see :class:`LexicalReaders`), - plain values in the lexical context are also wrapped as synthetic - read-only tools.""" result: dict[str, Tool] = {} - readers_on = expose_lexical_readers() for name, obj in env.items(): if isinstance(obj, Tool | Template): @@ -262,7 +243,36 @@ def _collect_tools( for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - elif readers_on and name.isidentifier(): + + # Same Tool can appear under multiple names when visible both in the + # enclosing scope and via an Agent instance's MRO. Keep only the + # last name for each unique tool object. + tool2name = {tool: name for name, tool in sorted(result.items())} + for name, tool in tuple(result.items()): + if tool2name[tool] != name: + del result[name] + + return result + + +class LexicalReaders(ObjectInterpretation): + """Override `collect_tools` to also expose plain values from the + lexical context as zero-argument read-only Tools. Each non-Tool, + non-Template, non-Agent value bound to a valid identifier is + wrapped via `_LexicalVariableTool` if `Encodable[T]` accepts it; + schema-generation failures cause the symbol to be skipped. + """ + + @implements(collect_tools) + def _collect( + self, env: collections.abc.Mapping[str, typing.Any] + ) -> collections.abc.Mapping[str, Tool]: + result = dict(fwd()) + for name, obj in env.items(): + if name in result or isinstance(obj, Tool | Template | Agent): + continue + if not name.isidentifier(): + continue try: result[name] = _LexicalVariableTool.define(obj, name=name) # `TypeError` joins the three Pydantic errors because the @@ -278,16 +288,7 @@ def _collect_tools( TypeError, ): continue - - # Same Tool can appear under multiple names when visible both in the - # enclosing scope and via an Agent instance's MRO. Keep only the - # last name for each unique tool object. - tool2name = {tool: name for name, tool in sorted(result.items())} - for name, tool in tuple(result.items()): - if tool2name[tool] != name: - del result[name] - - return result + return result @Operation.define @@ -324,7 +325,7 @@ def call_assistant[T]( ResultDecodingError: If the result cannot be decoded. The error includes the raw assistant message for retry handling. """ - tools = dict(_collect_tools(env)) + tools = dict(collect_tools(env)) tool_specs = { k: typing.cast( pydantic.TypeAdapter[typing.Any], diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 15f991b77..cffda38cf 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -217,9 +217,9 @@ def __prompt_template__(self) -> str: def tools(self) -> Mapping[str, Tool]: """Operations and Templates available as tools, plus synthetic readers for other lexical symbols. Auto-captured from lexical context.""" - from effectful.handlers.llm.completions import _collect_tools + from effectful.handlers.llm.completions import collect_tools - result = dict(_collect_tools(self.__context__)) + result = dict(collect_tools(self.__context__)) # We remove the template itself from the tool map unless it is explicitly # marked as recursive (see test_template_method, test_template_method_nested_class). diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 5cfa49708..4938e8d75 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1542,8 +1542,8 @@ def test_tool_forward_ref(): from effectful.handlers.llm.completions import ( LexicalReaders, - _collect_tools, _LexicalVariableTool, + collect_tools, ) @@ -1677,26 +1677,7 @@ def test_collect_tools_exposes_callable_shaped_values(name, make_value): value = make_value() env = {name: value} with handler(LexicalReaders()): - assert name in _collect_tools(env) - - -def test_collect_tools_skips_agent_instances_but_exposes_their_tools(): - """Agent instances themselves naturally fail the Encodable probe, - but the MRO walk in `_collect_tools` continues to expose their - contained Tools under `agent_name__method_name`.""" - - class _A(Agent): - @Tool.define - def t(self) -> int: - """Doc.""" - return 7 - - inst = _A() - env = {"a": inst} - result = _collect_tools(env) - assert "a" not in result - # The MRO walk picks up the contained Tool. - assert inst.t in result.values() + assert name in collect_tools(env) def test_lexical_reader_exposes_data_values(): @@ -1711,7 +1692,7 @@ def test_lexical_reader_exposes_data_values(): "model": _SimpleModel(x=1, y="hi"), } with handler(LexicalReaders()): - result = _collect_tools(env) + result = collect_tools(env) assert {"x", "s", "lst", "d", "model"} <= set(result) for k, v in env.items(): assert result[k]() is v @@ -1733,20 +1714,11 @@ def t() -> int: assert tools["_test_data"]() == [10, 20, 30] -def test_lexical_readers_off_by_default(): - """Without `LexicalReaders` installed, `_collect_tools` does not - wrap plain values as synthetic readers.""" - env = {"x": 42, "s": "hello"} - result = _collect_tools(env) - assert "x" not in result - assert "s" not in result - - def test_lexical_readers_handler_enables_collection(): """Installing `LexicalReaders` flips the gate; the same values are exposed as zero-arg reader tools.""" env = {"x": 42, "s": "hello"} with handler(LexicalReaders()): - result = _collect_tools(env) + result = collect_tools(env) assert result["x"]() == 42 assert result["s"]() == "hello" From c19d6f9456c99f9ed251f65a614c2fa251d7544b Mon Sep 17 00:00:00 2001 From: datvo06 Date: Tue, 9 Jun 2026 10:59:59 -0400 Subject: [PATCH 18/18] Drop redundant Tool|Template|Agent isinstance check in LexicalReaders --- effectful/handlers/llm/completions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 87ab9874f..30fad97ab 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -269,9 +269,7 @@ def _collect( ) -> collections.abc.Mapping[str, Tool]: result = dict(fwd()) for name, obj in env.items(): - if name in result or isinstance(obj, Tool | Template | Agent): - continue - if not name.isidentifier(): + if name in result or not name.isidentifier(): continue try: result[name] = _LexicalVariableTool.define(obj, name=name)