diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 2e169d932..30fad97ab 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -178,29 +178,75 @@ def to_feedback_message(self, include_traceback: bool) -> Message: type MessageResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] -def _collect_tools( +class _LexicalVariableTool[T](Tool[[], T]): + """A zero-arg `Tool` that returns the captured value of a variable + from a `Template`'s lexical context. + + Tools are constructed fresh each `call_assistant` invocation, so + the reader closes over the snapshot `value` rather than the + surrounding `env` — in-place mutation of a mutable value is still + visible (same object reference), but rebinding the source name is + not. + """ + + @classmethod + def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": + """Construct a synthetic reader Tool that returns `value`. + + Raises if `Encodable[nested_type(value)]` cannot be generated. + The caller is responsible for catching the failure and deciding + whether to skip the symbol. + """ + assert not isinstance(value, Tool), ( + "Tools are real tools and must not be re-wrapped as lexical readers." + ) + typ: typing.Any = nested_type(value).value + # Probe schema generation; raises if `Encodable[typ]` is not implemented. + pydantic.TypeAdapter(Encodable[typ]).json_schema() + + def tool_fn(): + return value + + tool_fn.__name__ = name + tool_fn.__qualname__ = name + tool_fn.__module__ = type(value).__module__ + tool_fn.__doc__ = ( + f"Reads the value of lexical variable `{name}` from the " + f"enclosing scope where this Template was defined. Takes " + f"no arguments; returns the current value." + ) + tool_fn.__annotations__ = {"return": typ} + return super().define(tool_fn) + + +@Operation.define +def collect_tools( env: collections.abc.Mapping[str, typing.Any], ) -> collections.abc.Mapping[str, Tool]: - """Operations and Templates available as tools. Auto-capture from lexical context.""" - result = {} + """Return the tools available to a Template given its lexical context. + + Default rule: real `Tool` and `Template` values bound directly in + `env`, plus `Tool` methods discovered through the MRO of any + `Agent` instance in `env`. Same-Tool-under-different-names is + deduped so each Tool appears exactly once. + + Handlers (see :class:`LexicalReaders`) may override this to add + synthetic readers, hide tools, etc. + """ + result: dict[str, Tool] = {} for name, obj in env.items(): - # Collect tools directly in context if isinstance(obj, Tool | Template): result[name] = obj - - # Collect tools as methods on Agent instances in context elif isinstance(obj, Agent): for cls in type(obj).__mro__: for attr_name in vars(cls): if isinstance(getattr(obj, attr_name), Tool): result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - # The same Tool can appear under multiple names when it is both - # visible in the enclosing scope *and* discovered via an Agent - # instance's MRO. Since Tools are hashable Operations and - # instance-method Tools are cached per instance, we keep only - # the last name for each unique tool object. + # Same Tool can appear under multiple names when visible both in the + # enclosing scope and via an Agent instance's MRO. Keep only the + # last name for each unique tool object. tool2name = {tool: name for name, tool in sorted(result.items())} for name, tool in tuple(result.items()): if tool2name[tool] != name: @@ -209,6 +255,40 @@ def _collect_tools( return result +class LexicalReaders(ObjectInterpretation): + """Override `collect_tools` to also expose plain values from the + lexical context as zero-argument read-only Tools. Each non-Tool, + non-Template, non-Agent value bound to a valid identifier is + wrapped via `_LexicalVariableTool` if `Encodable[T]` accepts it; + schema-generation failures cause the symbol to be skipped. + """ + + @implements(collect_tools) + def _collect( + self, env: collections.abc.Mapping[str, typing.Any] + ) -> collections.abc.Mapping[str, Tool]: + result = dict(fwd()) + for name, obj in env.items(): + if name in result or not name.isidentifier(): + continue + try: + result[name] = _LexicalVariableTool.define(obj, name=name) + # `TypeError` joins the three Pydantic errors because the + # `Encodable[T]` registry raises `TypeError` to signal + # "no schema possible" — e.g. `_pydantic_type_operation`, + # `_pydantic_type_term`, and `_pydantic_callable`'s + # incomplete-signature path. Same intent as the Pydantic + # cases, different exception class. + except ( + pydantic.errors.PydanticSchemaGenerationError, + pydantic.errors.PydanticInvalidForJsonSchema, + pydantic.errors.PydanticUserError, + TypeError, + ): + continue + return result + + @Operation.define @functools.wraps(litellm.completion) def completion(*args, **kwargs) -> typing.Any: @@ -243,7 +323,7 @@ def call_assistant[T]( ResultDecodingError: If the result cannot be decoded. The error includes the raw assistant message for retry handling. """ - tools = _collect_tools(env) + tools = dict(collect_tools(env)) tool_specs = { k: typing.cast( pydantic.TypeAdapter[typing.Any], diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index f56d6fad7..cffda38cf 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -215,15 +215,15 @@ def __prompt_template__(self) -> str: @property def tools(self) -> Mapping[str, Tool]: - """Operations and Templates available as tools. Auto-capture from lexical context.""" - from effectful.handlers.llm.completions import _collect_tools + """Operations and Templates available as tools, plus synthetic + readers for other lexical symbols. Auto-captured from lexical context.""" + from effectful.handlers.llm.completions import collect_tools - result = _collect_tools(self.__context__) + result = dict(collect_tools(self.__context__)) # We remove the template itself from the tool map unless it is explicitly # marked as recursive (see test_template_method, test_template_method_nested_class). if not _is_recursive_signature(self.__signature__): - result = dict(result) # copy to allow mutation for name, tool in tuple(result.items()): if tool is self: del result[name] @@ -312,7 +312,7 @@ def define[**Q, V]( op = super().define(default, *args, **kwargs) op.__context__ = context # type: ignore[attr-defined] mod = inspect.getmodule(_fn) - op.__system_prompt__ = inspect.getdoc(mod) if mod is not None else "" # type: ignore[attr-defined] + op.__system_prompt__ = inspect.getdoc(mod) or "" # type: ignore[attr-defined] # Keep validation on original define-time callables, but skip the bound wrapper path. # to avoid dropping `self` from the signature and falsely rejecting valid prompt fields like `{self.name}`. is_bound_wrapper = ( diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json new file mode 100644 index 000000000..9074bed8f --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoaWCNbxu6WP3Lx3l3Dmkmr5CBxwg", + "created": 1780949148, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{}", + "name": "_known_data" + }, + "id": "call_nEkf11zJbFZRFcBclPCAvTao", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 11, + "prompt_tokens": 3753, + "total_tokens": 3764, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 2816, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json new file mode 100644 index 000000000..1612450f2 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoaWFqScxo1oUrffPxQIyJYv8ENNr", + "created": 1780949151, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":10,\"b\":20}", + "name": "add_numbers" + }, + "id": "call_TABbctvXDmYyjourZxIgLFo1", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 3787, + "total_tokens": 3805, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 3712, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json new file mode 100644 index 000000000..d3d91c23b --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoaWHTP5RK0QyZdDhiHbaUNkv3IfS", + "created": 1780949153, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":30,\"b\":30}", + "name": "add_numbers" + }, + "id": "call_SpMwdgqjznHPUWgE19dxgSnc", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 3814, + "total_tokens": 3832, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 3712, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json new file mode 100644 index 000000000..8a8408247 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoaWJQAi5ihtJlJL7algRwDhzcgcx", + "created": 1780949155, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":60,\"b\":40}", + "name": "add_numbers" + }, + "id": "call_IZ66N2KVq8wyYt3mAOIqfxW0", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 3841, + "total_tokens": 3859, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 3712, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json new file mode 100644 index 000000000..3da2320ba --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoaWLlSqQM5WHmGa4dcjSRypJ0EXu", + "created": 1780949157, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"a\":100,\"b\":50}", + "name": "add_numbers" + }, + "id": "call_0unsPGaZPBHhTLObCfc5zAgf", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 18, + "prompt_tokens": 3868, + "total_tokens": 3886, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 3840, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json new file mode 100644 index 000000000..35aa33d94 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json @@ -0,0 +1,46 @@ +{ + "id": "chatcmpl-DoaWNFKpBO8KO9IuPrwpMf2aDayBQ", + "created": 1780949159, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"value\":150}", + "role": "assistant", + "tool_calls": null, + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 11, + "prompt_tokens": 3895, + "total_tokens": 3906, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 3840, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json new file mode 100644 index 000000000..0d50b8ef2 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json @@ -0,0 +1,55 @@ +{ + "id": "chatcmpl-DoaVmdWxIVHC9IAfb8gM7TohRDtBb", + "created": 1780949122, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{}", + "name": "threshold" + }, + "id": "call_LZbU6rs5AtcMphaKxdwuERBs", + "type": "function" + } + ], + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 3937, + "total_tokens": 3946, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json new file mode 100644 index 000000000..571902fa8 --- /dev/null +++ b/tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json @@ -0,0 +1,46 @@ +{ + "id": "chatcmpl-DoaVqPg5WV0Vjsp6Mklax4LPZSAD1", + "created": 1780949126, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_6c2953e649", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"value\":{\"module_code\":\"def above(x: float) -> bool:\\n return x > 0.85\"}}", + "role": "assistant", + "tool_calls": null, + "function_call": null, + "provider_specific_fields": { + "refusal": null + }, + "annotations": [] + }, + "provider_specific_fields": {} + } + ], + "usage": { + "completion_tokens": 31, + "prompt_tokens": 3956, + "total_tokens": 3987, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null + } + }, + "service_tier": "default" +} \ No newline at end of file diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index b56fd7bbd..198b4f106 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -28,6 +28,7 @@ from effectful.handlers.llm import Agent, Template from effectful.handlers.llm.completions import ( DecodedToolCall, + LexicalReaders, LiteLLMProvider, ResultDecodingError, RetryLLMHandler, @@ -2158,3 +2159,101 @@ def _completion(self, model, messages=None, **kwargs): assert messages[0]["role"] == "system", ( "System message should be the first message in history" ) + + +# --------------------------------------------------------------------------- +# Synthetic readers — integration (PR #545 finish-up) +# --------------------------------------------------------------------------- + + +class TestSyntheticReaderIntegration: + """The LLM can read lexical context through synthetic reader tools.""" + + @requires_llm + def test_llm_reads_lexical_value(self, request): + """Template asks LLM to inspect _known_data and report its sum. + The synthetic reader for _known_data is available in the tools + array; the LLM should call it, see [10,20,30,40,50], and report + the sum (150).""" + _known_data = [10, 20, 30, 40, 50] + + @Template.define + def report_sum() -> int: + """Use the `_known_data` tool to read the list of numbers, + then return their sum as an integer.""" + raise NotImplementedError + + with ( + handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)), + handler(LexicalReaders()), + ): + result = report_sum() + + assert isinstance(result, int) + assert result == sum(_known_data) # 150 + + @requires_llm + def test_template_synthesis_uses_lexical_reader(self, request): + """A Template that synthesizes a callable grounds its output + in a lexical value exposed as a synthetic reader. + + The Template asks the LLM to write a lambda comparing its + argument against `threshold`; the LLM must call the `threshold` + reader to inspect the value before emitting code. + """ + threshold = 0.85 + + @Template.define + def make_above_threshold() -> Callable[[float], bool]: + """Use the `threshold` reader tool to inspect its current + float value, then emit a single Python function definition: + + def above(x: float) -> bool: + return x > + + The function definition MUST be the last and only statement. + Do not emit any other code, no trailing assignment, no + imports, no comments after the function.""" + raise NotImplementedError + + with ( + handler(ReplayLiteLLMProvider(request, model=EFFECTFUL_LLM_MODEL)), + handler(UnsafeEvalProvider()), + handler(LimitLLMCallsHandler(max_calls=4)), + handler(LexicalReaders()), + ): + fn = make_above_threshold() + + assert fn(0.9) is True + assert fn(0.5) is False + assert fn(threshold) is False + + def test_template_exposes_lexical_classes(self): + """When `LexicalReaders` is installed, classes in the defining + scope are exposed as readers via the broad `Encodable[Callable]` + handler — the `Hand`/`Finger`/`generate_arm` motivating example + from #497. Without the handler the readers are gated off; this + test pins both contracts. + """ + + class Finger: + def wiggle(self) -> str: + return "wiggle" + + class Hand: + fingers: list[Finger] + + @Template.define + def describe_hand_action() -> str: + """Doc.""" + raise NotImplementedError + + # Off by default. + assert "Finger" not in describe_hand_action.tools + assert "Hand" not in describe_hand_action.tools + + # On under the handler. + with handler(LexicalReaders()): + tools = describe_hand_action.tools + assert "Finger" in tools + assert "Hand" in tools diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index d17723110..4938e8d75 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -778,7 +778,6 @@ def f(self) -> int: assert a.random in a.f.tools.values() # f is the template itself — found via self but correctly removed (non-recursive) assert a.f not in a.f.tools.values() - assert "local_variable" in a.f.__context__ and "local_variable" not in a.f.tools assert any(t() == 4 for t in a.f.tools.values() if t is a.random) class B(A): @@ -795,7 +794,6 @@ def reverse(self, s: str) -> str: assert isinstance(b.f, Template) assert b.random in b.f.tools.values() assert b.reverse in b.f.tools.values() - assert "local_variable" in b.f.__context__ and "local_variable" not in a.f.tools def test_template_method_nested_class(): @@ -826,7 +824,6 @@ def f(self) -> int: assert "random" in a.f.tools # f is the template itself — found via self but correctly removed (non-recursive) assert "f" not in a.f.tools - assert "local_variable" in a.f.__context__ and "local_variable" not in a.f.tools assert a.f.tools["random"]() == 4 @@ -1531,3 +1528,197 @@ def test_tool_forward_ref(): sig = inspect.signature(_tool_forward_ref) assert sig.parameters["x"].annotation is int assert sig.return_annotation is str + + +# --------------------------------------------------------------------------- +# Synthetic readers for lexical context (PR #545 finish-up) +# --------------------------------------------------------------------------- + +import re +import typing +from pathlib import Path + +import pydantic + +from effectful.handlers.llm.completions import ( + LexicalReaders, + _LexicalVariableTool, + collect_tools, +) + + +# Helpers for the test matrix +@dataclasses.dataclass +class _SimpleDataclass: + x: int + y: str + + +class _SimpleModel(pydantic.BaseModel): + """Pydantic model used in encodable-probe matrix tests.""" + + x: int + y: str + + +class _OpaqueNoEncoder: + """Plain user class with no Pydantic encoder.""" + + pass + + +def _example_unannotated(x): + """Unannotated function for the skip-via-catch tests.""" + return x + + +def _example_annotated(x: int) -> int: + """Annotated function returning x.""" + return x + + +def test_synthetic_reader_returns_captured_value(): + """The reader closes over the value snapshot taken at construction + time. In-place mutation of a mutable captured value is visible + (same object reference); rebinding the source name is not.""" + captured: list[int] = [1, 2, 3] + tool = _LexicalVariableTool.define(captured, name="x") + assert tool() == [1, 2, 3] + captured.append(4) + assert tool() == [1, 2, 3, 4] + + +def test_synthetic_reader_snapshot_survives_rebind(): + """Tools are constructed fresh each `call_assistant` invocation, + so rebinding the source name between construction and invocation + has no effect on the captured value.""" + env: dict = {"x": 42} + tool = _LexicalVariableTool.define(env["x"], name="x") + env["x"] = 99 + assert tool() == 42 + + +def test_synthetic_reader_snapshot_survives_deletion(): + """The closure holds the value directly, so deleting the source + name does not invalidate the reader.""" + env: dict = {"x": 42} + tool = _LexicalVariableTool.define(env["x"], name="x") + del env["x"] + assert tool() == 42 + + +_PROBE_OK_CASES: list[tuple[str, typing.Any]] = [ + ("primitive_int", 42), + ("primitive_str", "hello"), + ("list_of_int", [1, 2, 3]), + ("dict_value", {"a": 1}), + ("dataclass_simple", _SimpleDataclass(x=1, y="hello")), + ("pydantic_model", _SimpleModel(x=1, y="hello")), + # `re.Pattern` and `pathlib.PosixPath` are encodable in the matrix. + ("re_pattern", re.compile(r"x")), + ("pathlib_path", Path("/tmp")), +] + + +@pytest.mark.parametrize( + "name,value", _PROBE_OK_CASES, ids=lambda x: x[0] if isinstance(x, tuple) else None +) +def test_lexical_variable_tool_returns_value(name, value): + """`_LexicalVariableTool` builds a Tool when `Encodable[T]` + schema generates; calling it returns the captured value.""" + tool = _LexicalVariableTool.define(value, name=name) + assert tool() is value + + +# ---- Encodable-passthrough exposure (annotated callables, classes, +# builtins, methods) ---- + + +def _example_method_owner_unannotated(): + class _C: + def m(self): + return 1 + + return _C().m + + +def _example_method_owner_annotated(): + class _C: + def m(self) -> int: + return 1 + + return _C().m + + +_EXPOSED_THROUGH_ENCODABLE: list[tuple[str, typing.Callable[[], typing.Any]]] = [ + # Annotated function: nested_type → Callable[[int], int]; _pydantic_callable schema. + ("annotated_fn", lambda: _example_annotated), + # Unannotated function: nested_type → function; _pydantic_callable schema. + ("unannotated_fn", lambda: _example_unannotated), + # Plain class: nested_type → type; _pydantic_callable schema. + ("plain_class", lambda: type("Plain", (), {})), + # Builtin function. + ("builtin_fn", lambda: len), + # Bound method, annotated and unannotated. + ("annotated_method", _example_method_owner_annotated), + ("unannotated_method", _example_method_owner_unannotated), +] + + +@pytest.mark.parametrize( + "name,make_value", + _EXPOSED_THROUGH_ENCODABLE, + ids=lambda x: x[0] if isinstance(x, tuple) else None, +) +def test_collect_tools_exposes_callable_shaped_values(name, make_value): + """With `LexicalReaders` installed, annotated callables, classes, + builtins, and methods flow through Encodable's broad Callable + handler and become synthesis-shaped tools.""" + value = make_value() + env = {name: value} + with handler(LexicalReaders()): + assert name in collect_tools(env) + + +def test_lexical_reader_exposes_data_values(): + """Data-shaped values are exposed as readers (positive contract). + Readers snapshot the value, so calling one returns the *same* + object that was in env at construction time.""" + env = { + "x": 1, + "s": "hello", + "lst": [1, 2, 3], + "d": {"k": 1}, + "model": _SimpleModel(x=1, y="hi"), + } + with handler(LexicalReaders()): + result = collect_tools(env) + assert {"x", "s", "lst", "d", "model"} <= set(result) + for k, v in env.items(): + assert result[k]() is v + + +def test_template_tools_includes_synthetic_readers_for_locals(): + """The Template.tools property includes synthetic readers for + plain values in lexical scope when `LexicalReaders` is installed.""" + _test_data = [10, 20, 30] + + @Template.define + def t() -> int: + """Doc.""" + raise NotHandled + + with handler(LexicalReaders()): + tools = t.tools + assert "_test_data" in tools + assert tools["_test_data"]() == [10, 20, 30] + + +def test_lexical_readers_handler_enables_collection(): + """Installing `LexicalReaders` flips the gate; the same values are + exposed as zero-arg reader tools.""" + env = {"x": 42, "s": "hello"} + with handler(LexicalReaders()): + result = collect_tools(env) + assert result["x"]() == 42 + assert result["s"]() == "hello"