From d65dc4eb5f6e154a27e01a3700165bb64c53dff6 Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 7 Feb 2026 20:54:01 -0500 Subject: [PATCH 001/155] add dummy tools --- effectful/handlers/llm/completions.py | 54 ++++++++++++++++++++++++--- effectful/handlers/llm/template.py | 32 ++++++++++++++-- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index e7616fbfd..f2fd85f22 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -260,15 +260,19 @@ def call_assistant[T, U]( includes the raw assistant message for retry handling. """ tool_specs = {k: _function_model(t) for k, t in tools.items()} - response_model = pydantic.create_model( - "Response", value=response_format.enc, __config__={"extra": "forbid"} + response_model = ( + response_format.enc + if issubclass(response_format.enc, pydantic.BaseModel) + else pydantic.create_model( + "Response", value=response_format.enc, __config__={"extra": "forbid"} + ) ) messages = list(get_message_sequence().values()) response: litellm.types.utils.ModelResponse = completion( model, messages=list(messages), - response_format=response_model, + response_format=response_model if response_format.enc is not str else None, tools=list(tool_specs.values()), **kwargs, ) @@ -291,7 +295,7 @@ def call_assistant[T, U]( tool_calls.append(decoded_tool_call) result = None - if not tool_calls: + if not tool_calls and response_format.enc is not str: # return response serialized_result = message.get("content") or message.get("reasoning_content") assert isinstance(serialized_result, str), ( @@ -299,9 +303,18 @@ def call_assistant[T, U]( ) try: raw_result = response_model.model_validate_json(serialized_result) - result = response_format.decode(raw_result.value) # type: ignore + result = response_format.decode( + raw_result.value + if not issubclass(response_format.enc, pydantic.BaseModel) + else raw_result + ) # type: ignore except (pydantic.ValidationError, TypeError, ValueError, SyntaxError) as e: raise ResultDecodingError(e, raw_message=raw_message) from e + elif not tool_calls and response_format.enc is str: + # if expecting a string result, return the raw content as the result + content = message.get("content") or message.get("reasoning_content") + assert isinstance(content, str), "Expected content to be a string" + result = content return (raw_message, tool_calls, result) @@ -387,7 +400,36 @@ def flush_text() -> None: @Operation.define def call_system(template: Template) -> collections.abc.Sequence[Message]: """Get system instruction message(s) to prepend to all LLM prompts.""" - return () + + assert inspect.getdoc(type(template)) is not None + + system_prompt = inspect.cleandoc(f""" + You are responsible for implementing the `Template` '{template.__name__}' defined in the module source code below. + + First, as background, here is the class-level documentation for the `Template` class:: + + {inspect.getdoc(type(template))} + """) + + try: + system_prompt += inspect.cleandoc(f""" + Here is the source code of the module defining the `Template` instance '{template.__name__}':: + + {inspect.getsource(inspect.getmodule(template))} + """) + except (TypeError, OSError): + system_prompt += inspect.cleandoc(f""" + The source code for the module defining '{template.__name__}' is not available. + Instead, here are the signature and docstring of '{template.__name__}':: + + {template.__name__} :: {template.__signature__.format()} + + {inspect.cleandoc(template.__prompt_template__)} + """) + + msg = _make_message(dict(role="system", content=system_prompt)) + append_message(msg) + return (msg,) class RetryLLMHandler(ObjectInterpretation): diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 1a74b1005..f01018105 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -110,6 +110,20 @@ class _BoundInstance[T]: instance: T +def _make_context_tool[T](name: str, value: T) -> Tool[[], T]: + """Create a synthetic read-only Tool for a lexical variable.""" + from effectful.internals.unification import nested_type + + def reader(): + return value + + reader.__name__ = name + reader.__doc__ = f"Read the value of lexical variable `{name}`" + reader.__annotations__ = {"return": nested_type(value).value} + + return Tool.define(reader) + + class Template[**P, T](Tool[P, T]): """A :class:`Template` is a function that is implemented by a large language model. @@ -187,14 +201,14 @@ def tools(self) -> Mapping[str, Tool]: continue # Collect tools in context - if isinstance(obj, Tool): + elif isinstance(obj, Tool): result[name] = obj - if isinstance(obj, staticmethod) and isinstance(obj.__func__, Tool): + elif isinstance(obj, staticmethod) and isinstance(obj.__func__, Tool): result[name] = obj.__func__ # Collect tools as methods on any bound instances - if isinstance(obj, _BoundInstance): + elif isinstance(obj, _BoundInstance): for instance_name in obj.instance.__dir__(): if instance_name.startswith(INSTANCE_OP_PREFIX): continue @@ -202,6 +216,18 @@ def tools(self) -> Mapping[str, Tool]: if isinstance(instance_obj, Tool): result[instance_name] = instance_obj + # Make tools for lexical variables + elif not ( + name.startswith("__") + or isinstance(obj, Operation) + or inspect.isclass(obj) + or inspect.isbuiltin(obj) + or inspect.ismodule(obj) + or inspect.isroutine(obj) + or inspect.isabstract(obj) + ): + result[name] = _make_context_tool(name, obj) + return result def __get__[S](self, instance: S | None, owner: type[S] | None = None): From fa3fb6bca0dfe460d719f24295d5f803e42a230e Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 8 Feb 2026 02:59:44 -0500 Subject: [PATCH 002/155] Add Agent class --- docs/source/agent.py | 47 ----- docs/source/agent_example.rst | 18 -- docs/source/index.rst | 1 - effectful/handlers/llm/__init__.py | 4 +- effectful/handlers/llm/template.py | 70 ++++++- tests/test_handlers_llm_agent.py | 315 +++++++++++++++++++++++++++++ 6 files changed, 384 insertions(+), 71 deletions(-) delete mode 100644 docs/source/agent.py delete mode 100644 docs/source/agent_example.rst create mode 100644 tests/test_handlers_llm_agent.py diff --git a/docs/source/agent.py b/docs/source/agent.py deleted file mode 100644 index 77d526534..000000000 --- a/docs/source/agent.py +++ /dev/null @@ -1,47 +0,0 @@ -import functools -from collections import OrderedDict - -from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LiteLLMProvider, - Message, - get_message_sequence, -) -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled - - -class Agent: - __history__: OrderedDict[str, Message] - - def __init__(self): - self.__history__ = OrderedDict() # persist the list of messages - - def __init_subclass__(cls): - for method_name in dir(cls): - template = getattr(cls, method_name) - if not isinstance(template, Template): - continue - - @functools.wraps(template) - def wrapper(self, *args, **kwargs): - with handler({get_message_sequence: lambda: self.__history__}): - return template(self, *args, **kwargs) - - setattr(cls, method_name, wrapper) - - -if __name__ == "__main__": - - class ChatBot(Agent): - @Template.define - def send(self, user_input: str) -> str: - """User writes: {user_input}""" - raise NotHandled - - provider = LiteLLMProvider() - chatbot = ChatBot() - - with handler(provider): - print(chatbot.send("Hi!, how are you? I am in france.")) - print(chatbot.send("Remind me again, where am I?")) diff --git a/docs/source/agent_example.rst b/docs/source/agent_example.rst deleted file mode 100644 index a9993c568..000000000 --- a/docs/source/agent_example.rst +++ /dev/null @@ -1,18 +0,0 @@ -Contextual LLM Agents -====================== -Here we give an example of using effectful to implement chatbot-style context-aware LLM agents. - -In the code below, we define a helper class :class:`Agent` which wraps its -subclasses' template operations in a wrapper that stores and persists -the history of prior interactions with the LLM: - - :func:`_format_model_input` wraps every prompt sent to the LLM and - stashes the generated API message into a state variable. - - :func:`_compute_response` wraps the response from the LLM provider and - stashes the returned message into the state. - -Using this we can construct an agent which remembers the context of -the conversation: - -.. literalinclude:: ./agent.py - :language: python - diff --git a/docs/source/index.rst b/docs/source/index.rst index 2a5135735..33e7d3cf0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,7 +18,6 @@ Table of Contents lambda_example semi_ring_example beam_search_example - agent_example .. toctree:: :maxdepth: 2 diff --git a/effectful/handlers/llm/__init__.py b/effectful/handlers/llm/__init__.py index a87b481d6..cdda93479 100644 --- a/effectful/handlers/llm/__init__.py +++ b/effectful/handlers/llm/__init__.py @@ -1,3 +1,3 @@ -from .template import Template, Tool +from .template import Agent, Template, Tool -__all__ = ["Template", "Tool"] +__all__ = ["Agent", "Template", "Tool"] diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index f01018105..2dfab38f7 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -1,11 +1,14 @@ +import abc +import collections +import functools import inspect import types import typing -from collections import ChainMap from collections.abc import Callable, Mapping, MutableMapping from dataclasses import dataclass from typing import Annotated, Any +from effectful.ops.semantics import handler from effectful.ops.types import INSTANCE_OP_PREFIX, Annotation, Operation @@ -183,7 +186,7 @@ class Template[**P, T](Tool[P, T]): """ - __context__: ChainMap[str, Any] + __context__: collections.ChainMap[str, Any] @property def __prompt_template__(self) -> str: @@ -283,7 +286,7 @@ def define[**Q, V]( frame = frame.f_back contexts.append(globals_proxy) - context: ChainMap[str, Any] = ChainMap( + context: collections.ChainMap[str, Any] = collections.ChainMap( *typing.cast(list[MutableMapping[str, Any]], contexts) ) @@ -291,3 +294,64 @@ def define[**Q, V]( op.__context__ = context # type: ignore[attr-defined] return typing.cast(Template[Q, V], op) + + +class Agent(abc.ABC): + """Mixin that gives each instance a persistent LLM message history. + + Subclass and decorate methods with :func:`Template.define`. + Each instance accumulates messages across calls so the LLM sees + prior conversation context. + + Agents compose freely with :func:`dataclasses.dataclass` and other + base classes. Instance attributes are available in template + docstrings via ``{self.attr}``. + + Example:: + + import dataclasses + from effectful.handlers.llm import Agent, Template + from effectful.handlers.llm.completions import LiteLLMProvider + from effectful.ops.semantics import handler + from effectful.ops.types import NotHandled + + @dataclasses.dataclass + class ChatBot(Agent): + bot_name: str = dataclasses.field(default="ChatBot") + + @Template.define + def send(self, user_input: str) -> str: + \"""Friendly bot named {self.bot_name}. User writes: {user_input}\""" + raise NotHandled + + provider = LiteLLMProvider() + chatbot = ChatBot() + + with handler(provider): + chatbot.send("Hi! How are you? I am in France.") + chatbot.send("Remind me again, where am I?") # sees prior context + + """ + + __history__: collections.OrderedDict[str, Any] + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + prop = functools.cached_property(lambda _: collections.OrderedDict()) + prop.__set_name__(cls, "__history__") + cls.__history__ = prop + + for name in list(cls.__dict__): + attr = cls.__dict__[name] + if not isinstance(attr, Template): + continue + _template = attr + + @functools.wraps(_template) + def wrapper(self, *args, _t=_template, **kwargs): + from effectful.handlers.llm.completions import get_message_sequence + + with handler({get_message_sequence: lambda: self.__history__}): + return _t(self, *args, **kwargs) + + setattr(cls, name, wrapper) diff --git a/tests/test_handlers_llm_agent.py b/tests/test_handlers_llm_agent.py new file mode 100644 index 000000000..38c8aacd1 --- /dev/null +++ b/tests/test_handlers_llm_agent.py @@ -0,0 +1,315 @@ +"""Tests for Agent mixin message sequence semantics.""" + +import collections +import dataclasses + +from litellm import ModelResponse + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import ( + LiteLLMProvider, + RetryLLMHandler, + completion, +) +from effectful.ops.semantics import handler +from effectful.ops.syntax import ObjectInterpretation, implements +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Helpers (same pattern as test_handlers_llm_provider.py) +# --------------------------------------------------------------------------- + + +def make_text_response(content: str) -> ModelResponse: + return ModelResponse( + id="test", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + model="test-model", + ) + + +def make_tool_call_response( + tool_name: str, tool_args: str, tool_call_id: str = "call_1" +) -> ModelResponse: + return ModelResponse( + id="test", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_call_id, + "type": "function", + "function": {"name": tool_name, "arguments": tool_args}, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + model="test-model", + ) + + +class MockCompletionHandler(ObjectInterpretation): + """Returns pre-configured responses and captures messages sent to the LLM.""" + + def __init__(self, responses: list[ModelResponse]): + self.responses = responses + self.call_count = 0 + self.received_messages: list[list] = [] + + @implements(completion) + def _completion(self, model, messages=None, **kwargs): + self.received_messages.append(list(messages) if messages else []) + response = self.responses[min(self.call_count, len(self.responses) - 1)] + self.call_count += 1 + return response + + +# --------------------------------------------------------------------------- +# Agent subclass used by most tests +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class ChatBot(Agent): + """Simple chat agent for testing history accumulation.""" + + bot_name: str = dataclasses.field(default="ChatBot") + + @Template.define + def send(self, user_input: str) -> str: + """A friendly bot named {self.bot_name}. User writes: {user_input}""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestAgentHistoryAccumulation: + """History accumulates across sequential calls on the same instance.""" + + def test_second_call_sees_prior_messages(self): + mock = MockCompletionHandler( + [make_text_response("hi"), make_text_response("good")] + ) + bot = ChatBot() + + with handler(LiteLLMProvider()), handler(mock): + bot.send("hello") + bot.send("how are you") + + # First call: system + user → 2 messages + assert len(mock.received_messages[0]) == 2 + + # Second call: previous system + user + assistant, PLUS new system + user → 5 + assert len(mock.received_messages[1]) > len(mock.received_messages[0]) + + # Verify roles in second call + roles = [m["role"] for m in mock.received_messages[1]] + assert roles.count("assistant") >= 1 + assert roles.count("user") >= 2 + assert roles.count("system") >= 2 + + def test_history_contains_all_messages_after_two_calls(self): + mock = MockCompletionHandler( + [make_text_response("r1"), make_text_response("r2")] + ) + bot = ChatBot() + + with handler(LiteLLMProvider()), handler(mock): + bot.send("a") + bot.send("b") + + # After two complete calls the history should have: + # call 1: system, user, assistant (3) + # call 2: system, user, assistant (3) + assert len(bot.__history__) == 6 + + def test_message_ids_are_unique(self): + mock = MockCompletionHandler( + [make_text_response("r1"), make_text_response("r2")] + ) + bot = ChatBot() + + with handler(LiteLLMProvider()), handler(mock): + bot.send("a") + bot.send("b") + + ids = list(bot.__history__.keys()) + assert len(ids) == len(set(ids)), "message IDs must be unique" + + +class TestAgentIsolation: + """Each agent instance has independent history; non-agent templates are unaffected.""" + + def test_two_agents_have_independent_histories(self): + mock = MockCompletionHandler( + [ + make_text_response("from bot1"), + make_text_response("from bot2"), + ] + ) + bot1 = ChatBot() + bot2 = ChatBot() + + with handler(LiteLLMProvider()), handler(mock): + bot1.send("msg for bot1") + bot2.send("msg for bot2") + + # bot2's call should NOT contain bot1's messages + assert len(mock.received_messages[1]) == 2 # system + user only + + # Each bot has its own history + assert len(bot1.__history__) == 3 # system, user, assistant + assert len(bot2.__history__) == 3 + + # Histories share no message IDs + assert set(bot1.__history__.keys()).isdisjoint(set(bot2.__history__.keys())) + + def test_non_agent_template_gets_fresh_sequence(self): + @Template.define + def standalone(topic: str) -> str: + """Write about {topic}.""" + raise NotHandled + + mock = MockCompletionHandler( + [ + make_text_response("agent reply"), + make_text_response("standalone reply"), + make_text_response("agent reply 2"), + ] + ) + bot = ChatBot() + + with handler(LiteLLMProvider()), handler(mock): + bot.send("hello") + standalone("fish") + bot.send("bye") + + # standalone (call index 1) should see only system + user (fresh sequence) + assert len(mock.received_messages[1]) == 2 + + # bot's third call (call index 2) should see its accumulated history + # but NOT the standalone messages + assert len(mock.received_messages[2]) == 5 # 3 from first call + 2 new + + +class TestAgentCachedProperty: + """__history__ is lazily created per instance without requiring __init__.""" + + def test_no_init_required(self): + class MinimalAgent(Agent): + @Template.define + def greet(self, name: str) -> str: + """Hello {name}.""" + raise NotHandled + + agent = MinimalAgent() + # Should be an OrderedDict, created on first access + assert isinstance(agent.__history__, collections.OrderedDict) + assert len(agent.__history__) == 0 + + def test_subclass_with_own_init(self): + class CustomAgent(Agent): + def __init__(self, name: str): + self.name = name + + @Template.define + def greet(self) -> str: + """Say hello.""" + raise NotHandled + + agent = CustomAgent("Alice") + assert agent.name == "Alice" + assert isinstance(agent.__history__, collections.OrderedDict) + + def test_history_is_per_instance(self): + a = ChatBot() + b = ChatBot() + a.__history__["fake"] = {"id": "fake", "role": "user", "content": "x"} + assert "fake" not in b.__history__ + + +class TestAgentWithToolCalls: + """Agent methods that trigger tool calls maintain correct history.""" + + def test_tool_call_results_appear_in_history(self): + @Tool.define + def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + class MathAgent(Agent): + @Template.define + def compute(self, question: str) -> str: + """Answer: {question}""" + raise NotHandled + + mock = MockCompletionHandler( + [ + make_tool_call_response("add", '{"a": 2, "b": 3}'), + make_text_response("The answer is 5"), + ] + ) + agent = MathAgent() + + with handler(LiteLLMProvider()), handler(mock): + result = agent.compute("what is 2+3?") + + assert result == "The answer is 5" + + # History should contain: system, user, assistant (tool_call), + # tool (result), assistant (final) + roles = [m["role"] for m in agent.__history__.values()] + assert "tool" in roles + assert roles.count("assistant") == 2 + + +class TestAgentWithRetryHandler: + """RetryLLMHandler composes correctly with Agent history.""" + + def test_failed_retries_dont_pollute_history(self): + mock = MockCompletionHandler( + [ + # First attempt: invalid result for int + make_text_response('{"value": "not_an_int"}'), + # Retry: valid + make_text_response('{"value": 42}'), + ] + ) + + class NumberAgent(Agent): + @Template.define + def pick_number(self) -> int: + """Pick a number.""" + raise NotHandled + + agent = NumberAgent() + + with ( + handler(LiteLLMProvider()), + handler(RetryLLMHandler(num_retries=3)), + handler(mock), + ): + result = agent.pick_number() + + assert result == 42 + + # The malformed assistant message and error feedback from the retry + # should NOT appear in the agent's history. Only the final successful + # assistant message should be there. + roles = [m["role"] for m in agent.__history__.values()] + assert roles == ["system", "user", "assistant"] From eca5cb8fef22b83fadf267ba04c2dfaec3760c25 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 8 Feb 2026 13:28:31 -0500 Subject: [PATCH 003/155] stash examples --- docs/source/llm_examples/__init__.py | 0 docs/source/llm_examples/async_concurrency.py | 68 +++++ docs/source/llm_examples/batch_translate.py | 71 +++++ docs/source/llm_examples/chat_memory.py | 131 +++++++++ docs/source/llm_examples/chat_search.py | 113 ++++++++ docs/source/llm_examples/flight_booking.py | 255 ++++++++++++++++++ docs/source/llm_examples/guardrails.py | 74 +++++ .../llm_examples/hanoi_solver_iterative.py | 209 ++++++++++++++ .../llm_examples/hanoi_solver_recursive.py | 195 ++++++++++++++ docs/source/llm_examples/hitl.py | 173 ++++++++++++ docs/source/llm_examples/majority_vote.py | 88 ++++++ docs/source/llm_examples/map_reduce.py | 156 +++++++++++ docs/source/llm_examples/multi_agent.py | 164 +++++++++++ docs/source/llm_examples/rag.py | 193 +++++++++++++ docs/source/llm_examples/research_agent.py | 144 ++++++++++ docs/source/llm_examples/supervisor.py | 178 ++++++++++++ docs/source/llm_examples/tao_agent.py | 185 +++++++++++++ docs/source/llm_examples/text2sql.py | 170 ++++++++++++ docs/source/llm_examples/thinking.py | 113 ++++++++ 19 files changed, 2680 insertions(+) create mode 100644 docs/source/llm_examples/__init__.py create mode 100644 docs/source/llm_examples/async_concurrency.py create mode 100644 docs/source/llm_examples/batch_translate.py create mode 100644 docs/source/llm_examples/chat_memory.py create mode 100644 docs/source/llm_examples/chat_search.py create mode 100644 docs/source/llm_examples/flight_booking.py create mode 100644 docs/source/llm_examples/guardrails.py create mode 100644 docs/source/llm_examples/hanoi_solver_iterative.py create mode 100644 docs/source/llm_examples/hanoi_solver_recursive.py create mode 100644 docs/source/llm_examples/hitl.py create mode 100644 docs/source/llm_examples/majority_vote.py create mode 100644 docs/source/llm_examples/map_reduce.py create mode 100644 docs/source/llm_examples/multi_agent.py create mode 100644 docs/source/llm_examples/rag.py create mode 100644 docs/source/llm_examples/research_agent.py create mode 100644 docs/source/llm_examples/supervisor.py create mode 100644 docs/source/llm_examples/tao_agent.py create mode 100644 docs/source/llm_examples/text2sql.py create mode 100644 docs/source/llm_examples/thinking.py diff --git a/docs/source/llm_examples/__init__.py b/docs/source/llm_examples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/llm_examples/async_concurrency.py b/docs/source/llm_examples/async_concurrency.py new file mode 100644 index 000000000..32ec20c77 --- /dev/null +++ b/docs/source/llm_examples/async_concurrency.py @@ -0,0 +1,68 @@ +"""Fork/join async concurrency with templates. + +Demonstrates: +- Running multiple LLM template calls concurrently with ``asyncio.gather`` +- Using ``asyncio.to_thread`` to run synchronous template calls in parallel +""" + +import argparse +import asyncio +import functools +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Async template +# --------------------------------------------------------------------------- + + +@Template.define +def analyze_average_age(ages: list[int]) -> int: + """Analyze the dataset of ages {ages} and return the average age of + participants. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +async def main(provider: LiteLLMProvider): + analysis = functools.partial( + asyncio.to_thread, handler(provider)(analyze_average_age) + ) + results = await asyncio.gather( + analysis([25, 30, 35, 40]), + analysis([20, 28, 17, 30]), + analysis([22, 27, 31, 29]), + analysis([24, 26, 32, 38]), + analysis([21, 29, 33, 37]), + ) + for i, result in enumerate(results): + print(f"Group {i}: average age = {result}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Analyze average ages concurrently") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + asyncio.run(main(provider)) diff --git a/docs/source/llm_examples/batch_translate.py b/docs/source/llm_examples/batch_translate.py new file mode 100644 index 000000000..bcdb343d3 --- /dev/null +++ b/docs/source/llm_examples/batch_translate.py @@ -0,0 +1,71 @@ +"""Batch translation with instruction injection. + +Demonstrates: +- ``@Template.define`` for a translation template with injected instructions +""" + +import argparse +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.handlers.llm.evaluation import RestrictedEvalProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Translation template +# --------------------------------------------------------------------------- + + +@Template.define +def translate(target_language: str, instructions: str = "") -> Template[[str], str]: + """ + Write a `Template` that translates a string of English text into {target_language} + If any instructions are provided, include them in the prompt: {instructions} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Batch translation with instruction injection" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--max-steps", + type=int, + default=5, + help="Maximum number of steps before giving up", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed LLM output", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RestrictedEvalProvider()): + translator = translate( + target_language="french", instructions="Use formal language." + ) + print(translator("hello, how are you? how is your day going?")) diff --git a/docs/source/llm_examples/chat_memory.py b/docs/source/llm_examples/chat_memory.py new file mode 100644 index 000000000..42c8b46ac --- /dev/null +++ b/docs/source/llm_examples/chat_memory.py @@ -0,0 +1,131 @@ +"""Chat agent with embedding-based memory. + +Demonstrates: +- A stateful chat agent that maintains conversation history +- Embedding-based retrieval of relevant past context +- Simple in-memory vector store with L2 distance +""" + +import argparse +import dataclasses +import os + +import litellm +import numpy as np + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Embedding helpers +# --------------------------------------------------------------------------- + + +def get_embedding(text: str) -> np.ndarray: + """Get an embedding vector for the given text using litellm.""" + response = litellm.embedding(model="text-embedding-ada-002", input=text) + return np.array(response.data[0]["embedding"], dtype=np.float32) + + +def find_closest( + index: list[tuple[str, np.ndarray]], phrase: str +) -> tuple[str, float] | None: + """Find the closest entry in the index to the given phrase.""" + if not index: + return None + phrase_embedding = get_embedding(phrase) + + def dist(a: np.ndarray, b: np.ndarray) -> float: + return float(((a - b) ** 2).sum()) + + return min( + ((msg, dist(embedding, phrase_embedding)) for msg, embedding in index), + key=lambda elt: elt[1], + ) + + +# --------------------------------------------------------------------------- +# Chat template +# --------------------------------------------------------------------------- + + +@Template.define +def respond_to_user( + user_message: str, relevant_context: str, prev_messages: str +) -> str: + """Given the user wrote: {user_message} + Continue the conversation. + The last few messages were: {prev_messages} + Older relevant context: {relevant_context}""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Chat agent +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class ChatAgent: + """A chat agent that compresses old messages into an embedding index.""" + + history: list[dict[str, str]] = dataclasses.field(default_factory=list) + index: list[tuple[str, np.ndarray]] = dataclasses.field(default_factory=list) + + def _compress(self): + """Move the oldest pair of messages into the embedding index.""" + oldest_pair, self.history = self.history[:2], self.history[2:] + text = "\n".join(m["content"] for m in oldest_pair) + self.index.append((text, get_embedding(text))) + + def _find_relevant(self, query: str) -> str: + result = find_closest(self.index, query) + return result[0] if result else "No relevant context." + + def chat(self, user_input: str): + relevant = self._find_relevant(user_input) + prev_messages = "\n".join( + f"{m['author']}: {m['content']}" for m in self.history + ) + response = respond_to_user(user_input, relevant, prev_messages) + self.history.append({"author": "user", "content": user_input}) + self.history.append({"author": "agent", "content": response}) + if len(self.history) > 6: + self._compress() + print(f"user: {user_input}") + print(f"agent: {response}") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Chat agent with embedding-based memory" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + agent = ChatAgent() + + provider = LiteLLMProvider(model=args.model) + with handler(provider): + agent.chat("Hello! How are you doing?") + agent.chat("Lovely! I'm having a great day.") + agent.chat("What is the capital of France?") + agent.chat("I didn't know that! That's amazing!") diff --git a/docs/source/llm_examples/chat_search.py b/docs/source/llm_examples/chat_search.py new file mode 100644 index 000000000..8dcdd1691 --- /dev/null +++ b/docs/source/llm_examples/chat_search.py @@ -0,0 +1,113 @@ +import argparse +import dataclasses +import os +import urllib.parse + +import requests + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + + +@Tool.define +def search_web(query: str) -> str: + """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" + search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "list": "search", + "srsearch": query, + "srlimit": 1, + "format": "json", + } + ) + search_data = requests.get( + search_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + results = search_data.get("query", {}).get("search", []) + if not results: + raise ValueError(f"No results found for: {query}") + title = results[0]["title"] + + summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "titles": title, + "prop": "extracts", + "exintro": True, + "explaintext": True, + "format": "json", + } + ) + summary_data = requests.get( + summary_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + page = next(iter(summary_data["query"]["pages"].values())) + extract = page.get("extract", "No summary available.") + url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" + + return f"# {title}\n\n{extract}\n\nSource: {url}" + + +@dataclasses.dataclass +class ChatBot(Agent): + """Simple chat agent for testing history accumulation.""" + + bot_name: str = dataclasses.field(default="ChatBot") + + @Template.define + def send(self, user_input: str) -> str: + """ + You are a friendly and helpful AI assistant named {self.bot_name}. + If user input contains a question that you're not sure how to answer, + consider using the web search tool to find the answer and include it in your response. + + The user writes: + {user_input} + """ + raise NotHandled + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="LLM-guided research agent with web search" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--name", + type=str, + default="Chatty McChatface", + help="The name of the chatbot", + ) + parser.add_argument( + "--interactive", + action="store_true", + help="Run in interactive mode, allowing multiple back-and-forth messages", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + chatbot = ChatBot(bot_name=args.name) + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + if args.interactive: + while True: + print(chatbot.send(input("You: "))) + else: + print(chatbot.send("Hi! Can you tell me about the Statue of Liberty?")) + print(chatbot.send("Who designed it?")) + print(chatbot.send("What about the speed of light? How fast is it?")) diff --git a/docs/source/llm_examples/flight_booking.py b/docs/source/llm_examples/flight_booking.py new file mode 100644 index 000000000..35cdec35d --- /dev/null +++ b/docs/source/llm_examples/flight_booking.py @@ -0,0 +1,255 @@ +"""Flight booking with multi-agent delegation. + +Demonstrates: +- Multi-agent delegation: a tool that internally calls a separate + ``@Template.define`` (agent-to-agent delegation) +- Programmatic validation of LLM output with retry +- Interactive human-in-the-loop flow +- ``Agent`` history for conversational seat selection +""" + +import argparse +import dataclasses +import datetime +import enum +import os +from typing import Literal + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Structured output types +# --------------------------------------------------------------------------- + + +class Airport(enum.StrEnum): + SFO = "SFO" + ANC = "ANC" + FAI = "FAI" + JNU = "JNU" + NYC = "NYC" + LAX = "LAX" + CHI = "CHI" + MIA = "MIA" + BOS = "BOS" + SEA = "SEA" + DFW = "DFW" + DEN = "DEN" + ATL = "ATL" + HOU = "HOU" + + +@dataclasses.dataclass(frozen=True) +class FlightDetails: + flight_number: str + price: int + origin: Airport # three-letter airport code + destination: Airport # three-letter airport code + date: datetime.date # YYYY-MM-DD + + +@dataclasses.dataclass(frozen=True) +class SeatPreference: + row: int # 1-30 + seat: Literal["A", "B", "C", "D", "E", "F"] + + +# --------------------------------------------------------------------------- +# Sample data (in reality, downloaded from a booking site) +# --------------------------------------------------------------------------- + +FLIGHTS_PAGE = """\ +1. Flight SFO-AK123 - $350 - San Francisco (SFO) to Anchorage (ANC) - 2025-01-10 +2. Flight SFO-AK456 - $370 - San Francisco (SFO) to Fairbanks (FAI) - 2025-01-10 +3. Flight SFO-AK789 - $400 - San Francisco (SFO) to Juneau (JNU) - 2025-01-20 +4. Flight NYC-LA101 - $250 - San Francisco (SFO) to Anchorage (ANC) - 2025-01-10 +5. Flight CHI-MIA202 - $200 - Chicago (ORD) to Miami (MIA) - 2025-01-12 +6. Flight BOS-SEA303 - $120 - Boston (BOS) to Anchorage (ANC) - 2025-01-12 +7. Flight DFW-DEN404 - $150 - Dallas (DFW) to Denver (DEN) - 2025-01-10 +8. Flight ATL-HOU505 - $180 - Atlanta (ATL) to Houston (IAH) - 2025-01-10 +""" + +# --------------------------------------------------------------------------- +# Extraction template (inner "agent") +# --------------------------------------------------------------------------- + + +@Template.define +def extract_flights(web_page_text: str) -> list[FlightDetails]: + """Extract all flight details from the following text. + + {web_page_text} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Tool that delegates to the extraction template +# --------------------------------------------------------------------------- + +# The tool is defined at module scope so that FlightFinder's template +# captures it via lexical scope (same pattern as search_web in other examples). + + +@Tool.define +def get_available_flights() -> list[FlightDetails]: + """Retrieve all available flights from the booking page.""" + return extract_flights(FLIGHTS_PAGE) + + +# --------------------------------------------------------------------------- +# Flight search agent +# --------------------------------------------------------------------------- + + +class FlightFinder(Agent): + """Agent that finds flights matching user criteria.""" + + @Template.define + def find_flight( + self, origin: Airport, destination: Airport, date: datetime.date + ) -> FlightDetails: + """Find the cheapest flight from {origin} to {destination} on {date}. + + Use the get_available_flights tool to retrieve all flights, then + select the cheapest one that matches the origin, destination, + and date exactly. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Seat selection agent +# --------------------------------------------------------------------------- + + +class SeatSelector(Agent): + """Agent that extracts seat preferences from natural language.""" + + @Template.define + def select_seat(self, user_input: str) -> SeatPreference: + """Extract the user's seat preference from their message. + + {user_input} + + Seats A and F are window seats. Seats C and D are aisle seats. + Row 1 is the front row with extra legroom. + Rows 14 and 20 also have extra legroom. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Validation (plain Python, no LLM needed) +# --------------------------------------------------------------------------- + + +def validate_flight( + flight: FlightDetails, origin: Airport, destination: Airport, date: datetime.date +) -> list[str]: + """Check that the selected flight matches the requested criteria.""" + errors = [] + if flight.origin != origin: + errors.append(f"origin should be {origin}, got {flight.origin}") + if flight.destination != destination: + errors.append(f"destination should be {destination}, got {flight.destination}") + if flight.date != date: + errors.append(f"date should be {date}, got {flight.date}") + return errors + + +# --------------------------------------------------------------------------- +# Booking flow +# --------------------------------------------------------------------------- + + +def book_flight( + origin: Airport, + destination: Airport, + date: datetime.date, + interactive: bool = False, + max_retries: int = 3, +) -> None: + """End-to-end flight booking with search, validation, and seat selection.""" + searcher = FlightFinder() + + # --- Search with validation retry --- + flight = None + for attempt in range(max_retries): + candidate = searcher.find_flight(origin, destination, date) + errors = validate_flight(candidate, origin, destination, date) + if errors: + print(f" [attempt {attempt}] Rejected: {'; '.join(errors)}") + continue + flight = candidate + break + + if flight is None: + print("Could not find a valid flight.") + return + + print( + f" Found: {flight.flight_number} ${flight.price} " + f"({flight.origin}->{flight.destination} on {flight.date})" + ) + + # --- User approval (interactive only) --- + if interactive: + if input(" Book this flight? (yes/no): ").strip().lower() != "yes": + print(" Cancelled.") + return + + # --- Seat selection --- + selector = SeatSelector() + seat_requests = ( + [input(" Seat preference: ")] + if interactive + else ["I'd like a window seat with extra legroom please"] + ) + for request in seat_requests: + seat = selector.select_seat(request) + print(f" Seat: row {seat.row}, seat {seat.seat}") + + print(f" Booked {flight.flight_number}, seat {seat.row}{seat.seat}!") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Flight booking with multi-agent delegation" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--interactive", + action="store_true", + help="Run in interactive mode with user prompts", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=5)): + book_flight( + origin=Airport.SFO, + destination=Airport.ANC, + date=datetime.date(2025, 1, 10), + interactive=args.interactive, + ) diff --git a/docs/source/llm_examples/guardrails.py b/docs/source/llm_examples/guardrails.py new file mode 100644 index 000000000..304818ffb --- /dev/null +++ b/docs/source/llm_examples/guardrails.py @@ -0,0 +1,74 @@ +"""Travel advisor with input guardrails. + +Demonstrates: +- Using one template to validate/guard input before passing it to another +- Simple control-flow gating based on LLM classification +""" + +import argparse +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def travel_query(user_query: str) -> str: + """ + Produce a concise (<100 word) answer to: {user_query} + """ + raise NotHandled + + +@Template.define +def is_safe_query(user_query: str) -> bool: + """ + Determine whether the user's query is purely related to travel advice: {user_query} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Guarded agent +# --------------------------------------------------------------------------- + + +def answer_travel_query(user_query: str) -> str: + """Only answer travel-related queries; reject everything else.""" + if is_safe_query(user_query): + return travel_query(user_query) + else: + return f"Rejected: '{user_query}' is not related to travel advice." + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Analyze average ages concurrently") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + with handler(provider), handler(RetryLLMHandler(num_retries=5)): + print(answer_travel_query("What are great places to check out in NYC?")) + print(answer_travel_query("Should I buy apple stocks?")) diff --git a/docs/source/llm_examples/hanoi_solver_iterative.py b/docs/source/llm_examples/hanoi_solver_iterative.py new file mode 100644 index 000000000..8733c64ac --- /dev/null +++ b/docs/source/llm_examples/hanoi_solver_iterative.py @@ -0,0 +1,209 @@ +"""LLM-guided Towers of Hanoi solver with tool-based validation. + +Adapted from https://github.com/BasisResearch/effectful/pull/404 + +Demonstrates: +- A static Pydantic ``Step`` model for structured output +- ``@Tool.define`` inside a closure to expose game-state validation as a tool +- ``RetryLLMHandler`` to retry on malformed LLM output +- Templates defined inside a function that auto-capture closure-scoped tools +""" + +import argparse +import itertools +import os +from dataclasses import dataclass, field + +from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Step model +# --------------------------------------------------------------------------- + + +@dataclass +class Step: + """A single move: take the top disk from tower ``start`` and place it on + tower ``end``. Tower indices are zero-based.""" + + start: int + end: int + explanation: str = field(default="") # optional reasoning from the LLM + + +# --------------------------------------------------------------------------- +# Game state +# --------------------------------------------------------------------------- + + +@dataclass +class GameState: + """State of a Towers of Hanoi game. + + Higher numbers represent larger disks, so ``(2, 1, 0)`` is a valid + tower (largest on bottom). The goal is to move all disks from the + leftmost tower (index 0) to the rightmost tower (index -1). + + This is a plain ``dataclass`` (not a Pydantic model) so the type checker + can see its methods. + """ + + size: int + towers: tuple[tuple[int, ...], ...] = field(default=()) + + def __post_init__(self): + if self.size > 0 and not self.towers: + self.towers = tuple( + tuple(reversed(range(self.size))) if i == 0 else () + for i in range(self.size) + ) + + def apply(self, step: Step) -> "GameState": + """Apply a move, returning the new state. Raises ``ValueError`` if + the move is invalid.""" + start, end = step.start, step.end + if not (0 <= start < len(self.towers) and 0 <= end < len(self.towers)): + raise ValueError(f"tower index out of range: ({start}, {end})") + if len(self.towers[start]) == 0: + raise ValueError(f"tower {start} is empty") + if len(self.towers[end]) > 0 and self.towers[start][-1] > self.towers[end][-1]: + raise ValueError( + f"cannot place disk {self.towers[start][-1]} on top of " + f"disk {self.towers[end][-1]}" + ) + new_towers = [list(t) for t in self.towers] + disk = new_towers[start].pop() + new_towers[end].append(disk) + return GameState(self.size, tuple(tuple(t) for t in new_towers)) + + def is_done(self) -> bool: + return all(len(t) == 0 for t in self.towers[:-1]) and all( + self.towers[-1][i] > self.towers[-1][i + 1] + for i in range(len(self.towers[-1]) - 1) + ) + + def valid_steps(self) -> list[Step]: + steps = [] + for i, ti in enumerate(self.towers): + for j, tj in enumerate(self.towers): + if i == j or len(ti) == 0: + continue + if len(tj) == 0 or ti[-1] < tj[-1]: + steps.append(Step(i, j)) + return steps + + def __str__(self) -> str: + return " | ".join(str(list(t)) for t in self.towers) + + +# --------------------------------------------------------------------------- +# LLM move predictor +# --------------------------------------------------------------------------- + + +def predict_next_step(state: GameState) -> Step: + """Ask the LLM to predict the next move. + + A ``get_valid_moves`` tool is defined in the closure so the template + can query which moves are legal for the current game state. A + ``validate_move`` tool checks whether a proposed move is legal and + raises ``ValueError`` if not — when wrapped by ``RetryLLMHandler``, + this error is fed back to the LLM so it can correct itself. + """ + valid = state.valid_steps() + + @Tool.define + def get_valid_moves() -> list[Step]: + """Return the list of valid moves for the current game state.""" + return valid + + @Tool.define + def validate_move(proposed: Step) -> bool: + """Check whether moving from tower ``start`` to tower ``end`` is legal.""" + return proposed in state.valid_steps() + + @Template.define + def predict(game_state: GameState) -> Step: + """Given the state of the game of Towers of Hanoi: + + {game_state} + + Predict the next step to complete the game (move all disks to the + rightmost tower). You MUST call get_valid_moves first to see which + moves are legal, then pick the best one. Give a brief reasoning. + """ + raise NotHandled + + return predict(state) + + +# --------------------------------------------------------------------------- +# Solver loop +# --------------------------------------------------------------------------- + + +def solve_hanoi(state: GameState, max_steps: int = 30): + """Solve Towers of Hanoi by repeatedly asking the LLM for the next move.""" + for i in itertools.count(): + print(f"step {i}: {state}") + if state.is_done(): + print("Solved!") + return + if i >= max_steps: + print("Gave up after max steps.") + return + + step: Step = predict_next_step(state) + try: + state = state.apply(step) + print(f" move: {step.start} -> {step.end}") + except ValueError as e: + print(f" attempt {i}: invalid move {step}: {e}") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="LLM-guided Towers of Hanoi solver") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--game-size", + type=int, + default=3, + help="Number of disks in the Towers of Hanoi game", + ) + parser.add_argument( + "--max-steps", + type=int, + default=30, + help="Maximum number of steps before giving up", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed LLM output", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=args.num_retries)): + solve_hanoi(GameState(size=args.game_size), max_steps=args.max_steps) diff --git a/docs/source/llm_examples/hanoi_solver_recursive.py b/docs/source/llm_examples/hanoi_solver_recursive.py new file mode 100644 index 000000000..b5da9107a --- /dev/null +++ b/docs/source/llm_examples/hanoi_solver_recursive.py @@ -0,0 +1,195 @@ +"""Recursive LLM-based Towers of Hanoi solver. + +Adapted from https://github.com/BasisResearch/effectful/pull/404 + +Demonstrates: +- ``IsRecursive`` annotation to let a template call itself as a tool +- Recursive problem decomposition via LLM tool calls +- Post-hoc validation of the LLM-generated move sequence + +The classic recursive algorithm for Tower of Hanoi is: + + hanoi(n, source, target, auxiliary): + if n == 1: move disk from source to target + else: + hanoi(n-1, source, auxiliary, target) # move n-1 disks out of the way + move largest disk from source to target # move the bottom disk + hanoi(n-1, auxiliary, target, source) # move n-1 disks to target + +This solver defines a recursive ``Template`` that can call itself as a tool. +The LLM decomposes the n-disk problem into three sub-steps, making recursive +tool calls for the (n-1)-disk sub-problems, and returns the concatenated +list of moves. + +See: https://en.wikipedia.org/wiki/Tower_of_Hanoi +""" + +import argparse +import os +import typing +from dataclasses import dataclass, field + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.handlers.llm.template import IsRecursive +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Step model +# --------------------------------------------------------------------------- + + +@dataclass +class Step: + """A single move: take the top disk from tower ``start`` and place it on + tower ``end``. Tower indices are zero-based.""" + + start: int + end: int + + +# --------------------------------------------------------------------------- +# Game state (for validation only) +# --------------------------------------------------------------------------- + + +@dataclass +class GameState: + """State of a Towers of Hanoi game. + + Higher numbers represent larger disks, so ``(2, 1, 0)`` is a valid + tower (largest on bottom). The goal is to move all disks from the + leftmost tower (index 0) to the rightmost tower (index -1). + """ + + size: int + towers: tuple[tuple[int, ...], ...] = field(default=()) + + def __post_init__(self): + if self.size > 0 and not self.towers: + self.towers = tuple( + tuple(reversed(range(self.size))) if i == 0 else () + for i in range(self.size) + ) + + def apply(self, step: Step) -> "GameState": + """Apply a move, returning the new state. Raises ``ValueError`` if + the move is invalid.""" + start, end = step.start, step.end + if not (0 <= start < len(self.towers) and 0 <= end < len(self.towers)): + raise ValueError(f"tower index out of range: ({start}, {end})") + if len(self.towers[start]) == 0: + raise ValueError(f"tower {start} is empty") + if len(self.towers[end]) > 0 and self.towers[start][-1] > self.towers[end][-1]: + raise ValueError( + f"cannot place disk {self.towers[start][-1]} on top of " + f"disk {self.towers[end][-1]}" + ) + new_towers = [list(t) for t in self.towers] + disk = new_towers[start].pop() + new_towers[end].append(disk) + return GameState(self.size, tuple(tuple(t) for t in new_towers)) + + def is_done(self) -> bool: + return all(len(t) == 0 for t in self.towers[:-1]) and all( + self.towers[-1][i] > self.towers[-1][i + 1] + for i in range(len(self.towers[-1]) - 1) + ) + + def __str__(self) -> str: + return " | ".join(str(list(t)) for t in self.towers) + + +# --------------------------------------------------------------------------- +# Recursive LLM solver +# --------------------------------------------------------------------------- + + +@Template.define +def solve( + n_disks: int, source: int, target: int, auxiliary: int +) -> typing.Annotated[list[Step], IsRecursive]: + """Solve Tower of Hanoi: move {n_disks} disks from tower {source} to + tower {target}, using tower {auxiliary} as temporary storage. + + Recursive strategy: + - Base case (n_disks == 1): return [Step(start=source, end=target)] + - Recursive case (n_disks > 1): + 1. Call solve(n_disks - 1, source, auxiliary, target) to move the + top n_disks-1 disks out of the way onto the auxiliary tower. + 2. Move the largest disk: Step(start=source, end=target). + 3. Call solve(n_disks - 1, auxiliary, target, source) to move the + n_disks-1 disks from auxiliary to the target tower. + 4. Return the concatenated list of all steps from (1), (2), and (3). + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +def validate_solution(size: int, steps: list[Step]) -> bool: + """Apply all steps to the initial state and check that the puzzle is solved.""" + state = GameState(size=size) + print(f" initial: {state}") + for i, step in enumerate(steps): + try: + state = state.apply(step) + print(f" step {i}: move {step.start} -> {step.end} => {state}") + except ValueError as e: + print(f" step {i}: INVALID move {step.start} -> {step.end}: {e}") + return False + if state.is_done(): + print(f" Solved in {len(steps)} moves!") + return True + else: + print(f" Not solved after {len(steps)} moves. Final state: {state}") + return False + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Recursive LLM-based Towers of Hanoi solver" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--game-size", + type=int, + default=3, + help="Number of disks in the Towers of Hanoi game", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed LLM output", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=args.num_retries)): + n = args.game_size + print(f"Solving Tower of Hanoi with {n} disks...") + steps = solve(n_disks=n, source=0, target=n - 1, auxiliary=1) + print(f"\nLLM returned {len(steps)} steps. Validating...\n") + validate_solution(n, steps) diff --git a/docs/source/llm_examples/hitl.py b/docs/source/llm_examples/hitl.py new file mode 100644 index 000000000..5b2ebe17c --- /dev/null +++ b/docs/source/llm_examples/hitl.py @@ -0,0 +1,173 @@ +"""Human-in-the-loop task planner. + +Demonstrates: +- An ``Agent`` that proposes a plan of action steps +- Human approval/rejection of each step before execution +- Feedback from rejection is fed back to the agent via history +- ``@Tool.define`` for executing approved actions +- Non-interactive mode for testing (auto-approves all steps) +""" + +import argparse +import dataclasses +import enum +import os + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Structured output +# --------------------------------------------------------------------------- + + +class ActionType(enum.StrEnum): + send_email = "send_email" + create_file = "create_file" + schedule_meeting = "schedule_meeting" + done = "done" + + +@dataclasses.dataclass(frozen=True) +class ProposedAction: + action: ActionType + description: str + details: str + + +# --------------------------------------------------------------------------- +# Simulated action execution +# --------------------------------------------------------------------------- + + +execution_log: list[str] = [] + + +@Tool.define +def execute_action(action: ActionType, details: str) -> str: + """Execute an approved action. Returns a confirmation message.""" + msg = f"[executed] {action}: {details}" + execution_log.append(msg) + return msg + + +# --------------------------------------------------------------------------- +# Planner agent +# --------------------------------------------------------------------------- + + +class Planner(Agent): + """Agent that proposes actions one at a time for human approval.""" + + @Template.define + def propose_next(self, task: str, feedback: str) -> ProposedAction: + """You are a task planner helping the user accomplish a goal. + + Task: {task} + + Feedback from the last step: {feedback} + + Review the conversation history for previously completed actions. + Propose the next action to take. If the task is complete, + set action to "done". + + If a previous proposal was rejected, propose something different + that addresses the feedback. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Human-in-the-loop execution +# --------------------------------------------------------------------------- + + +def run_with_approval( + task: str, interactive: bool = False, max_steps: int = 5 +) -> list[str]: + """Run a task planner with human approval for each step.""" + planner = Planner() + feedback = "No actions taken yet. Start planning." + + for step in range(max_steps): + proposal = planner.propose_next(task, feedback) + + if proposal.action == ActionType.done: + print(f" [step {step + 1}] Done: {proposal.description}") + break + + print( + f" [step {step + 1}] Proposed: {proposal.action} - {proposal.description}" + ) + print(f" Details: {proposal.details}") + + if interactive: + answer = input(" Approve? (yes/no + reason): ").strip() + approved = answer.lower().startswith("y") + else: + answer = "yes" + approved = True + + if approved: + result = execute_action(proposal.action, proposal.details) + print(f" {result}") + feedback = f"Approved and executed: {result}" + else: + print(f" [rejected] {answer}") + feedback = f"Rejected: {answer}" + + return list(execution_log) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Human-in-the-loop task planner") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--interactive", + action="store_true", + help="Run in interactive mode with human approval prompts", + ) + parser.add_argument( + "--max-steps", + type=int, + default=5, + help="Maximum number of action steps", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + task = ( + "Organize a team lunch for next Friday. " + "Send an email to the team, create a shared document for " + "restaurant suggestions, and schedule a meeting to finalize plans." + ) + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + print(f"Task: {task}\n") + log = run_with_approval( + task, + interactive=args.interactive, + max_steps=args.max_steps, + ) + print(f"\nExecution log ({len(log)} actions):") + for entry in log: + print(f" {entry}") diff --git a/docs/source/llm_examples/majority_vote.py b/docs/source/llm_examples/majority_vote.py new file mode 100644 index 000000000..1ad8c6296 --- /dev/null +++ b/docs/source/llm_examples/majority_vote.py @@ -0,0 +1,88 @@ +"""Majority voting ensemble. + +Demonstrates: +- Running the same template multiple times and taking a majority vote +- ``collections.Counter`` for tallying responses +""" + +import argparse +import collections +import collections.abc +import enum +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Template +# --------------------------------------------------------------------------- + + +class Answer(enum.StrEnum): + yes = "yes" + no = "no" + maybe = "maybe" + + +@Template.define +def yes_or_no(question: str) -> Answer: + """ + Answer the following yes/no/maybe question: {question} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Majority vote +# --------------------------------------------------------------------------- + + +def majority_vote[Q]( + oracle: collections.abc.Callable[[Q], Answer], query: Q, voters: int = 3 +) -> tuple[Answer, int]: + """Call ``oracle(query)`` multiple times and return the most common answer.""" + counter = collections.Counter(oracle(query) for _ in range(voters)) + return counter.most_common(1)[0] + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Majority voting ensemble for yes/no questions" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--num-voters", type=int, default=3, help="Number of voters for majority vote" + ) + parser.add_argument( + "--question", + type=str, + default="Is Paris the capital of France?", + help="Yes/no question to ask", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + with handler(provider): + answer, count = majority_vote(yes_or_no, args.question, voters=args.num_voters) + print( + f"Question: {args.question}\nAnswer: {answer} (voted {count}/{args.num_voters})" + ) diff --git a/docs/source/llm_examples/map_reduce.py b/docs/source/llm_examples/map_reduce.py new file mode 100644 index 000000000..f7e00ac1d --- /dev/null +++ b/docs/source/llm_examples/map_reduce.py @@ -0,0 +1,156 @@ +"""Map-reduce resume evaluation. + +Demonstrates: +- Fan-out: evaluating multiple items independently with the same template +- Reduce: aggregating individual results into a summary +- ``asyncio.gather`` with ``asyncio.to_thread`` for parallel LLM calls +- Structured output with dataclasses +""" + +import argparse +import asyncio +import dataclasses +import functools +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Structured output +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class Evaluation: + name: str + qualified: bool + strengths: str + weaknesses: str + score: int # 1-10 + + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def evaluate_resume(resume: str, job_description: str) -> Evaluation: + """You are a hiring manager. Evaluate this resume against the job + description and produce a structured evaluation. + + Job description: {job_description} + + Resume: + {resume} + + Score from 1 (poor fit) to 10 (perfect fit). + """ + raise NotHandled + + +@Template.define +def summarize_evaluations(job_description: str, evaluations_text: str) -> str: + """You are a hiring manager summarizing candidate evaluations. + + Job description: {job_description} + + Individual evaluations: + {evaluations_text} + + Provide a brief summary: rank the candidates from best to worst, + highlight the top candidate, and note any concerns. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Sample data +# --------------------------------------------------------------------------- + +JOB_DESCRIPTION = ( + "Senior Python Developer: 5+ years Python experience, " + "familiarity with web frameworks (Django/Flask), " + "database design, and cloud deployment (AWS/GCP)." +) + +RESUMES = [ + "Alice Chen - 7 years Python, Django expert, AWS certified, " + "led team of 5, built microservices architecture at FinTech startup.", + "Bob Smith - 3 years Python, 2 years JavaScript, some Flask experience, " + "junior developer at small agency, strong communication skills.", + "Carol Davis - 10 years software engineering, 6 years Python, " + "GCP specialist, PostgreSQL expert, open-source contributor, " + "previously senior engineer at Google.", + "Dave Wilson - 4 years Python, self-taught, built several side projects, " + "no professional experience with web frameworks or cloud platforms.", +] + +# --------------------------------------------------------------------------- +# Map-reduce pipeline +# --------------------------------------------------------------------------- + + +async def map_reduce_evaluate( + provider: LiteLLMProvider, + resumes: list[str], + job_description: str, +) -> str: + """Evaluate resumes in parallel (map), then summarize (reduce).""" + # Map: evaluate each resume concurrently + evaluate = functools.partial( + asyncio.to_thread, + handler(provider)(handler(RetryLLMHandler(num_retries=3))(evaluate_resume)), + ) + evaluations: list[Evaluation] = list( + await asyncio.gather(*(evaluate(resume, job_description) for resume in resumes)) + ) + + # Print individual evaluations + for ev in evaluations: + print(f" {ev.name}: score={ev.score}/10, qualified={ev.qualified}") + print(f" + {ev.strengths}") + print(f" - {ev.weaknesses}") + + # Reduce: summarize all evaluations + evaluations_text = "\n\n".join( + f"Candidate: {ev.name}\n" + f"Score: {ev.score}/10\n" + f"Qualified: {ev.qualified}\n" + f"Strengths: {ev.strengths}\n" + f"Weaknesses: {ev.weaknesses}" + for ev in evaluations + ) + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + return summarize_evaluations(job_description, evaluations_text) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Map-reduce resume evaluation") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + print(f"Evaluating {len(RESUMES)} resumes for: {JOB_DESCRIPTION}\n") + summary = asyncio.run(map_reduce_evaluate(provider, RESUMES, JOB_DESCRIPTION)) + print(f"\n{summary}") diff --git a/docs/source/llm_examples/multi_agent.py b/docs/source/llm_examples/multi_agent.py new file mode 100644 index 000000000..448cf7a4f --- /dev/null +++ b/docs/source/llm_examples/multi_agent.py @@ -0,0 +1,164 @@ +"""Multi-agent Taboo word guessing game. + +Demonstrates: +- Two ``Agent`` instances with independent conversation histories +- Inter-agent communication via plain function calls +- Each agent has a different persona and goal +- ``Agent.__history__`` keeps each agent's context isolated +""" + +import argparse +import dataclasses +import enum +import os + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Structured output +# --------------------------------------------------------------------------- + + +class Confidence(enum.Enum): + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + + +@dataclasses.dataclass(frozen=True) +class Guess: + guess: str + confidence: Confidence + + +# --------------------------------------------------------------------------- +# Agents +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Hinter(Agent): + """Agent that gives hints about a secret word without saying it.""" + + secret_word: str = dataclasses.field(default="") + taboo_words: list[str] = dataclasses.field(default_factory=list) + + @Tool.define + def is_taboo(self, hint: str) -> bool: + """Check if the given hint contains any taboo words or the secret word.""" + lowered_hint = hint.lower() + if self.secret_word.lower() in lowered_hint: + return True + for taboo in self.taboo_words: + if taboo.lower() in lowered_hint: + return True + return False + + @Template.define + def give_hint(self, guesser_response: str) -> str: + """You are playing a word guessing game. You must help the guesser + figure out the secret word by giving creative hints. + + RULES: + - You MUST NOT say the secret word: {self.secret_word} + - You MUST NOT use any of these taboo words: {self.taboo_words} + - Give a single, concise hint (one sentence) + - Review conversation history to avoid repeating hints + - Use the is_taboo tool to check if your hint is valid + + The guesser's last response was: {guesser_response} + """ + raise NotHandled + + +class Guesser(Agent): + """Agent that tries to guess the secret word from hints.""" + + @Template.define + def make_guess(self, hint: str) -> Guess: + """You are playing a word guessing game. Based on the hints you've + received, guess the secret word. + + Latest hint: {hint} + + Review the conversation history for all previous hints. + Make your best guess. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Game loop +# --------------------------------------------------------------------------- + + +def play_taboo( + secret_word: str, + taboo_words: list[str], + max_rounds: int = 5, +) -> bool: + """Play a round of Taboo between a hinter and a guesser.""" + hinter = Hinter(secret_word=secret_word, taboo_words=taboo_words) + guesser = Guesser() + + guesser_response = "I'm ready to guess!" + + for round_num in range(max_rounds): + # Hinter gives a hint + hint = hinter.give_hint(guesser_response) + print(f" [round {round_num}] Hinter: {hint}") + + # Guesser tries to guess + guess = guesser.make_guess(hint) + guesser_response = f"I guessed '{guess.guess}' ({guess.confidence})" + print(f" [round {round_num}] Guesser: {guess.guess} ({guess.confidence})") + + if guess.guess.lower().strip() == secret_word.lower(): + print(f" Correct! Guessed in {round_num} round(s).") + return True + + print(f" Failed to guess '{secret_word}' in {max_rounds} rounds.") + return False + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Multi-agent Taboo word guessing game") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--max-rounds", + type=int, + default=5, + help="Maximum rounds per game", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + games = [ + ("piano", ["music", "keys", "instrument", "play"]), + ("volcano", ["lava", "eruption", "mountain", "hot"]), + ] + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + for secret, taboo in games: + print(f"\nGame: '{secret}' (taboo: {taboo})") + play_taboo(secret, taboo, max_rounds=args.max_rounds) diff --git a/docs/source/llm_examples/rag.py b/docs/source/llm_examples/rag.py new file mode 100644 index 000000000..166f1dbb4 --- /dev/null +++ b/docs/source/llm_examples/rag.py @@ -0,0 +1,193 @@ +"""Retrieval-augmented generation (RAG). + +Demonstrates: +- Offline: chunking documents, embedding, and indexing +- Online: embedding a query, retrieving relevant chunks, and generating + a grounded answer +- ``@Tool.define`` to expose retrieval as a tool the LLM can call +- Separation of indexing (plain Python) from generation (``@Template.define``) +""" + +import argparse +import dataclasses +import os + +import litellm +import numpy as np + +from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Embedding helpers +# --------------------------------------------------------------------------- + + +def get_embedding(text: str, model: str) -> np.ndarray: + """Get an embedding vector for the given text using litellm.""" + response = litellm.embedding(model=model, input=text) + return np.array(response.data[0]["embedding"], dtype=np.float32) + + +# --------------------------------------------------------------------------- +# Vector index +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class VectorIndex: + """Simple in-memory vector index using L2 distance.""" + + model: str + chunks: list[str] = dataclasses.field(default_factory=list) + embeddings: list[np.ndarray] = dataclasses.field(default_factory=list) + + def add(self, text: str) -> None: + """Add a text chunk to the index.""" + self.chunks.append(text) + self.embeddings.append(get_embedding(text, model=self.model)) + + @Tool.define + def retrieve(self, query: str, top_k: int = 3) -> list[str]: + """Return the top-k most similar chunks to the query.""" + if not self.embeddings: + return [] + query_emb = get_embedding(query, model=self.model) + distances = [float(((emb - query_emb) ** 2).sum()) for emb in self.embeddings] + indices = sorted(range(len(distances)), key=lambda i: distances[i]) + return [self.chunks[i] for i in indices[:top_k]] + + +# --------------------------------------------------------------------------- +# Chunking +# --------------------------------------------------------------------------- + + +def chunk_text(text: str, chunk_size: int = 200, overlap: int = 50) -> list[str]: + """Split text into overlapping word-level chunks.""" + words = text.split() + chunks = [] + start = 0 + while start < len(words): + end = start + chunk_size + chunks.append(" ".join(words[start:end])) + start += chunk_size - overlap + return chunks + + +# --------------------------------------------------------------------------- +# Sample documents +# --------------------------------------------------------------------------- + +DOCUMENTS = [ + """The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars + in Paris, France. It is named after the engineer Gustave Eiffel, whose + company designed and built the tower from 1887 to 1889 as the centerpiece + of the 1889 World's Fair. Although initially criticized by some of France's + leading artists and intellectuals, the tower has become a global icon of + France and one of the most recognizable structures in the world. The tower + is 330 metres tall, about the same height as an 81-storey building, and + is the tallest structure in Paris. It was the first structure in the world + to reach a height of 300 metres.""", + """The Great Wall of China is a series of fortifications that were built + across the historical northern borders of ancient Chinese states and + Imperial China as protection against various nomadic groups. The total + length of all sections ever built is more than 20,000 km. Several walls + were built from as early as the 7th century BC, with selective stretches + later joined together by Qin Shi Huang, the first emperor of China. The + best-preserved sections of the wall date from the Ming dynasty + (1368-1644). The wall's purpose was defensive, and it featured + watchtowers, troop barracks, and signaling capabilities.""", + """The Colosseum, also known as the Flavian Amphitheatre, is an oval + amphitheatre in the centre of the city of Rome, Italy. It is the largest + ancient amphitheatre ever built, and is still the largest standing + amphitheatre in the world, despite its age. Construction began under + the emperor Vespasian in AD 72 and was completed in AD 80 under his + successor and heir, Titus. The Colosseum could hold an estimated 50,000 + to 80,000 spectators at various points in its history, and was used for + gladiatorial contests and public spectacles including animal hunts, + executions, re-enactments of famous battles, and dramas.""", +] + +# --------------------------------------------------------------------------- +# Build the index (offline phase) +# --------------------------------------------------------------------------- + + +def build_index(documents: list[str], embedding_model: str) -> VectorIndex: + """Chunk and index a collection of documents.""" + index = VectorIndex(model=embedding_model) + for doc in documents: + for chunk in chunk_text(doc, chunk_size=60, overlap=15): + index.add(chunk) + print(f"Indexed {len(index.chunks)} chunks from {len(documents)} documents") + return index + + +# --------------------------------------------------------------------------- +# RAG query (online phase) +# --------------------------------------------------------------------------- + + +@Template.define +def answer_question(question: str) -> str: + """You are a helpful assistant. Answer the user's question using ONLY + information retrieved from the knowledge base via the retrieve tool. + + If the retrieved information doesn't contain the answer, say so. + Always cite which document your information comes from. + + Question: {question} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Retrieval-augmented generation (RAG)") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--embedding-model", + type=str, + default="lm_studio/nomic-ai/nomic-embed-text-v1.5-GGUF", + help="Embedding model to use", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + # Offline: build the index + index = build_index(DOCUMENTS, embedding_model=args.embedding_model) + + # Create the retrieval tool bound to our index + retrieve: Tool = index.retrieve + + # Online: answer questions + questions = [ + "How tall is the Eiffel Tower?", + "When was the Great Wall of China built?", + "How many spectators could the Colosseum hold?", + ] + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + for question in questions: + print(f"\nQ: {question}") + answer = answer_question(question) + print(f"A: {answer}") diff --git a/docs/source/llm_examples/research_agent.py b/docs/source/llm_examples/research_agent.py new file mode 100644 index 000000000..bc193db1b --- /dev/null +++ b/docs/source/llm_examples/research_agent.py @@ -0,0 +1,144 @@ +"""Research agent with web search. + +Demonstrates: +- ``@defop`` + ``ObjectInterpretation`` to define a pluggable web search effect +- ``@Template.define`` for LLM-implemented answer/refine/judge templates +- Handler composition: stacking a search provider alongside an LLM provider +- Iterative refinement loop: answer → judge → refine → judge → ... +""" + +import argparse +import os +import urllib.parse + +import requests + +from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm.completions import ( + LiteLLMProvider, +) +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Search effect + handler +# --------------------------------------------------------------------------- + + +@Tool.define +def search_web(query: str) -> str: + """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" + search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "list": "search", + "srsearch": query, + "srlimit": 1, + "format": "json", + } + ) + search_data = requests.get( + search_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + results = search_data.get("query", {}).get("search", []) + if not results: + return f"No results found for: {query}" + title = results[0]["title"] + + summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "titles": title, + "prop": "extracts", + "exintro": True, + "explaintext": True, + "format": "json", + } + ) + summary_data = requests.get( + summary_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + page = next(iter(summary_data["query"]["pages"].values())) + extract = page.get("extract", "No summary available.") + url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" + + return f"# {title}\n\n{extract}\n\nSource: {url}" + + +# --------------------------------------------------------------------------- +# Templates (auto-capture `search_web` from lexical scope) +# --------------------------------------------------------------------------- + + +@Template.define +def answer_question(question: str) -> str: + """Acting as a research assistant that can search the web, + construct an answer to the user's question: {question}.""" + raise NotHandled + + +@Template.define +def refine_answer(question: str, answer: str) -> str: + """Acting as a research assistant that can search the web, + given the user's original question ({question}), + refine this previous answer: {answer}.""" + raise NotHandled + + +@Template.define +def is_question_answered(question: str, answer: str) -> bool: + """Acting as a research assistant, decide if the user's question + ({question}) is appropriately answered by: {answer}. + Respond only true or false.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Agent loop +# --------------------------------------------------------------------------- + + +def research_agent(question: str, max_attempts: int = 3) -> str: + """Answer a question, iteratively refining until satisfactory.""" + answer = answer_question(question) + for _ in range(max_attempts): + if is_question_answered(question, answer): + break + answer = refine_answer(question, answer) + return answer + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="LLM-guided research agent with web search" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--question", + type=str, + default="What is the meaning of life?", + help="The question to research", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + with handler(provider): + result = research_agent(args.question) + print(result) diff --git a/docs/source/llm_examples/supervisor.py b/docs/source/llm_examples/supervisor.py new file mode 100644 index 000000000..1009d6a62 --- /dev/null +++ b/docs/source/llm_examples/supervisor.py @@ -0,0 +1,178 @@ +"""Supervisor quality-control wrapper. + +Demonstrates: +- Wrapping an agent's output with a quality-control check +- Using one ``Template`` to judge another's output +- Retry loop driven by LLM-based evaluation +""" + +import argparse +import dataclasses +import os +import urllib.parse + +import requests + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Search tool +# --------------------------------------------------------------------------- + + +@Tool.define +def search_web(query: str) -> str: + """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" + search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "list": "search", + "srsearch": query, + "srlimit": 1, + "format": "json", + } + ) + search_data = requests.get( + search_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + results = search_data.get("query", {}).get("search", []) + if not results: + return f"No results found for: {query}" + title = results[0]["title"] + + summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "titles": title, + "prop": "extracts", + "exintro": True, + "explaintext": True, + "format": "json", + } + ) + summary_data = requests.get( + summary_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + page = next(iter(summary_data["query"]["pages"].values())) + extract = page.get("extract", "No summary available.") + url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" + + return f"# {title}\n\n{extract}\n\nSource: {url}" + + +# --------------------------------------------------------------------------- +# Structured output for quality judgment +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class QualityJudgment: + is_acceptable: bool + feedback: str + + +# --------------------------------------------------------------------------- +# Research agent +# --------------------------------------------------------------------------- + + +class Researcher(Agent): + """Agent that answers research questions using web search.""" + + @Template.define + def answer(self, question: str) -> str: + """You are a research assistant. Answer the following question using + the search tool to find accurate information. + + Question: {question} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Supervisor (quality judge) +# --------------------------------------------------------------------------- + + +@Template.define +def judge_quality(question: str, answer: str) -> QualityJudgment: + """You are a strict quality reviewer. Evaluate whether this answer + adequately addresses the question with accurate, specific information. + + Question: {question} + Answer: {answer} + + An answer is acceptable if it contains specific facts (names, dates, + numbers) relevant to the question. Vague or generic answers should + be rejected. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Supervised agent loop +# --------------------------------------------------------------------------- + + +def supervised_research(question: str, max_retries: int = 3) -> str: + """Answer a question with quality-control supervision. + + The researcher agent answers, the supervisor judges quality, + and if rejected the researcher tries again with feedback. + """ + researcher = Researcher() + + for attempt in range(max_retries + 1): + answer = researcher.answer(question) + judgment = judge_quality(question, answer) + + if judgment.is_acceptable: + print(f"[supervisor] Accepted on attempt {attempt + 1}") + return answer + + print(f"[supervisor] Rejected attempt {attempt + 1}: {judgment.feedback}") + + print("[supervisor] Returning best effort after max retries") + return answer + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Supervised research agent with quality control" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--max-retries", + type=int, + default=3, + help="Maximum number of supervisor rejections before accepting", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + result = supervised_research( + "What year was the Eiffel Tower completed and how tall is it?", + max_retries=args.max_retries, + ) + print(f"\nFinal answer: {result}") diff --git a/docs/source/llm_examples/tao_agent.py b/docs/source/llm_examples/tao_agent.py new file mode 100644 index 000000000..8f9fbb0c1 --- /dev/null +++ b/docs/source/llm_examples/tao_agent.py @@ -0,0 +1,185 @@ +"""Think-Act-Observe chain-of-thought agent. + +Demonstrates: +- ``Agent`` mixin for persistent conversation history +- Structured output with Pydantic models (``AgentThought``) +- A think → act → observe reasoning loop +- Pattern matching for action dispatch +""" + +import argparse +import dataclasses +import enum +import os +import urllib.parse + +import requests + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import ( + LiteLLMProvider, + RetryLLMHandler, +) +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Search tool +# --------------------------------------------------------------------------- + + +@Tool.define +def search_web(query: str) -> str: + """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" + search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "list": "search", + "srsearch": query, + "srlimit": 1, + "format": "json", + } + ) + search_data = requests.get( + search_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + results = search_data.get("query", {}).get("search", []) + if not results: + return f"No results found for: {query}" + title = results[0]["title"] + + summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( + { + "action": "query", + "titles": title, + "prop": "extracts", + "exintro": True, + "explaintext": True, + "format": "json", + } + ) + summary_data = requests.get( + summary_url, headers={"User-Agent": "effectful-example/1.0"} + ).json() + page = next(iter(summary_data["query"]["pages"].values())) + extract = page.get("extract", "No summary available.") + url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" + + return f"# {title}\n\n{extract}\n\nSource: {url}" + + +# --------------------------------------------------------------------------- +# Structured output types +# --------------------------------------------------------------------------- + + +class AgentAction(enum.StrEnum): + search_the_web = "search_the_web" + calculate = "calculate" + answer = "answer" + + +@dataclasses.dataclass(frozen=True) +class AgentThought: + thinking: str + action: AgentAction + action_input: str + is_final: bool + + +# --------------------------------------------------------------------------- +# TAO Agent +# --------------------------------------------------------------------------- + + +class TAOAgent(Agent): + """Think-Act-Observe agent that reasons step by step.""" + + @Template.define + def think(self, query: str) -> AgentThought: + """You are an AI assistant solving a problem. Based on the user's query + ({query}) and prior conversation context, think about what action to + take next. + """ + raise NotHandled + + @Template.define + def observe(self, action: str, action_input: str, action_result: str) -> str: + """You are an observer. Provide a concise, objective observation of this result. + + Action: {action} + Action input: {action_input} + Action result: {action_result} + + + Do not make decisions, just describe what you see. + + """ + raise NotHandled + + def run(self, query: str, max_steps: int = 5) -> str: + result = "" + for _ in range(max_steps): + thought = self.think(query) + result = self._act(thought.action, thought.action_input) + self.observe(str(thought.action), thought.action_input, result) + if thought.is_final: + break + return result + + def _act(self, action: AgentAction, action_input: str) -> str: + match action: + case AgentAction.search_the_web: + return search_web(action_input) + case AgentAction.calculate: + try: + return action_input # eval(action_input)) # noqa: S307 + except Exception as e: + return str(e) + case AgentAction.answer: + return action_input + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="TAO chain-of-thought agent") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--max-steps", + type=int, + default=5, + help="Maximum number of steps before giving up", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed LLM output", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + agent = TAOAgent() + + with handler(provider), handler(RetryLLMHandler(num_retries=args.num_retries)): + answer = agent.run( + "How many tennis balls would fill an Olympic swimming pool?", + max_steps=args.max_steps, + ) + print("Answer:", answer) diff --git a/docs/source/llm_examples/text2sql.py b/docs/source/llm_examples/text2sql.py new file mode 100644 index 000000000..5d511b5f8 --- /dev/null +++ b/docs/source/llm_examples/text2sql.py @@ -0,0 +1,170 @@ +"""Natural language to SQL with LLM-powered debug loop. + +Demonstrates: +- Generating SQL from natural language using ``@Template.define`` +- Executing SQL against a real SQLite database +- Feeding execution errors back to the LLM for iterative fixing +- ``@Tool.define`` to expose the database schema as a tool +""" + +import argparse +import os +import sqlite3 +import textwrap + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# In-memory database setup +# --------------------------------------------------------------------------- + + +def create_sample_db() -> sqlite3.Connection: + """Create a sample SQLite database with employee data.""" + conn = sqlite3.connect(":memory:") + conn.executescript( + textwrap.dedent("""\ + CREATE TABLE departments ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + budget REAL NOT NULL + ); + CREATE TABLE employees ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + department_id INTEGER REFERENCES departments(id), + salary REAL NOT NULL, + hire_date TEXT NOT NULL + ); + INSERT INTO departments VALUES (1, 'Engineering', 500000); + INSERT INTO departments VALUES (2, 'Marketing', 200000); + INSERT INTO departments VALUES (3, 'Sales', 300000); + INSERT INTO employees VALUES (1, 'Alice', 1, 120000, '2020-01-15'); + INSERT INTO employees VALUES (2, 'Bob', 1, 110000, '2021-03-22'); + INSERT INTO employees VALUES (3, 'Carol', 2, 95000, '2019-07-01'); + INSERT INTO employees VALUES (4, 'Dave', 3, 105000, '2022-11-10'); + INSERT INTO employees VALUES (5, 'Eve', 1, 130000, '2018-05-20'); + INSERT INTO employees VALUES (6, 'Frank', 3, 98000, '2023-01-05'); + """) + ) + return conn + + +def get_schema(conn: sqlite3.Connection) -> str: + """Extract the schema from a SQLite database.""" + cursor = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' ORDER BY name" + ) + return "\n\n".join(row[0] for row in cursor if row[0]) + + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def generate_sql(question: str, db_schema: str) -> str: + """You are a SQL expert. Given this database schema: + + {db_schema} + + Write a SQLite query that answers: {question} + + Return ONLY the SQL query, no explanation. + """ + raise NotHandled + + +@Template.define +def fix_sql(question: str, db_schema: str, bad_sql: str, error: str) -> str: + """You are a SQL expert. Your previous query had an error. + + Database schema: + {db_schema} + + Original question: {question} + Failed SQL: {bad_sql} + Error: {error} + + Write a corrected SQLite query. Return ONLY the SQL query. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Text-to-SQL agent with debug loop +# --------------------------------------------------------------------------- + + +def text_to_sql( + conn: sqlite3.Connection, question: str, max_retries: int = 3 +) -> list[tuple]: + """Convert a natural language question to SQL and execute it. + + If the query fails, feed the error back to the LLM to fix it, + up to ``max_retries`` times. + """ + schema = get_schema(conn) + sql = generate_sql(question, schema) + + for attempt in range(max_retries + 1): + # Strip markdown fences if the LLM wraps the SQL + clean_sql = sql.strip().removeprefix("```sql").removesuffix("```").strip() + print(f" [attempt {attempt + 1}] {clean_sql}") + + try: + cursor = conn.execute(clean_sql) + return cursor.fetchall() + except Exception as e: + if attempt < max_retries: + print(f" [error] {e}") + sql = fix_sql(question, schema, clean_sql, str(e)) + else: + raise + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Natural language to SQL with LLM-powered debug loop" + ) + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + conn = create_sample_db() + provider = LiteLLMProvider(model=args.model) + + questions = [ + "What is the average salary by department?", + "Who is the highest paid employee?", + "How many employees were hired after 2021?", + ] + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + for question in questions: + print(f"\nQ: {question}") + try: + rows = text_to_sql(conn, question) + for row in rows: + print(f" => {row}") + except Exception as e: + print(f" FAILED: {e}") diff --git a/docs/source/llm_examples/thinking.py b/docs/source/llm_examples/thinking.py new file mode 100644 index 000000000..101d5cb2e --- /dev/null +++ b/docs/source/llm_examples/thinking.py @@ -0,0 +1,113 @@ +"""Chain-of-thought reasoning with structured self-loop. + +Demonstrates: +- Structured output with a ``ThoughtStep`` dataclass +- An ``Agent`` that loops until it decides it has a final answer +- The LLM sees its own prior reasoning via ``Agent.__history__`` +""" + +import argparse +import dataclasses +import os + +from effectful.handlers.llm import Agent, Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Structured output +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class ThoughtStep: + reasoning: str + conclusion: str + is_final: bool + + +# --------------------------------------------------------------------------- +# Chain-of-thought agent +# --------------------------------------------------------------------------- + + +class Thinker(Agent): + """Agent that reasons step-by-step until it reaches a final answer.""" + + @Template.define + def think(self, problem: str) -> ThoughtStep: + """You are solving a problem step by step. + + Problem: {problem} + + Review the conversation history for any prior reasoning steps. + Continue from where you left off. Break the problem into small, + logical steps. Set is_final=true only when you have a complete, + well-supported answer. + """ + raise NotHandled + + def solve(self, problem: str, max_steps: int = 10) -> str: + """Solve a problem by iterative chain-of-thought reasoning.""" + for i in range(max_steps): + step = self.think(problem) + print(f" [step {i + 1}] {step.reasoning}") + if step.is_final: + return step.conclusion + + return step.conclusion + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Chain-of-thought reasoning agent") + parser.add_argument( + "--model", + type=str, + default="lm_studio/zai-org/glm-4.7-flash", + help="LLM model to use", + ) + parser.add_argument( + "--max-steps", + type=int, + default=10, + help="Maximum reasoning steps before stopping", + ) + parser.add_argument( + "--problem", + type=str, + default=( + "A farmer has 17 sheep. All but 9 run away. " + "Then he buys 5 more. How many sheep does he have now?" + ), + help="The problem to solve", + ) + args = parser.parse_args() + + if args.model.startswith("lm_studio/"): + assert os.environ.get("LM_STUDIO_API_BASE") + elif args.model.startswith("gpt-"): + assert os.environ.get("OPENAI_API_KEY") + elif args.model.startswith("claude-"): + assert os.environ.get("ANTHROPIC_API_KEY") + + provider = LiteLLMProvider(model=args.model) + + problems = [ + args.problem, + ( + "If you have a 3-gallon jug and a 5-gallon jug, " + "how do you measure exactly 4 gallons of water?" + ), + ] + + with handler(provider), handler(RetryLLMHandler(num_retries=3)): + for problem in problems: + thinker = Thinker() + print(f"\nProblem: {problem}") + answer = thinker.solve(problem, max_steps=args.max_steps) + print(f"Answer: {answer}") From 310e3db636f9409d0b149c375558ef4d20efbd22 Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 12:44:56 -0400 Subject: [PATCH 004/155] no changes to library code --- effectful/handlers/llm/completions.py | 472 +++++++------- effectful/handlers/llm/encoding.py | 903 ++++++++++++++------------ effectful/handlers/llm/evaluation.py | 3 +- effectful/handlers/llm/template.py | 223 ++++--- effectful/handlers/numpyro.py | 8 + effectful/internals/unification.py | 80 +++ effectful/ops/semantics.py | 3 +- effectful/ops/types.py | 32 +- 8 files changed, 944 insertions(+), 780 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index f2fd85f22..2e169d932 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -1,8 +1,10 @@ +import abc import collections import collections.abc import dataclasses import functools import inspect +import json import string import textwrap import traceback @@ -11,22 +13,32 @@ import litellm import pydantic +import tenacity from litellm import ( ChatCompletionFunctionMessage, ChatCompletionMessageToolCall, ChatCompletionTextObject, ChatCompletionToolMessage, - ChatCompletionToolParam, OpenAIChatCompletionAssistantMessage, OpenAIChatCompletionSystemMessage, OpenAIChatCompletionUserMessage, OpenAIMessageContentListBlock, ) -from effectful.handlers.llm.encoding import Encodable -from effectful.handlers.llm.template import Template, Tool +from effectful.handlers.llm.encoding import ( + DecodedToolCall, + Encodable, + to_content_blocks, +) +from effectful.handlers.llm.template import ( + Agent, + Template, + Tool, + _is_recursive_signature, +) +from effectful.internals.unification import nested_type from effectful.ops.semantics import fwd, handler -from effectful.ops.syntax import ObjectInterpretation, defop, implements +from effectful.ops.syntax import ObjectInterpretation, implements from effectful.ops.types import Operation @@ -52,14 +64,29 @@ class UserMessage(OpenAIChatCompletionUserMessage): Message = AssistantMessage | ToolMessage | FunctionMessage | SystemMessage | UserMessage +DEFAULT_SYSTEM_PROMPT = ( + "You are a helpful assistant, you need to follow user's instruction" +) + + +class _NoActiveHistoryException(Exception): + """Raised when there is no active message history to append to.""" -@defop -def get_message_sequence() -> collections.OrderedDict[str, Message]: - return collections.OrderedDict() +@Operation.define +def _get_history() -> collections.OrderedDict[str, Message]: + raise _NoActiveHistoryException( + "No active message history. This operation should only be used within a handler that provides a message history." + ) -def append_message(message: Message): - get_message_sequence()[message["id"]] = message + +def append_message(message: Message, last: bool = True) -> None: + try: + _get_history()[message["id"]] = message + if not last: + _get_history().move_to_end(message["id"], last=False) + except _NoActiveHistoryException: + pass def _make_message(content: dict) -> Message: @@ -68,20 +95,27 @@ def _make_message(content: dict) -> Message: return message -type ToolCallID = str +class DecodingError[E: Exception](abc.ABC, Exception): + """Base class for decoding errors that can occur during LLM response processing.""" + + original_error: E + + @abc.abstractmethod + def to_feedback_message(self, include_traceback: bool) -> Message: + """Convert the decoding error into a feedback message to be sent back to the LLM.""" + raise NotImplementedError @dataclasses.dataclass -class ToolCallDecodingError(Exception): +class ToolCallDecodingError[E: Exception](DecodingError[E]): """Error raised when decoding a tool call fails.""" - tool_name: str - tool_call_id: str - original_error: Exception + original_error: E raw_message: Message + raw_tool_call: ChatCompletionMessageToolCall def __str__(self) -> str: - return f"Error decoding tool call '{self.tool_name}': {self.original_error}. Please provide a valid response and try again." + return f"Error decoding tool call '{self.raw_tool_call.function.name}': {self.original_error}. Please provide a valid response and try again." def to_feedback_message(self, include_traceback: bool) -> Message: error_message = f"{self}" @@ -91,17 +125,17 @@ def to_feedback_message(self, include_traceback: bool) -> Message: return _make_message( { "role": "tool", - "tool_call_id": self.tool_call_id, + "tool_call_id": self.raw_tool_call.id, "content": error_message, }, ) @dataclasses.dataclass -class ResultDecodingError(Exception): +class ResultDecodingError[E: Exception](DecodingError[E]): """Error raised when decoding the LLM response result fails.""" - original_error: Exception + original_error: E raw_message: Message def __str__(self) -> str: @@ -118,15 +152,14 @@ def to_feedback_message(self, include_traceback: bool) -> Message: @dataclasses.dataclass -class ToolCallExecutionError(Exception): +class ToolCallExecutionError[E: Exception, T](DecodingError[E]): """Error raised when a tool execution fails at runtime.""" - tool_name: str - tool_call_id: str - original_error: BaseException + original_error: E + raw_tool_call: DecodedToolCall[T] def __str__(self) -> str: - return f"Tool execution failed: Error executing tool '{self.tool_name}': {self.original_error}" + return f"Tool execution failed: Error executing tool '{self.raw_tool_call.name}': {self.original_error}" def to_feedback_message(self, include_traceback: bool) -> Message: error_message = f"{self}" @@ -136,97 +169,44 @@ def to_feedback_message(self, include_traceback: bool) -> Message: return _make_message( { "role": "tool", - "tool_call_id": self.tool_call_id, + "tool_call_id": self.raw_tool_call.id, "content": error_message, }, ) -class DecodedToolCall[T](typing.NamedTuple): - tool: Tool[..., T] - bound_args: inspect.BoundArguments - id: ToolCallID - - type MessageResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] -@functools.cache -def _param_model(tool: Tool) -> type[pydantic.BaseModel]: - sig = inspect.signature(tool) - return pydantic.create_model( - "Params", - __config__={"extra": "forbid"}, - **{ - name: Encodable.define(param.annotation).enc - for name, param in sig.parameters.items() - }, # type: ignore - ) - - -@functools.cache -def _function_model(tool: Tool) -> ChatCompletionToolParam: - response_format = litellm.utils.type_to_response_format_param(_param_model(tool)) - assert response_format is not None - assert tool.__default__.__doc__ is not None - return { - "type": "function", - "function": { - "name": tool.__name__, - "description": textwrap.dedent(tool.__default__.__doc__), - "parameters": response_format["json_schema"]["schema"], - "strict": True, - }, - } - - -def decode_tool_call( - tool_call: ChatCompletionMessageToolCall, - tools: collections.abc.Mapping[str, Tool], - raw_message: Message, -) -> DecodedToolCall: - """Decode a tool call from the LLM response into a DecodedToolCall. - - Args: - tool_call: The tool call to decode. - tools: Mapping of tool names to Tool objects. - raw_message: Optional raw assistant message for error context. - - Raises: - ToolCallDecodingError: If the tool call cannot be decoded. - """ - tool_name = tool_call.function.name - assert tool_name is not None - - try: - tool = tools[tool_name] - except KeyError as e: - raise ToolCallDecodingError( - tool_name, tool_call.id, e, raw_message=raw_message - ) from e - - json_str = tool_call.function.arguments - sig = inspect.signature(tool) - - try: - # build dict of raw encodable types U - raw_args = _param_model(tool).model_validate_json(json_str) - - # use encoders to decode Us to python types T - bound_sig: inspect.BoundArguments = sig.bind( - **{ - param_name: Encodable.define( - sig.parameters[param_name].annotation, {} - ).decode(getattr(raw_args, param_name)) - for param_name in raw_args.model_fields_set - } - ) - except (pydantic.ValidationError, TypeError, ValueError, SyntaxError) as e: - raise ToolCallDecodingError( - tool_name, tool_call.id, e, raw_message=raw_message - ) from e - - return DecodedToolCall(tool, bound_sig, tool_call.id) +def _collect_tools( + env: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.Mapping[str, Tool]: + """Operations and Templates available as tools. Auto-capture from lexical context.""" + result = {} + + for name, obj in env.items(): + # Collect tools directly in context + if isinstance(obj, Tool | Template): + result[name] = obj + + # Collect tools as methods on Agent instances in context + elif isinstance(obj, Agent): + for cls in type(obj).__mro__: + for attr_name in vars(cls): + if isinstance(getattr(obj, attr_name), Tool): + result[f"{name}__{attr_name}"] = getattr(obj, attr_name) + + # The same Tool can appear under multiple names when it is both + # visible in the enclosing scope *and* discovered via an Agent + # instance's MRO. Since Tools are hashable Operations and + # instance-method Tools are cached per instance, we keep only + # the last name for each unique tool object. + tool2name = {tool: name for name, tool in sorted(result.items())} + for name, tool in tuple(result.items()): + if tool2name[tool] != name: + del result[name] + + return result @Operation.define @@ -241,10 +221,14 @@ def completion(*args, **kwargs) -> typing.Any: return litellm.completion(*args, **kwargs) +class _BoxedResponse[T](pydantic.BaseModel): + value: T + + @Operation.define -def call_assistant[T, U]( - tools: collections.abc.Mapping[str, Tool], - response_format: Encodable[T, U], +def call_assistant[T]( + env: collections.abc.Mapping[str, typing.Any], + response_type: type[T], model: str, **kwargs, ) -> MessageResult[T]: @@ -259,20 +243,28 @@ def call_assistant[T, U]( ResultDecodingError: If the result cannot be decoded. The error includes the raw assistant message for retry handling. """ - tool_specs = {k: _function_model(t) for k, t in tools.items()} - response_model = ( - response_format.enc - if issubclass(response_format.enc, pydantic.BaseModel) - else pydantic.create_model( - "Response", value=response_format.enc, __config__={"extra": "forbid"} - ) + tools = _collect_tools(env) + tool_specs = { + k: typing.cast( + pydantic.TypeAdapter[typing.Any], + pydantic.TypeAdapter(Encodable[type(t)]), # type: ignore[misc] + ).dump_python(t, mode="json", context={k: t}) + for k, t in tools.items() + } + + # The OpenAI API requires a wrapper object for non-object structured output types, + # so we create one on the fly here. Using a Pydantic model offloads JSON schema + # generation and validation logic to litellm, and offers better error messages. + response_format: type[_BoxedResponse[T]] = pydantic.create_model( + "BoxedResponse", + value=Encodable[response_type], # type: ignore[valid-type] + __base__=_BoxedResponse, ) - messages = list(get_message_sequence().values()) response: litellm.types.utils.ModelResponse = completion( model, - messages=list(messages), - response_format=response_model if response_format.enc is not str else None, + messages=list(_get_history().values()), + response_format=None if response_type is str else response_format, tools=list(tool_specs.values()), **kwargs, ) @@ -286,35 +278,35 @@ def call_assistant[T, U]( append_message(raw_message) tool_calls: list[DecodedToolCall] = [] - raw_tool_calls = message.get("tool_calls") or [] - for raw_tool_call in raw_tool_calls: - validated_tool_call = ChatCompletionMessageToolCall.model_validate( - raw_tool_call - ) - decoded_tool_call = decode_tool_call(validated_tool_call, tools, raw_message) - tool_calls.append(decoded_tool_call) + encoding: pydantic.TypeAdapter[DecodedToolCall] = pydantic.TypeAdapter( + Encodable[DecodedToolCall] + ) + for raw_tool_call in message.get("tool_calls") or []: + try: + tool_calls += [encoding.validate_python(raw_tool_call, context=tools)] + except Exception as e: + raise ToolCallDecodingError( + raw_tool_call=raw_tool_call, + original_error=e, + raw_message=raw_message, + ) from e result = None - if not tool_calls and response_format.enc is not str: + if not tool_calls: # return response serialized_result = message.get("content") or message.get("reasoning_content") assert isinstance(serialized_result, str), ( "final response from the model should be a string" ) - try: - raw_result = response_model.model_validate_json(serialized_result) - result = response_format.decode( - raw_result.value - if not issubclass(response_format.enc, pydantic.BaseModel) - else raw_result - ) # type: ignore - except (pydantic.ValidationError, TypeError, ValueError, SyntaxError) as e: - raise ResultDecodingError(e, raw_message=raw_message) from e - elif not tool_calls and response_format.enc is str: - # if expecting a string result, return the raw content as the result - content = message.get("content") or message.get("reasoning_content") - assert isinstance(content, str), "Expected content to be a string" - result = content + if response_type is str: + result = typing.cast(T, serialized_result) + else: + try: + result = response_format.model_validate( + json.loads(serialized_result), context=env + ).value + except Exception as e: + raise ResultDecodingError(e, raw_message=raw_message) from e return (raw_message, tool_calls, result) @@ -327,16 +319,19 @@ def call_tool(tool_call: DecodedToolCall) -> Message: """ # call tool with python types - # call_tool invariant: tool is called in a context with a fresh message sequence - message_sequence: collections.OrderedDict[str, Message] = collections.OrderedDict() - with handler({get_message_sequence: lambda: message_sequence}): + try: result = tool_call.tool( *tool_call.bound_args.args, **tool_call.bound_args.kwargs ) + except Exception as e: + raise ToolCallExecutionError(raw_tool_call=tool_call, original_error=e) from e - # serialize back to U using encoder for return type - return_type = Encodable.define(type(result)) - encoded_result = return_type.serialize(return_type.encode(result)) + return_type: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( + Encodable[nested_type(result).value] # type: ignore[misc] + ) + encoded_result = to_content_blocks( + return_type.dump_python(result, mode="json", context={}) + ) message = _make_message( dict(role="tool", content=encoded_result, tool_call_id=tool_call.id), ) @@ -372,11 +367,11 @@ def flush_text() -> None: continue obj, _ = formatter.get_field(field_name, (), env) - encoder = Encodable.define(type(obj)) - encoded_obj: typing.Sequence[OpenAIMessageContentListBlock] = encoder.serialize( - encoder.encode(obj) + encoder: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( + Encodable[nested_type(obj).value] # type: ignore[misc] ) - for part in encoded_obj: + encoded_obj = encoder.dump_python(obj, mode="json", context=env) + for part in to_content_blocks(encoded_obj): if part["type"] == "text": text = ( formatter.convert_field(part["text"], conversion) @@ -398,38 +393,12 @@ def flush_text() -> None: @Operation.define -def call_system(template: Template) -> collections.abc.Sequence[Message]: +def call_system(template: Template) -> Message: """Get system instruction message(s) to prepend to all LLM prompts.""" - - assert inspect.getdoc(type(template)) is not None - - system_prompt = inspect.cleandoc(f""" - You are responsible for implementing the `Template` '{template.__name__}' defined in the module source code below. - - First, as background, here is the class-level documentation for the `Template` class:: - - {inspect.getdoc(type(template))} - """) - - try: - system_prompt += inspect.cleandoc(f""" - Here is the source code of the module defining the `Template` instance '{template.__name__}':: - - {inspect.getsource(inspect.getmodule(template))} - """) - except (TypeError, OSError): - system_prompt += inspect.cleandoc(f""" - The source code for the module defining '{template.__name__}' is not available. - Instead, here are the signature and docstring of '{template.__name__}':: - - {template.__name__} :: {template.__signature__.format()} - - {inspect.cleandoc(template.__prompt_template__)} - """) - - msg = _make_message(dict(role="system", content=system_prompt)) - append_message(msg) - return (msg,) + system_prompt = template.__system_prompt__ or DEFAULT_SYSTEM_PROMPT + message = _make_message(dict(role="system", content=system_prompt)) + append_message(message, last=False) + return message class RetryLLMHandler(ObjectInterpretation): @@ -444,69 +413,68 @@ class RetryLLMHandler(ObjectInterpretation): captured and returned as tool response messages. Args: - num_retries: The maximum number of retries (default: 3). include_traceback: If True, include full traceback in error feedback - for better debugging context (default: False). + for better debugging context (default: True). catch_tool_errors: Exception type(s) to catch during tool execution. Can be a single exception class or a tuple of exception classes. Defaults to Exception (catches all exceptions). + stop: tenacity stop condition for retrying `call_assistant`. Defaults to + `tenacity.stop_after_attempt(4)`, which stops after 4 attempts. + **kwargs: Additional keyword arguments forwarded to `tenacity.Retrying`. """ + call_assistant_retryer: tenacity.Retrying + + _user_before_sleep: collections.abc.Callable[[tenacity.RetryCallState], None] | None + def __init__( self, - num_retries: int = 3, - include_traceback: bool = False, + include_traceback: bool = True, catch_tool_errors: type[BaseException] | tuple[type[BaseException], ...] = Exception, + stop: tenacity.stop.stop_base = tenacity.stop_after_attempt(4), + **kwargs, ): - self.num_retries = num_retries self.include_traceback = include_traceback self.catch_tool_errors = catch_tool_errors + assert "retry" not in kwargs, "Cannot override retry logic of RetryLLMHandler" + assert "reraise" not in kwargs, ( + "Cannot override reraise logic of RetryLLMHandler" + ) + self._user_before_sleep = kwargs.pop("before_sleep", None) + self.call_assistant_retryer = tenacity.Retrying( + retry=tenacity.retry_if_exception_type( + (ToolCallDecodingError, ResultDecodingError) + ), + reraise=True, + before_sleep=self._before_sleep, + stop=stop, + **kwargs, + ) + + def _before_sleep(self, retry_state: tenacity.RetryCallState) -> None: + e = retry_state.outcome.exception() # type: ignore + assert isinstance(e, (ToolCallDecodingError, ResultDecodingError)) + append_message(e.raw_message) + append_message(e.to_feedback_message(self.include_traceback)) + if self._user_before_sleep is not None: + self._user_before_sleep(retry_state) @implements(call_assistant) - def _call_assistant[T, U]( + def _call_assistant[T]( self, - tools: collections.abc.Mapping[str, Tool], - response_format: Encodable[T, U], + env: collections.abc.Mapping[str, typing.Any], + response_type: type[T], model: str, **kwargs, ) -> MessageResult[T]: - message_sequence = get_message_sequence().copy() - last_attempt = self.num_retries - - for attempt in range(self.num_retries + 1): - try: - # call assistant, use saved message_sequence - with handler({get_message_sequence: lambda: message_sequence}): - message, tool_calls, result = fwd( - tools, response_format, model, **kwargs - ) - - # Success! The returned message is the final successful response. - # Malformed messages from retries are only in local message_sequence copy, - # not in the enclosing message sequence. - append_message(message) - return (message, tool_calls, result) - - except (ToolCallDecodingError, ResultDecodingError) as e: - # On last attempt, re-raise to preserve full traceback - if attempt == last_attempt: - raise - - # Add the malformed assistant message - message_sequence[e.raw_message["id"]] = e.raw_message + _message_sequence = _get_history().copy() - # Add error feedback as a tool response - error_feedback: Message = e.to_feedback_message(self.include_traceback) - message_sequence[error_feedback["id"]] = error_feedback + with handler({_get_history: lambda: _message_sequence}): + message, tool_calls, result = self.call_assistant_retryer(fwd) - # Should never reach here - either we return on success or raise on final failure - raise AssertionError("Unreachable: retry loop exited without return or raise") - - @implements(completion) - def _completion(self, *args, **kwargs) -> typing.Any: - """Inject num_retries for litellm's built-in network error handling.""" - return fwd(*args, **({"num_retries": self.num_retries} | kwargs)) + append_message(message) + return (message, tool_calls, result) @implements(call_tool) def _call_tool(self, tool_call: DecodedToolCall) -> Message: @@ -518,11 +486,13 @@ def _call_tool(self, tool_call: DecodedToolCall) -> Message: """ try: return fwd(tool_call) - except self.catch_tool_errors as e: - error = ToolCallExecutionError(tool_call.tool.__name__, tool_call.id, e) - message = error.to_feedback_message(self.include_traceback) - append_message(message) - return message + except ToolCallExecutionError as e: + if isinstance(e.original_error, self.catch_tool_errors): + message = e.to_feedback_message(self.include_traceback) + append_message(message) + return message + else: + raise class LiteLLMProvider(ObjectInterpretation): @@ -540,19 +510,25 @@ def __init__(self, model="gpt-4o", **config): def _call[**P, T]( self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs ) -> T: - message_sequence: collections.OrderedDict[str, Message] = get_message_sequence() - with handler({get_message_sequence: lambda: message_sequence}): - # encode arguments - bound_args = inspect.signature(template).bind(*args, **kwargs) - bound_args.apply_defaults() - env = template.__context__.new_child(bound_args.arguments) - - # Create response_model with env so tools passed as arguments are available - response_model = Encodable.define( - template.__signature__.return_annotation, env - ) - - call_system(template) + # encode arguments + bound_args = inspect.signature(template).bind(*args, **kwargs) + bound_args.apply_defaults() + env = template.__context__.new_child(bound_args.arguments) + + if not _is_recursive_signature(template.__signature__): + env = env.new_child({k: None for k, v in env.items() if v is template}) + + history: collections.OrderedDict[str, Message] = getattr( + template, "__history__", collections.OrderedDict() + ) # type: ignore + history_copy = history.copy() + + with handler({_get_history: lambda: history_copy}): + if ( + not _get_history() + or next(iter(_get_history().values()))["role"] != "system" + ): + call_system(template) message: Message = call_user(template.__prompt_template__, env) @@ -561,12 +537,14 @@ def _call[**P, T]( result: T | None = None while message["role"] != "assistant" or tool_calls: message, tool_calls, result = call_assistant( - template.tools, response_model, **self.config + env, template.__signature__.return_annotation, **self.config ) for tool_call in tool_calls: message = call_tool(tool_call) - assert result is not None, ( - "call_assistant did not produce a result nor tool_calls" - ) - return result + try: + _get_history() + except _NoActiveHistoryException: + history.clear() + history.update(history_copy) + return typing.cast(T, result) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 66edf6add..cfbb08aec 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -1,286 +1,347 @@ import ast import base64 +import dataclasses +import functools import inspect import io +import json import textwrap import types import typing -from abc import ABC, abstractmethod -from collections.abc import Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass -from types import CodeType +from collections.abc import ( + Callable, + Mapping, + MutableMapping, +) from typing import Any +import litellm import pydantic from litellm import ( - ChatCompletionImageUrlObject, + ChatCompletionImageObject, + ChatCompletionMessageToolCall, + ChatCompletionTextObject, + ChatCompletionToolParam, OpenAIMessageContentListBlock, ) +from openai.lib._pydantic import _ensure_strict_json_schema +from openai.types.chat import ( + ChatCompletionMessageToolCall as OpenAIChatCompletionMessageToolCall, +) from PIL import Image import effectful.handlers.llm.evaluation as evaluation -from effectful.ops.semantics import _simple_type -from effectful.ops.syntax import _CustomSingleDispatchCallable +from effectful.handlers.llm.template import Tool +from effectful.internals.unification import GenericAlias, TypeEvaluator, nested_type from effectful.ops.types import Operation, Term +type ToolCallID = str + +CONTENT_BLOCK_TYPES: frozenset[str] = frozenset( + literal + for member in typing.get_args(OpenAIMessageContentListBlock) + for literal in typing.get_args(typing.get_type_hints(member).get("type", str)) + if isinstance(literal, str) +) -def _pil_image_to_base64_data(pil_image: Image.Image) -> str: - buf = io.BytesIO() - pil_image.save(buf, format="PNG") - return base64.b64encode(buf.getvalue()).decode("utf-8") +@pydantic.validate_call(validate_return=True) +def to_content_blocks(value: typing.Any) -> list[OpenAIMessageContentListBlock]: + """Convert an encoded JSON-compatible value into a flat list of content blocks. -def _pil_image_to_base64_data_uri(pil_image: Image.Image) -> str: - return f"data:image/png;base64,{_pil_image_to_base64_data(pil_image)}" + Walks the value tree, extracting content-block-shaped dicts (identified by + their ``type`` discriminator) and emitting JSON syntax as text around them. + Top-level strings are emitted bare (for natural template rendering). + Inside JSON structures, separators match ``json.dumps`` defaults so that + the linearization law holds for non-string encoded values: + ``linearize(to_content_blocks(v)) == json.dumps(v)``. + """ + if isinstance(value, str): + return [ChatCompletionTextObject(type="text", text=value)] + + buf: list[str] = [] + blocks: list[OpenAIMessageContentListBlock] = [] + + def flush() -> None: + if buf: + blocks.append(ChatCompletionTextObject(type="text", text="".join(buf))) + buf.clear() + + def walk(v: typing.Any) -> None: + if isinstance(v, dict) and v.get("type") in CONTENT_BLOCK_TYPES: + flush() + blocks.append(typing.cast(OpenAIMessageContentListBlock, v)) + elif isinstance(v, dict): + buf.append("{") + for i, (k, val) in enumerate(v.items()): + if i: + buf.append(", ") + buf.append(json.dumps(k) + ": ") + walk(val) + buf.append("}") + elif isinstance(v, list): + buf.append("[") + for i, item in enumerate(v): + if i: + buf.append(", ") + walk(item) + buf.append("]") + else: + buf.append(json.dumps(v)) + + walk(value) + flush() + return blocks -class Encodable[T, U](ABC): - base: type[T] - enc: type[U] - ctx: Mapping[str, Any] - @abstractmethod - def encode(self, value: T) -> U: - raise NotImplementedError +@dataclasses.dataclass(frozen=True, eq=True) +class DecodedToolCall[T]: + """ + Structured representation of a tool call decoded from an LLM response. + """ - @abstractmethod - def decode(self, encoded_value: U) -> T: - raise NotImplementedError + tool: Tool[..., T] + bound_args: inspect.BoundArguments + id: ToolCallID + name: str - @abstractmethod - def serialize(self, encoded_value: U) -> Sequence[OpenAIMessageContentListBlock]: - raise NotImplementedError - # serialize and deserialize have different types reflecting the LLM api chat.completions(list[content]) -> str - @abstractmethod - def deserialize(self, serialized_value: str) -> U: - raise NotImplementedError +if typing.TYPE_CHECKING: + type Encodable[T] = typing.Annotated[T, "encoded"] +else: + + class Encodable: + def __class_getitem__(cls, item): + return TypeToPydanticType().evaluate(item) + + +class TypeToPydanticType(TypeEvaluator): + """Substitute custom types with their Pydantic Annotated equivalents. + + Recursively walks a type annotation tree, replacing leaf types that have + registered Pydantic annotations (e.g., Image.Image -> PydanticImage) and + reconstructing the full generic type. + + The result can be passed to pydantic.TypeAdapter() for automatic + validation and serialization of nested structures. + """ - @typing.final @staticmethod - @_CustomSingleDispatchCallable - def define( - __dispatch: Callable[ - [type[T]], Callable[[type[T], Mapping[str, Any] | None], "Encodable[T, U]"] - ], - t: type[T], - ctx: Mapping[str, Any] | None = None, - ) -> "Encodable[T, U]": - dispatch_ty = _simple_type(t) - return __dispatch(dispatch_ty)(t, ctx) + @functools.singledispatch + def _registry(ty: type): + raise RuntimeError("should not be here!") + + @classmethod + def register(cls, *args, **kwargs): + return cls._registry.register(*args, **kwargs) + + def evaluate(self, ty): + app = super().evaluate(ty) + origin = typing.get_origin(app) + # Only dispatch on regular types. Special forms (Literal, Annotated, + # Union) have non-type origins that singledispatch can't resolve; pass + # them through for Pydantic to handle natively. + if isinstance(app, type | GenericAlias) and ( + origin is None or isinstance(origin, type) + ): + return self._registry.dispatch(origin or app)(app) + else: + return app -@dataclass -class BaseEncodable[T](Encodable[T, T]): - base: type[T] - enc: type[T] - ctx: Mapping[str, Any] - adapter: pydantic.TypeAdapter[T] +@TypeToPydanticType.register(str) +def _pydantic_type_str[T](ty: type[T]) -> type[T]: + return ty - def encode(self, value: T) -> T: - return typing.cast(T, self.adapter.validate_python(value)) - def decode(self, encoded_value: T) -> T: - return typing.cast(T, self.adapter.validate_python(encoded_value)) +@TypeToPydanticType.register(object) +def _pydantic_type_base(ty: type) -> Any: + return ty - def serialize(self, encoded_value: T) -> Sequence[OpenAIMessageContentListBlock]: - json_str = self.adapter.dump_json(encoded_value).decode("utf-8") - return [{"type": "text", "text": json_str}] - def deserialize(self, serialized_value: str) -> T: - # Parse JSON string into the encoded value, validated as `ty`. - return typing.cast(T, self.adapter.validate_json(serialized_value)) +class _ComplexModel(typing.TypedDict): + real: float + imag: float -@dataclass -class StrEncodable(Encodable[str, str]): - base: type[str] - enc: type[str] - ctx: Mapping[str, Any] +@pydantic.validate_call(validate_return=True) +def _validate_complex(value: _ComplexModel) -> complex: + return complex(value["real"], value["imag"]) - def encode(self, value: str) -> str: - return value - def decode(self, encoded_value: str) -> str: - return encoded_value +@pydantic.validate_call(validate_return=True) +def _serialize_complex(value: complex) -> _ComplexModel: + return {"real": value.real, "imag": value.imag} - def serialize(self, encoded_value: str) -> Sequence[OpenAIMessageContentListBlock]: - # Serialize strings without JSON encoding (no extra quotes) - return [{"type": "text", "text": encoded_value}] - def deserialize(self, serialized_value: str) -> str: - return serialized_value +@TypeToPydanticType.register(complex) +def _pydantic_type_complex(ty): + """Encode ``complex`` as ``{"real": float, "imag": float}``.""" + adapted_schema = pydantic.TypeAdapter(_ComplexModel).json_schema() -@dataclass -class PydanticBaseModelEncodable[T: pydantic.BaseModel](Encodable[T, T]): - base: type[T] - enc: type[T] - ctx: Mapping[str, Any] + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate_complex), + pydantic.PlainSerializer(_serialize_complex), + pydantic.WithJsonSchema({**adapted_schema, "additionalProperties": False}), + ] - def decode(self, encoded_value: T) -> T: - return encoded_value - def encode(self, value: T) -> T: - return value +def _inline_refs(schema: dict) -> dict: + """Inline ``$ref`` pointers so ``WithJsonSchema`` never emits orphan refs. - def serialize(self, encoded_value: T) -> Sequence[OpenAIMessageContentListBlock]: - return [{"type": "text", "text": encoded_value.model_dump_json()}] + Workaround for https://github.com/pydantic/pydantic/issues/12145 — + Pydantic's ``GenerateJsonSchema`` does not merge user-provided ``$defs`` + into its internal ref map, so any ``$ref`` in a ``WithJsonSchema`` value + causes a ``KeyError`` when the annotated type is composed into a model. + """ + defs = schema.get("$defs", {}) - def deserialize(self, serialized_value: str) -> T: - return typing.cast(T, self.base.model_validate_json(serialized_value)) + def _resolve(obj): + if isinstance(obj, dict): + if "$ref" in obj: + ref_name = obj["$ref"].split("/")[-1] + if ref_name in defs: + return _resolve(defs[ref_name]) + return {k: _resolve(v) for k, v in obj.items() if k != "$defs"} + if isinstance(obj, list): + return [_resolve(item) for item in obj] + return obj + return _resolve(schema) -@dataclass -class ImageEncodable(Encodable[Image.Image, ChatCompletionImageUrlObject]): - base: type[Image.Image] - enc: type[ChatCompletionImageUrlObject] - ctx: Mapping[str, Any] - def encode(self, value: Image.Image) -> ChatCompletionImageUrlObject: - return { - "detail": "auto", - "url": _pil_image_to_base64_data_uri(value), - } +@TypeToPydanticType.register(tuple) +def _pydantic_type_tuple(ty): + """Convert finitary tuples to object-based schemas (``properties/required``). - def decode(self, encoded_value: ChatCompletionImageUrlObject) -> Image.Image: - image_url = encoded_value["url"] - if not image_url.startswith("data:image/"): - raise RuntimeError( - f"expected base64 encoded image as data uri, received {image_url}" - ) - data = image_url.split(",")[1] - return Image.open(fp=io.BytesIO(base64.b64decode(data))) - - def serialize( - self, encoded_value: ChatCompletionImageUrlObject - ) -> Sequence[OpenAIMessageContentListBlock]: - return [{"type": "image_url", "image_url": encoded_value}] - - def deserialize(self, serialized_value: str) -> ChatCompletionImageUrlObject: - # Images are serialized as image_url blocks, not text - # This shouldn't be called in normal flow, but provide a fallback - raise NotImplementedError("Image deserialization from string is not supported") - - -@dataclass -class TupleEncodable[T](Encodable[T, typing.Any]): - base: type[T] - enc: type[typing.Any] - ctx: Mapping[str, Any] - has_image: bool - element_encoders: list[Encodable] - - def encode(self, value: T) -> typing.Any: - if not isinstance(value, tuple): - raise TypeError(f"Expected tuple, got {type(value)}") - if len(value) != len(self.element_encoders): - raise ValueError( - f"Tuple length {len(value)} does not match expected length {len(self.element_encoders)}" - ) - return tuple( - [enc.encode(elem) for enc, elem in zip(self.element_encoders, value)] + OpenAI's strict mode rejects the ``prefixItems`` array schema that Pydantic + emits for fixed-length tuples. We convert them to a Pydantic model with + positional ``item_0``, ``item_1``, … fields instead. + + NamedTuples are handled similarly using their field names. + Bare ``tuple`` and variadic ``tuple[T, ...]`` are passed through unchanged. + """ + # NamedTuple subclasses dispatch here via MRO; use field names. + if isinstance(ty, type) and hasattr(ty, "_fields"): + hints = typing.get_type_hints(ty) + nt_fields: list[str] = list(ty._fields) + nt_types = [hints.get(f, typing.Any) for f in nt_fields] + nt_adapters = [pydantic.TypeAdapter(t) for t in nt_types] + nt_model = pydantic.create_model( + ty.__name__, + __config__={"extra": "forbid"}, + **{f: (t, ...) for f, t in zip(nt_fields, nt_types)}, ) - def decode(self, encoded_value: typing.Any) -> T: - if len(encoded_value) != len(self.element_encoders): - raise ValueError( - f"tuple length {len(encoded_value)} does not match expected length {len(self.element_encoders)}" + def _nt_validate(value, info: pydantic.ValidationInfo): + if isinstance(value, tuple | list): + value = dict(zip(nt_fields, value)) + return ty( + **{ + f: nt_adapters[i].validate_python(value[f], context=info.context) + for i, f in enumerate(nt_fields) + } ) - decoded_elements: list[typing.Any] = [ - enc.decode(elem) for enc, elem in zip(self.element_encoders, encoded_value) - ] - return typing.cast(T, tuple(decoded_elements)) - - def serialize( - self, encoded_value: typing.Any - ) -> Sequence[OpenAIMessageContentListBlock]: - if self.has_image: - # If tuple contains images, serialize each element and flatten the results - result: list[OpenAIMessageContentListBlock] = [] - if not isinstance(encoded_value, tuple): - raise TypeError(f"Expected tuple, got {type(encoded_value)}") - if len(encoded_value) != len(self.element_encoders): - raise ValueError( - f"Tuple length {len(encoded_value)} does not match expected length {len(self.element_encoders)}" + + def _nt_serialize(value, info: pydantic.SerializationInfo): + return { + f: nt_adapters[i].dump_python( + getattr(value, f), mode="json", context=info.context ) - for enc, elem in zip(self.element_encoders, encoded_value): - result.extend(enc.serialize(elem)) - return result - else: - # Use base serialization for non-image tuples - adapter: pydantic.TypeAdapter[tuple] = pydantic.TypeAdapter(self.enc) - json_str = adapter.dump_json(encoded_value).decode("utf-8") - return [{"type": "text", "text": json_str}] - - def deserialize(self, serialized_value: str) -> typing.Any: - adapter: pydantic.TypeAdapter[tuple] = pydantic.TypeAdapter(self.enc) - return typing.cast(typing.Any, adapter.validate_json(serialized_value)) - - -@dataclass -class ListEncodable[T](Encodable[list[T], typing.Any]): - base: type[list[T]] - enc: type[typing.Any] - ctx: Mapping[str, Any] - has_image: bool - element_encoder: Encodable[T, typing.Any] - - def encode(self, value: list[T]) -> typing.Any: - if not isinstance(value, list): - raise TypeError(f"Expected list, got {type(value)}") - return [self.element_encoder.encode(elem) for elem in value] - - def decode(self, encoded_value: typing.Any) -> list[T]: - decoded_elements: list[T] = [ - self.element_encoder.decode(elem) for elem in encoded_value + for i, f in enumerate(nt_fields) + } + + return typing.Annotated[ + ty, + pydantic.PlainValidator(_nt_validate), + pydantic.PlainSerializer(_nt_serialize), + pydantic.WithJsonSchema(_inline_refs(nt_model.model_json_schema())), ] - return typing.cast(list[T], decoded_elements) - - def serialize( - self, encoded_value: typing.Any - ) -> Sequence[OpenAIMessageContentListBlock]: - if self.has_image: - # If list contains images, serialize each element and flatten the results - result: list[OpenAIMessageContentListBlock] = [] - if not isinstance(encoded_value, list): - raise TypeError(f"Expected list, got {type(encoded_value)}") - for elem in encoded_value: - result.extend(self.element_encoder.serialize(elem)) - return result - else: - # Use base serialization for non-image lists - adapter = pydantic.TypeAdapter(self.enc) - json_str = adapter.dump_json(encoded_value).decode("utf-8") - return [{"type": "text", "text": json_str}] - def deserialize(self, serialized_value: str) -> typing.Any: - adapter = pydantic.TypeAdapter(self.enc) - return typing.cast(typing.Any, adapter.validate_json(serialized_value)) + args = typing.get_args(ty) + # Bare tuple or tuple[T, ...] — Pydantic's native handling is fine. + # Note: tuple[()] also has get_args() == (), but has origin=tuple. + if (not args and typing.get_origin(ty) is None) or ( + len(args) == 2 and args[1] is Ellipsis + ): + return ty -def _format_callable_type(callable_type: type[Callable]) -> str: - """Format a Callable type annotation as a string for LLM instructions.""" - args = typing.get_args(callable_type) - if not args: - return "Callable" + # tuple[()] (empty args with origin) maps to zero fields; otherwise use args. + effective: list[typing.Any] = list(args) - # Callable[[arg1, arg2, ...], return_type] - if len(args) >= 2: - param_types = args[0] - return_type = args[-1] + adapters = [pydantic.TypeAdapter(a) for a in effective] - if param_types is ...: - params_str = "..." - elif isinstance(param_types, list | tuple): - params_str = ", ".join(getattr(t, "__name__", str(t)) for t in param_types) - else: - params_str = str(param_types) + model = pydantic.create_model( + "TupleItems", + __config__={"extra": "forbid"}, + **{f"item_{i}": (a, ...) for i, a in enumerate(effective)}, + ) - return_str = getattr(return_type, "__name__", str(return_type)) - return f"Callable[[{params_str}], {return_str}]" + def _validate(value, info: pydantic.ValidationInfo): + if isinstance(value, tuple | list): + value = {f"item_{i}": v for i, v in enumerate(value)} + return tuple( + adapters[i].validate_python(value[f"item_{i}"], context=info.context) + for i in range(len(effective)) + ) + + def _serialize(value, info: pydantic.SerializationInfo): + return { + f"item_{i}": adapters[i].dump_python(v, mode="json", context=info.context) + for i, v in enumerate(value) + } - return str(callable_type) + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate), + pydantic.PlainSerializer(_serialize), + pydantic.WithJsonSchema(_inline_refs(model.model_json_schema())), + ] + + +@TypeToPydanticType.register(Term) +def _pydantic_type_term(ty: type[Term]): + raise TypeError("Terms cannot be converted to Pydantic types.") + + +@TypeToPydanticType.register(Operation) +def _pydantic_type_operation(ty: type[Operation]): + raise TypeError("Operations cannot be converted to Pydantic types.") + + +@pydantic.validate_call(validate_return=False) +def _validate_image(value: ChatCompletionImageObject) -> Image.Image: + value = pydantic.TypeAdapter(ChatCompletionImageObject).validate_python(value) + image_url: litellm.ChatCompletionImageUrlObject | str = value["image_url"] + url: str = image_url["url"] if isinstance(image_url, dict) else image_url + prefix, data = url.split(",") + if not prefix.startswith("data:image/"): + raise ValueError(f"expected base64 encoded image as data uri, received {url}") + return Image.open(fp=io.BytesIO(base64.b64decode(data))) + + +def _serialize_image(value: Image.Image) -> ChatCompletionImageObject: + buf = io.BytesIO() + value.save(buf, format="PNG") + url = f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}" + return pydantic.TypeAdapter(ChatCompletionImageObject).validate_python( + {"type": "image_url", "image_url": {"detail": "auto", "url": url}} + ) + + +@TypeToPydanticType.register(Image.Image) +def _pydantic_type_image(ty: type[Image.Image]): + adapter = pydantic.TypeAdapter(ChatCompletionImageObject) + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate_image), + pydantic.PlainSerializer(_serialize_image), + pydantic.WithJsonSchema(_inline_refs(adapter.json_schema())), + ] class SynthesizedFunction(pydantic.BaseModel): @@ -303,7 +364,24 @@ def _create_typed_synthesized_function( Uses pydantic.create_model to ensure the description is included in the JSON schema sent to the LLM, informing it of the expected function signature. """ - type_signature = _format_callable_type(callable_type) + if not typing.get_args(callable_type): + type_signature = "Callable" + # Callable[[arg1, arg2, ...], return_type] + elif len(typing.get_args(callable_type)) >= 2: + param_types = typing.get_args(callable_type)[0] + return_type = typing.get_args(callable_type)[-1] + + if param_types is ...: + params_str = "..." + elif isinstance(param_types, list | tuple): + params_str = ", ".join(getattr(t, "__name__", str(t)) for t in param_types) + else: + params_str = str(param_types) + + return_str = getattr(return_type, "__name__", str(return_type)) + type_signature = f"Callable[[{params_str}], {return_str}]" + else: + type_signature = str(callable_type) description = f"""Given the specification above, generate a Python function satisfying the following specification and type signature. @@ -367,70 +445,56 @@ def _validate_signature_callable( ) -@dataclass -class CallableEncodable(Encodable[Callable, SynthesizedFunction]): - base: type[Callable] - enc: type[SynthesizedFunction] - ctx: Mapping[str, Any] - expected_params: list[type] | None = None - expected_return: type | None = None # None means decode is disabled - - def encode(self, t: Callable) -> SynthesizedFunction: - # (https://github.com/python/mypy/issues/14928) - if not isinstance(t, Callable): # type: ignore - raise TypeError(f"Expected callable, got {type(t)}") +@TypeToPydanticType.register(Callable) +def _pydantic_callable(callable_type: Any) -> Any: + """Create a Pydantic-compatible Annotated type for a parameterized Callable. - try: - source = inspect.getsource(t) - except (OSError, TypeError): - source = None + Usage: PydanticCallable(Callable[[int, str], bool]) + """ + type_args = typing.get_args(callable_type) - if source: - return self.enc(module_code=textwrap.dedent(source)) - - # Source not available - create stub from name, signature, and docstring - # This is useful for builtins and C extensions - name = getattr(t, "__name__", None) - if not name: - raise RuntimeError( - f"Cannot encode callable {t}: no source code and no __name__" + if not type_args: + typed_enc = _create_typed_synthesized_function(Callable[..., typing.Any]) # type: ignore[arg-type] + expected_params = None + expected_return = None + else: + if len(type_args) < 2: + raise TypeError( + f"Callable type signature incomplete: {callable_type}. " + "Expected Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType]." ) - - try: - sig = inspect.signature(t) - sig_str = str(sig) - except (ValueError, TypeError): - # Some builtins don't have inspectable signatures - sig_str = "(...)" - - docstring = inspect.getdoc(t) - if not docstring: - raise RuntimeError( - f"Cannot encode callable {t}: no source code and no docstring" + param_types, expected_return = type_args[0], type_args[-1] + typed_enc = _create_typed_synthesized_function(callable_type) + if param_types is not ... and isinstance(param_types, list | tuple): + expected_params = list(param_types) + else: + expected_params = None + + def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: + if callable(value) and not isinstance(value, dict): + return value + if isinstance(value, SynthesizedFunction): + encoded = value + elif isinstance(value, dict): + encoded = typed_enc.model_validate(value) + elif isinstance(value, str): + encoded = typed_enc.model_validate_json(value) + else: + raise ValueError( + f"Expected callable, SynthesizedFunction dict, or JSON string, " + f"got {type(value)}" ) - # Format as a stub function with docstring - stub_code = f'''def {name}{sig_str}: - """{docstring}""" - ... -''' - return self.enc(module_code=stub_code) - - def decode(self, encoded_value: SynthesizedFunction) -> Callable: - # Decode requires a concrete return type for synthesis - if self.expected_return is None: + if expected_return is None: raise TypeError( "Cannot decode/synthesize callable without a concrete type signature. " "Use Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType] " "with a concrete return type (not Any)." ) - filename = f"" - - module_code = encoded_value.module_code - - # Parse and validate AST before execution - module: ast.AST = evaluation.parse(module_code, filename) + ctx = info.context or {} + filename = f"" + module: ast.AST = evaluation.parse(encoded.module_code, filename) if not isinstance(module, ast.Module) or not module.body: raise ValueError( @@ -444,20 +508,12 @@ def decode(self, encoded_value: SynthesizedFunction) -> Callable: f"got {type(last_stmt).__name__}" ) - # Validate signature from AST before execution - _validate_signature_ast(last_stmt, self.expected_params) - - # Type-check with mypy; pass original module_code so mypy sees exact source - evaluation.type_check( - module, self.ctx, self.expected_params, self.expected_return - ) + _validate_signature_ast(last_stmt, expected_params) + evaluation.type_check(module, ctx, expected_params, expected_return) - # Compile and execute - # https://docs.python.org/3/library/functions.html#exec g: MutableMapping[str, Any] = {} - g.update(self.ctx or {}) - - bytecode: CodeType = evaluation.compile(module, filename) + g.update(ctx) + bytecode: types.CodeType = evaluation.compile(module, filename) evaluation.exec(bytecode, g) func_name = last_stmt.name @@ -472,152 +528,159 @@ def decode(self, encoded_value: SynthesizedFunction) -> Callable: f"decode() expected '{func_name}' to be callable, got {type(result)}" ) - # Validate signature from runtime callable after execution - _validate_signature_callable(result, self.expected_params, self.expected_return) - + _validate_signature_callable(result, expected_params, expected_return) return result - def serialize( - self, encoded_value: SynthesizedFunction - ) -> Sequence[OpenAIMessageContentListBlock]: - return [{"type": "text", "text": encoded_value.model_dump_json()}] - - def deserialize(self, serialized_value: str) -> SynthesizedFunction: - return SynthesizedFunction.model_validate_json(serialized_value) - - -@Encodable.define.register(object) -def _encodable_object[T, U]( - ty: type[T], ctx: Mapping[str, Any] | None -) -> Encodable[T, U]: - adapter = pydantic.TypeAdapter(ty) - ctx = {} if ctx is None else ctx - return typing.cast(Encodable[T, U], BaseEncodable(ty, ty, ctx, adapter)) - - -@Encodable.define.register(str) -def _encodable_str(ty: type[str], ctx: Mapping[str, Any] | None) -> Encodable[str, str]: - """Handler for str type that serializes without JSON encoding.""" - return StrEncodable(ty, ty, ctx or {}) - - -@Encodable.define.register(Term) -def _encodable_term[T: Term, U]( - ty: type[T], ctx: Mapping[str, Any] | None -) -> Encodable[T, U]: - raise TypeError("Terms cannot be encoded or decoded in general.") - - -@Encodable.define.register(Operation) -def _encodable_operation[T: Operation, U]( - ty: type[T], ctx: Mapping[str, Any] | None -) -> Encodable[T, U]: - raise TypeError("Operations cannot be encoded or decoded in general.") - - -@Encodable.define.register(pydantic.BaseModel) -def _encodable_pydantic_base_model[T: pydantic.BaseModel]( - ty: type[T], ctx: Mapping[str, Any] | None -) -> Encodable[T, T]: - return PydanticBaseModelEncodable(ty, ty, ctx or {}) - - -@Encodable.define.register(Image.Image) -def _encodable_image( - ty: type[Image.Image], ctx: Mapping[str, Any] | None -) -> Encodable[Image.Image, ChatCompletionImageUrlObject]: - return ImageEncodable(ty, ChatCompletionImageUrlObject, ctx or {}) + def _serialize(value: Callable) -> dict: + if not callable(value): + raise TypeError(f"Expected callable, got {type(value)}") + try: + source = inspect.getsource(value) + except (OSError, TypeError): + source = None -@Encodable.define.register(tuple) -def _encodable_tuple[T, U]( - ty: type[T], ctx: Mapping[str, Any] | None -) -> Encodable[T, U]: - args = typing.get_args(ty) - ctx = {} if ctx is None else ctx - - # handle namedtuples - origin = typing.get_origin(ty) - if origin is None: - return _encodable_object(ty, ctx) - # Handle empty tuple, or tuple with no args - if not args or args == ((),): - return _encodable_object(ty, ctx) + if source: + return typed_enc(module_code=textwrap.dedent(source)).model_dump() - # Create encoders for each element type - element_encoders = [Encodable.define(arg, ctx) for arg in args] + name = getattr(value, "__name__", None) + docstring = inspect.getdoc(value) + if name is None or docstring is None: + raise ValueError( + f"Cannot encode callable {value}: no source code and no __name__ or docstring" + ) - # Check if any element type is Image.Image - has_image = any(arg is Image.Image for arg in args) + try: + sig = inspect.signature(value) + sig_str = str(sig) + except (ValueError, TypeError): + sig_str = "(...)" - encoded_ty: type[typing.Any] = typing.cast( - type[typing.Any], - tuple[*(enc.enc for enc in element_encoders)], # type: ignore + stub_code = f'''def {name}{sig_str}: + """{docstring}""" + ... +''' + return typed_enc(module_code=stub_code).model_dump() + + return typing.Annotated[ + callable_type, + pydantic.PlainValidator(_validate), + pydantic.PlainSerializer(_serialize), + pydantic.WithJsonSchema( + _inline_refs(pydantic.TypeAdapter(typed_enc).json_schema()) + ), + ] + + +def _validate_tool( + value: ChatCompletionToolParam, info: pydantic.ValidationInfo +) -> Tool: + assert isinstance(info.context, Mapping), "Tool decoding requires context" + value = pydantic.TypeAdapter(ChatCompletionToolParam).validate_python(value) + try: + return info.context[value["function"]["name"]] + except KeyError as e: + raise NotImplementedError(f"Unknown tool: {value['function']['name']}") from e + + +def _serialize_tool(value: Tool) -> ChatCompletionToolParam: + fields: dict[str, Any] = { + name: TypeToPydanticType().evaluate(param.annotation) + for name, param in inspect.signature(value).parameters.items() + } + sig_model = pydantic.create_model( + "Params", + __config__={"extra": "forbid"}, + **fields, ) - - return typing.cast( - Encodable[T, U], - TupleEncodable(ty, encoded_ty, ctx, has_image, element_encoders), + response_format = litellm.utils.type_to_response_format_param(sig_model) + assert response_format is not None + assert value.__default__.__doc__ is not None + return pydantic.TypeAdapter(ChatCompletionToolParam).validate_python( + { + "type": "function", + "function": { + "name": value.__name__, + "description": textwrap.dedent(value.__default__.__doc__), + "parameters": response_format["json_schema"]["schema"], + "strict": True, + }, + } ) -@Encodable.define.register(list) -def _encodable_list[T, U]( - ty: type[list[T]], ctx: Mapping[str, Any] | None -) -> Encodable[T, U]: - args = typing.get_args(ty) - ctx = {} if ctx is None else ctx - - # Handle unparameterized list (list without type args) - if not args: - return _encodable_object(ty, ctx) - - # Get the element type (first type argument) - element_ty = args[0] - element_encoder = Encodable.define(element_ty, ctx) - - # Check if element type is Image.Image - has_image = element_ty is Image.Image - - # Build the encoded type (list of encoded element type) - runtime-created, use Any - encoded_ty: type[typing.Any] = typing.cast( - type[typing.Any], - list[element_encoder.enc], # type: ignore - ) - - return typing.cast( - Encodable[T, U], ListEncodable(ty, encoded_ty, ctx, has_image, element_encoder) +@TypeToPydanticType.register(Tool) +def _pydantic_type_tool(ty: type[Tool]): + schema = _inline_refs(pydantic.TypeAdapter(ChatCompletionToolParam).json_schema()) + schema = _ensure_strict_json_schema(schema, path=(), root={}) + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate_tool), + pydantic.PlainSerializer(_serialize_tool), + pydantic.WithJsonSchema(schema), + ] + + +def _validate_tool_call( + value: ChatCompletionMessageToolCall, + info: pydantic.ValidationInfo, +) -> DecodedToolCall: + if isinstance(value, dict): + value = OpenAIChatCompletionMessageToolCall.model_validate(value) + ctx = info.context or {} + assert value.function.name is not None + tool = ctx[value.function.name] + assert isinstance(tool, Tool) + sig = inspect.signature(tool) + decoded_args = {} + for name, raw_arg in json.loads(value.function.arguments).items(): + assert name in sig.parameters, ( + f"Unexpected argument {name} for tool {tool.__name__}" + ) + param = sig.parameters[name] + arg_enc: pydantic.TypeAdapter[Any] = pydantic.TypeAdapter( + Encodable[param.annotation] # type: ignore[name-defined] + ) + decoded_args[name] = arg_enc.validate_python(raw_arg, context=ctx) + return DecodedToolCall( + tool=tool, + bound_args=sig.bind(**decoded_args), + id=value.id, + name=value.function.name, ) -@Encodable.define.register(Callable) -def _encodable_callable( - ty: type[Callable], ctx: Mapping[str, Any] | None -) -> Encodable[Callable, SynthesizedFunction]: - ctx = ctx or {} - - type_args = typing.get_args(ty) - - # Bare Callable without type args - allow encoding but disable decode - # this occurs when decoding the result of Tools which return callable (need to Encodable.define(return_type) for return type) - if not type_args: - assert ty is types.FunctionType, f"Callable must have type signatures {ty}" - typed_enc = _create_typed_synthesized_function(Callable[..., typing.Any]) # type: ignore[arg-type] - return CallableEncodable(ty, typed_enc, ctx) - - if len(type_args) < 2: - raise TypeError( - f"Callable type signature incomplete: {ty}. " - "Expected Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType]." +def _serialize_tool_call( + value: DecodedToolCall, info: pydantic.SerializationInfo +) -> dict: + ctx = info.context or {} + encoded_args = {} + for k, v in value.bound_args.arguments.items(): + v_enc: pydantic.TypeAdapter[Any] = pydantic.TypeAdapter( + Encodable[nested_type(v).value] # type: ignore[misc] ) - - param_types, expected_return = type_args[0], type_args[-1] - - typed_enc = _create_typed_synthesized_function(ty) - - # Ellipsis means any params, skip param validation - expected_params: list[type] | None = None - if param_types is not ... and isinstance(param_types, list | tuple): - expected_params = list(param_types) - - return CallableEncodable(ty, typed_enc, ctx, expected_params, expected_return) + encoded_args[k] = v_enc.dump_python(v, mode="json", context=ctx) + return OpenAIChatCompletionMessageToolCall.model_validate( + { + "type": "function", + "id": value.id, + "function": { + "name": value.tool.__name__, + "arguments": json.dumps(encoded_args), + }, + } + ).model_dump(mode="json") + + +@TypeToPydanticType.register(DecodedToolCall) +def _pydantic_type_tool_call(ty: type[DecodedToolCall]): + # Use OpenAI's ChatCompletionMessageToolCall (has actual fields: id, function, + # type) rather than litellm's (empty dict with extra="allow"). + schema = _inline_refs(OpenAIChatCompletionMessageToolCall.model_json_schema()) + schema = _ensure_strict_json_schema(schema, path=(), root={}) + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate_tool_call), + pydantic.PlainSerializer(_serialize_tool_call), + pydantic.WithJsonSchema(schema), + ] diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index 07348cc98..b4c4ecf67 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -392,7 +392,7 @@ def signature_to_ast(name: str, sig: inspect.Signature) -> ast.FunctionDef: except TypeError: returns = type_to_ast(typing.Any) - node = ast.FunctionDef( # type: ignore + node = ast.FunctionDef( name=name, args=ast.arguments( posonlyargs=[], @@ -415,6 +415,7 @@ def signature_to_ast(name: str, sig: inspect.Signature) -> ast.FunctionDef: ], decorator_list=[], returns=returns, + type_params=[], ) return ast.fix_missing_locations(node) diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 2dfab38f7..f56d6fad7 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -1,15 +1,15 @@ import abc -import collections import functools import inspect +import re +import string import types import typing +from collections import ChainMap, OrderedDict from collections.abc import Callable, Mapping, MutableMapping -from dataclasses import dataclass from typing import Annotated, Any -from effectful.ops.semantics import handler -from effectful.ops.types import INSTANCE_OP_PREFIX, Annotation, Operation +from effectful.ops.types import Annotation, Operation class _IsRecursiveAnnotation(Annotation): @@ -90,13 +90,14 @@ def vacation() -> str: """ - def __init__( - self, signature: inspect.Signature, name: str, default: Callable[P, T] - ): + def __init__(self, default: Callable[P, T], name: str | None = None): if not default.__doc__: raise ValueError("Tools must have docstrings.") - signature = IsRecursive.infer_annotations(signature) - super().__init__(signature, name, default) + super().__init__(default, name=name) + + @property + def __signature__(self): + return IsRecursive.infer_annotations(super().__signature__) @classmethod def define(cls, *args, **kwargs) -> "Tool[P, T]": @@ -108,25 +109,6 @@ def define(cls, *args, **kwargs) -> "Tool[P, T]": return typing.cast("Tool[P, T]", super().define(*args, **kwargs)) -@dataclass -class _BoundInstance[T]: - instance: T - - -def _make_context_tool[T](name: str, value: T) -> Tool[[], T]: - """Create a synthetic read-only Tool for a lexical variable.""" - from effectful.internals.unification import nested_type - - def reader(): - return value - - reader.__name__ = name - reader.__doc__ = f"Read the value of lexical variable `{name}`" - reader.__annotations__ = {"return": nested_type(value).value} - - return Tool.define(reader) - - class Template[**P, T](Tool[P, T]): """A :class:`Template` is a function that is implemented by a large language model. @@ -186,7 +168,45 @@ class Template[**P, T](Tool[P, T]): """ - __context__: collections.ChainMap[str, Any] + __context__: ChainMap[str, Any] + __system_prompt__: str + + @classmethod + def _validate_prompt( + cls, + template: "Template", + context: ChainMap[str, Any], + ) -> None: + """Validate that all format string variables in the docstring + refer to names resolvable at call time. + + Each variable must be either a parameter in the signature + or a name captured in the lexical context. + + :raises TypeError: If any format string variable cannot be resolved. + """ + doc = template.__prompt_template__ + formatter = string.Formatter() + param_names = set(template.__signature__.parameters.keys()) + context_keys = set(context.keys()) + allowed_names = param_names | context_keys + + unresolved: list[str] = [] + for _, field_name, _, _ in formatter.parse(doc): + if field_name is None: + continue + # Extract root identifier from compound names like + match = re.match(r"^(\w+)", field_name) + root = match.group(1) if match else field_name + if root not in allowed_names: + unresolved.append(field_name) + + if unresolved: + raise TypeError( + f"Template '{template.__name__}' docstring references undefined " + f"variables {list(sorted(unresolved))} that are not in the signature " + f"{{{template.__signature__}}} or lexical scope." + ) @property def __prompt_template__(self) -> str: @@ -196,40 +216,17 @@ def __prompt_template__(self) -> str: @property def tools(self) -> Mapping[str, Tool]: """Operations and Templates available as tools. Auto-capture from lexical context.""" - result = {} - is_recursive = _is_recursive_signature(self.__signature__) + from effectful.handlers.llm.completions import _collect_tools - for name, obj in self.__context__.items(): - if obj is self and not is_recursive: - continue + result = _collect_tools(self.__context__) - # Collect tools in context - elif isinstance(obj, Tool): - result[name] = obj - - elif isinstance(obj, staticmethod) and isinstance(obj.__func__, Tool): - result[name] = obj.__func__ - - # Collect tools as methods on any bound instances - elif isinstance(obj, _BoundInstance): - for instance_name in obj.instance.__dir__(): - if instance_name.startswith(INSTANCE_OP_PREFIX): - continue - instance_obj = getattr(obj.instance, instance_name) - if isinstance(instance_obj, Tool): - result[instance_name] = instance_obj - - # Make tools for lexical variables - elif not ( - name.startswith("__") - or isinstance(obj, Operation) - or inspect.isclass(obj) - or inspect.isbuiltin(obj) - or inspect.ismodule(obj) - or inspect.isroutine(obj) - or inspect.isabstract(obj) - ): - result[name] = _make_context_tool(name, obj) + # We remove the template itself from the tool map unless it is explicitly + # marked as recursive (see test_template_method, test_template_method_nested_class). + if not _is_recursive_signature(self.__signature__): + result = dict(result) # copy to allow mutation + for name, tool in tuple(result.items()): + if tool is self: + del result[name] return result @@ -241,8 +238,18 @@ def __get__[S](self, instance: S | None, owner: type[S] | None = None): result = super().__get__(instance, owner) self_param_name = list(self.__signature__.parameters.keys())[0] - self_context = {self_param_name: _BoundInstance(instance)} - result.__context__ = self.__context__.new_child(self_context) + result.__context__ = self.__context__.new_child({self_param_name: instance}) + if isinstance(instance, Agent): + assert isinstance(result, Template) and not hasattr(result, "__history__") + result.__history__ = instance.__history__ # type: ignore[attr-defined] + result.__system_prompt__ = "\n\n".join( + part + for part in ( + getattr(result, "__system_prompt__", ""), + instance.__system_prompt__, + ) + if part + ) return result @classmethod @@ -263,35 +270,56 @@ def define[**Q, V]( frame = frame.f_back assert frame is not None - # Check if we're in a class definition by looking for __qualname__ + # Skip class body frames: in Python, class bodies are not lexical + # scopes for methods, so their locals should not be captured. qualname = frame.f_locals.get("__qualname__") - n_frames = 1 if qualname is not None: - name_components = qualname.split(".") - for name in reversed(name_components): + for name in reversed(qualname.split(".")): if name == "": break - n_frames += 1 - - contexts = [] - for offset in range(n_frames): - assert frame is not None - locals_proxy: types.MappingProxyType[str, Any] = types.MappingProxyType( - frame.f_locals - ) - globals_proxy: types.MappingProxyType[str, Any] = types.MappingProxyType( - frame.f_globals - ) - contexts.append(locals_proxy) - frame = frame.f_back + assert frame is not None + frame = frame.f_back + # Use the qualname of the decorated function to identify which + # frames are *lexical* enclosers (as opposed to dynamic callers). + # A segment preceding "" in the qualname is an enclosing + # function; everything else (class names, the function itself) is not. + assert frame is not None + _fn = default + if isinstance(_fn, staticmethod | classmethod): + _fn = _fn.__func__ + parts = _fn.__qualname__.split(".") + enclosing_fns = [ + parts[i] for i in range(len(parts) - 1) if parts[i + 1] == "" + ] + enclosing_fns.reverse() # innermost first for frame walking + + globals_proxy: types.MappingProxyType[str, Any] = types.MappingProxyType( + frame.f_globals + ) + contexts: list[types.MappingProxyType[str, Any]] = [] + for fn_name in enclosing_fns: + while frame is not None and frame.f_locals is not frame.f_globals: + if frame.f_code.co_name == fn_name: + contexts.append(types.MappingProxyType(frame.f_locals)) + frame = frame.f_back + break + frame = frame.f_back contexts.append(globals_proxy) - context: collections.ChainMap[str, Any] = collections.ChainMap( + context: ChainMap[str, Any] = ChainMap( *typing.cast(list[MutableMapping[str, Any]], contexts) ) - op = super().define(default, *args, **kwargs) op.__context__ = context # type: ignore[attr-defined] + mod = inspect.getmodule(_fn) + op.__system_prompt__ = inspect.getdoc(mod) if mod is not None else "" # type: ignore[attr-defined] + # Keep validation on original define-time callables, but skip the bound wrapper path. + # to avoid dropping `self` from the signature and falsely rejecting valid prompt fields like `{self.name}`. + is_bound_wrapper = ( + isinstance(default, types.MethodType) and default.__self__ is not None + ) + if not isinstance(op, staticmethod | classmethod) and not is_bound_wrapper: + cls._validate_prompt(typing.cast(Template, op), context) return typing.cast(Template[Q, V], op) @@ -333,25 +361,18 @@ def send(self, user_input: str) -> str: """ - __history__: collections.OrderedDict[str, Any] + __history__: OrderedDict[str, Mapping[str, Any]] + __system_prompt__: str def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - prop = functools.cached_property(lambda _: collections.OrderedDict()) - prop.__set_name__(cls, "__history__") - cls.__history__ = prop - - for name in list(cls.__dict__): - attr = cls.__dict__[name] - if not isinstance(attr, Template): - continue - _template = attr - - @functools.wraps(_template) - def wrapper(self, *args, _t=_template, **kwargs): - from effectful.handlers.llm.completions import get_message_sequence - - with handler({get_message_sequence: lambda: self.__history__}): - return _t(self, *args, **kwargs) - - setattr(cls, name, wrapper) + if not hasattr(cls, "__history__"): + prop = functools.cached_property(lambda _: OrderedDict()) + prop.__set_name__(cls, "__history__") + cls.__history__ = prop + if not hasattr(cls, "__system_prompt__"): + sp = functools.cached_property( + lambda self: inspect.getdoc(type(self)) or "" + ) + sp.__set_name__(cls, "__system_prompt__") + cls.__system_prompt__ = sp diff --git a/effectful/handlers/numpyro.py b/effectful/handlers/numpyro.py index 74010de41..f0369d379 100644 --- a/effectful/handlers/numpyro.py +++ b/effectful/handlers/numpyro.py @@ -1,4 +1,5 @@ try: + import numpyro import numpyro.distributions as dist except ImportError: raise ImportError("Numpyro is required to use effectful.handlers.numpyro") @@ -332,6 +333,13 @@ def variance(self) -> jax.Array: except NotImplementedError: raise RuntimeError(f"variance is not implemented for {type(self).__name__}") + @property + @defop + def support(self) -> numpyro.distributions.constraints.Constraint: + if not self._is_eager: + raise NotHandled + return self._pos_base_dist.support + @defop def enumerate_support(self, expand=True) -> jax.Array: if not self._is_eager: diff --git a/effectful/internals/unification.py b/effectful/internals/unification.py index 71d6583f2..77f5c9613 100644 --- a/effectful/internals/unification.py +++ b/effectful/internals/unification.py @@ -104,6 +104,86 @@ class Box[T]: value: T +class TypeEvaluator(abc.ABC): + """ + Abstract base class for evaluating type expressions. + + This class defines the interface for evaluating type expressions, which may + involve resolving type variables, computing canonical forms of types, or + performing other transformations. Subclasses should implement the evaluate + method to provide specific evaluation logic. + + The TypeEvaluator can be used in contexts where type expressions need to be + processed or normalized before unification or other type operations. + """ + + @functools.singledispatchmethod + def evaluate(self, typ) -> TypeExpressions: + """ + Normalize generic types + """ + raise TypeError(f"Cannot traverse type {typ}.") + + @evaluate.register + def _(self, typ: TypeConstant | TypeVariable): + return typ + + @evaluate.register + def _(self, typ: GenericAlias): + origin, args = typing.get_origin(typ), typing.get_args(typ) + return origin[self.evaluate(args)] # type: ignore[index] + + @evaluate.register + def _(self, typ: UnionType): + ctyp = self.evaluate(typing.get_args(typ)[0]) + for arg in typing.get_args(typ)[1:]: + ctyp = ctyp | self.evaluate(arg) # type: ignore + return ctyp + + @evaluate.register + def _(self, typ: typing._AnnotatedAlias): # type: ignore + return typing.Annotated[ + self.evaluate(typing.get_args(typ)[0]), + typ.__metadata__, + ] + + @evaluate.register + def _(self, typ: typing._LiteralGenericAlias): # type: ignore + return typ + + @evaluate.register + def _(self, typ: typing.ParamSpecArgs | typing.ParamSpecKwargs): + return typ + + @evaluate.register + def _(self, typ: typing._SpecialGenericAlias): # type: ignore + assert not typing.get_args(typ), "Should not have type arguments" + return typ + + @evaluate.register + def _(self, typ: typing._ConcatenateGenericAlias): # type: ignore + return typing.Concatenate[self.evaluate(typing.get_args(typ))] + + @evaluate.register + def _(self, typ: list | tuple): + return type(typ)(self.evaluate(item) for item in typ) + + @evaluate.register + def _(self, typ: typing.NewType): + return typing.NewType(typ.__name__, self.evaluate(typ.__supertype__)) # type: ignore[attr-defined,unused-ignore] + + @evaluate.register + def _(self, typ: typing.TypeAliasType): + return self.evaluate(typ.__value__) + + @evaluate.register + def _(self, typ: typing.ForwardRef): + if typ.__forward_value__ is not None: + return self.evaluate(typ.__forward_value__) + else: + return typ + + @typing.overload def unify( typ: inspect.Signature, diff --git a/effectful/ops/semantics.py b/effectful/ops/semantics.py index f7475dc71..f7678fd24 100644 --- a/effectful/ops/semantics.py +++ b/effectful/ops/semantics.py @@ -8,7 +8,7 @@ from collections.abc import Callable from typing import Any -from effectful.ops.syntax import _CustomSingleDispatchCallable, defop +from effectful.ops.syntax import _CustomSingleDispatchCallable, defdata, defop from effectful.ops.types import ( Expr, Interpretation, @@ -364,6 +364,7 @@ def _update_fvs(op, *args, **kwargs): assert isinstance(bound_var, Operation) if bound_var in _fvs: _fvs.remove(bound_var) + return defdata(op, *args, **kwargs) with interpreter({apply: _update_fvs}): evaluate(term) diff --git a/effectful/ops/types.py b/effectful/ops/types.py index 43772b4e1..40c1f4af5 100644 --- a/effectful/ops/types.py +++ b/effectful/ops/types.py @@ -74,19 +74,30 @@ class Operation[**Q, V]: """ - __signature__: inspect.Signature __name__: str __default__: Callable[Q, V] __apply__: typing.ClassVar["Operation"] - def __init__( - self, signature: inspect.Signature, name: str, default: Callable[Q, V] - ): + def __init__(self, default: Callable[Q, V], name: str | None = None): functools.update_wrapper(self, default) - - self.__signature__ = signature - self.__name__ = name self.__default__ = default + self.__name__ = name or default.__name__ + + @property + def __signature__(self): + # Resolve forward references (e.g. -> "MyClass") using the + # default function's __globals__. This handles module-level + # forward refs; local forward refs will raise NameError. + # Python 3.14's annotationlib.get_annotations(format=FORWARDREF) + # could resolve local refs too via PEP 649 __annotate__ functions. + annots = typing.get_type_hints(self.__default__, include_extras=True) + sig = inspect.signature(self.__default__) + updated_params = [ + p.replace(annotation=annots[p.name]) if p.name in annots else p + for p in sig.parameters.values() + ] + updated_ret = annots.get("return", sig.return_annotation) + return sig.replace(parameters=updated_params, return_annotation=updated_ret) def __eq__(self, other): if not isinstance(other, Operation): @@ -267,8 +278,7 @@ def func(*args, **kwargs): op = cls.define(func, name=name) else: - name = name or t.__name__ - op = cls(inspect.signature(t), name, t) # type: ignore[arg-type] + op = cls(t, name=name) # type: ignore[arg-type] return op # type: ignore[return-value] @@ -441,7 +451,9 @@ def __str__(self): def __set_name__[T](self, owner: type[T], name: str) -> None: if not issubclass(owner, Term): assert not hasattr(self, "_name_on_instance"), "should only be called once" - self._name_on_instance: str = f"{INSTANCE_OP_PREFIX}_{name}" + self._name_on_instance: str = ( + f"{INSTANCE_OP_PREFIX}_{owner.__name__}_{name}" + ) def __get__[T](self, instance: T | None, owner: type[T] | None = None): if hasattr(instance, "__dict__") and hasattr(self, "_name_on_instance"): From 136b3a8dde68b27ffb5421d3521afa9989504624 Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 12:45:37 -0400 Subject: [PATCH 005/155] no agent test --- tests/test_handlers_llm_agent.py | 315 ------------------------------- 1 file changed, 315 deletions(-) delete mode 100644 tests/test_handlers_llm_agent.py diff --git a/tests/test_handlers_llm_agent.py b/tests/test_handlers_llm_agent.py deleted file mode 100644 index 38c8aacd1..000000000 --- a/tests/test_handlers_llm_agent.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Tests for Agent mixin message sequence semantics.""" - -import collections -import dataclasses - -from litellm import ModelResponse - -from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import ( - LiteLLMProvider, - RetryLLMHandler, - completion, -) -from effectful.ops.semantics import handler -from effectful.ops.syntax import ObjectInterpretation, implements -from effectful.ops.types import NotHandled - -# --------------------------------------------------------------------------- -# Helpers (same pattern as test_handlers_llm_provider.py) -# --------------------------------------------------------------------------- - - -def make_text_response(content: str) -> ModelResponse: - return ModelResponse( - id="test", - choices=[ - { - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop", - } - ], - model="test-model", - ) - - -def make_tool_call_response( - tool_name: str, tool_args: str, tool_call_id: str = "call_1" -) -> ModelResponse: - return ModelResponse( - id="test", - choices=[ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": tool_call_id, - "type": "function", - "function": {"name": tool_name, "arguments": tool_args}, - } - ], - }, - "finish_reason": "tool_calls", - } - ], - model="test-model", - ) - - -class MockCompletionHandler(ObjectInterpretation): - """Returns pre-configured responses and captures messages sent to the LLM.""" - - def __init__(self, responses: list[ModelResponse]): - self.responses = responses - self.call_count = 0 - self.received_messages: list[list] = [] - - @implements(completion) - def _completion(self, model, messages=None, **kwargs): - self.received_messages.append(list(messages) if messages else []) - response = self.responses[min(self.call_count, len(self.responses) - 1)] - self.call_count += 1 - return response - - -# --------------------------------------------------------------------------- -# Agent subclass used by most tests -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass -class ChatBot(Agent): - """Simple chat agent for testing history accumulation.""" - - bot_name: str = dataclasses.field(default="ChatBot") - - @Template.define - def send(self, user_input: str) -> str: - """A friendly bot named {self.bot_name}. User writes: {user_input}""" - raise NotHandled - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestAgentHistoryAccumulation: - """History accumulates across sequential calls on the same instance.""" - - def test_second_call_sees_prior_messages(self): - mock = MockCompletionHandler( - [make_text_response("hi"), make_text_response("good")] - ) - bot = ChatBot() - - with handler(LiteLLMProvider()), handler(mock): - bot.send("hello") - bot.send("how are you") - - # First call: system + user → 2 messages - assert len(mock.received_messages[0]) == 2 - - # Second call: previous system + user + assistant, PLUS new system + user → 5 - assert len(mock.received_messages[1]) > len(mock.received_messages[0]) - - # Verify roles in second call - roles = [m["role"] for m in mock.received_messages[1]] - assert roles.count("assistant") >= 1 - assert roles.count("user") >= 2 - assert roles.count("system") >= 2 - - def test_history_contains_all_messages_after_two_calls(self): - mock = MockCompletionHandler( - [make_text_response("r1"), make_text_response("r2")] - ) - bot = ChatBot() - - with handler(LiteLLMProvider()), handler(mock): - bot.send("a") - bot.send("b") - - # After two complete calls the history should have: - # call 1: system, user, assistant (3) - # call 2: system, user, assistant (3) - assert len(bot.__history__) == 6 - - def test_message_ids_are_unique(self): - mock = MockCompletionHandler( - [make_text_response("r1"), make_text_response("r2")] - ) - bot = ChatBot() - - with handler(LiteLLMProvider()), handler(mock): - bot.send("a") - bot.send("b") - - ids = list(bot.__history__.keys()) - assert len(ids) == len(set(ids)), "message IDs must be unique" - - -class TestAgentIsolation: - """Each agent instance has independent history; non-agent templates are unaffected.""" - - def test_two_agents_have_independent_histories(self): - mock = MockCompletionHandler( - [ - make_text_response("from bot1"), - make_text_response("from bot2"), - ] - ) - bot1 = ChatBot() - bot2 = ChatBot() - - with handler(LiteLLMProvider()), handler(mock): - bot1.send("msg for bot1") - bot2.send("msg for bot2") - - # bot2's call should NOT contain bot1's messages - assert len(mock.received_messages[1]) == 2 # system + user only - - # Each bot has its own history - assert len(bot1.__history__) == 3 # system, user, assistant - assert len(bot2.__history__) == 3 - - # Histories share no message IDs - assert set(bot1.__history__.keys()).isdisjoint(set(bot2.__history__.keys())) - - def test_non_agent_template_gets_fresh_sequence(self): - @Template.define - def standalone(topic: str) -> str: - """Write about {topic}.""" - raise NotHandled - - mock = MockCompletionHandler( - [ - make_text_response("agent reply"), - make_text_response("standalone reply"), - make_text_response("agent reply 2"), - ] - ) - bot = ChatBot() - - with handler(LiteLLMProvider()), handler(mock): - bot.send("hello") - standalone("fish") - bot.send("bye") - - # standalone (call index 1) should see only system + user (fresh sequence) - assert len(mock.received_messages[1]) == 2 - - # bot's third call (call index 2) should see its accumulated history - # but NOT the standalone messages - assert len(mock.received_messages[2]) == 5 # 3 from first call + 2 new - - -class TestAgentCachedProperty: - """__history__ is lazily created per instance without requiring __init__.""" - - def test_no_init_required(self): - class MinimalAgent(Agent): - @Template.define - def greet(self, name: str) -> str: - """Hello {name}.""" - raise NotHandled - - agent = MinimalAgent() - # Should be an OrderedDict, created on first access - assert isinstance(agent.__history__, collections.OrderedDict) - assert len(agent.__history__) == 0 - - def test_subclass_with_own_init(self): - class CustomAgent(Agent): - def __init__(self, name: str): - self.name = name - - @Template.define - def greet(self) -> str: - """Say hello.""" - raise NotHandled - - agent = CustomAgent("Alice") - assert agent.name == "Alice" - assert isinstance(agent.__history__, collections.OrderedDict) - - def test_history_is_per_instance(self): - a = ChatBot() - b = ChatBot() - a.__history__["fake"] = {"id": "fake", "role": "user", "content": "x"} - assert "fake" not in b.__history__ - - -class TestAgentWithToolCalls: - """Agent methods that trigger tool calls maintain correct history.""" - - def test_tool_call_results_appear_in_history(self): - @Tool.define - def add(a: int, b: int) -> int: - """Add two numbers.""" - return a + b - - class MathAgent(Agent): - @Template.define - def compute(self, question: str) -> str: - """Answer: {question}""" - raise NotHandled - - mock = MockCompletionHandler( - [ - make_tool_call_response("add", '{"a": 2, "b": 3}'), - make_text_response("The answer is 5"), - ] - ) - agent = MathAgent() - - with handler(LiteLLMProvider()), handler(mock): - result = agent.compute("what is 2+3?") - - assert result == "The answer is 5" - - # History should contain: system, user, assistant (tool_call), - # tool (result), assistant (final) - roles = [m["role"] for m in agent.__history__.values()] - assert "tool" in roles - assert roles.count("assistant") == 2 - - -class TestAgentWithRetryHandler: - """RetryLLMHandler composes correctly with Agent history.""" - - def test_failed_retries_dont_pollute_history(self): - mock = MockCompletionHandler( - [ - # First attempt: invalid result for int - make_text_response('{"value": "not_an_int"}'), - # Retry: valid - make_text_response('{"value": 42}'), - ] - ) - - class NumberAgent(Agent): - @Template.define - def pick_number(self) -> int: - """Pick a number.""" - raise NotHandled - - agent = NumberAgent() - - with ( - handler(LiteLLMProvider()), - handler(RetryLLMHandler(num_retries=3)), - handler(mock), - ): - result = agent.pick_number() - - assert result == 42 - - # The malformed assistant message and error feedback from the retry - # should NOT appear in the agent's history. Only the final successful - # assistant message should be there. - roles = [m["role"] for m in agent.__history__.values()] - assert roles == ["system", "user", "assistant"] From 52dc30eb83e705c09a31f05aa56999cae74a982b Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 14:25:54 -0400 Subject: [PATCH 006/155] updates --- docs/source/llm_examples/async_concurrency.py | 9 +--- docs/source/llm_examples/batch_translate.py | 19 ++++---- docs/source/llm_examples/chat_memory.py | 9 +--- docs/source/llm_examples/chat_search.py | 21 +++++---- docs/source/llm_examples/flight_booking.py | 22 +++++---- docs/source/llm_examples/guardrails.py | 38 ++++++++------- .../llm_examples/hanoi_solver_iterative.py | 16 +++---- .../llm_examples/hanoi_solver_recursive.py | 16 +++---- docs/source/llm_examples/hitl.py | 22 +++++---- docs/source/llm_examples/majority_vote.py | 9 +--- docs/source/llm_examples/map_reduce.py | 46 ++++++++++--------- docs/source/llm_examples/multi_agent.py | 22 +++++---- docs/source/llm_examples/rag.py | 23 ++++++---- docs/source/llm_examples/research_agent.py | 9 +--- docs/source/llm_examples/supervisor.py | 21 +++++---- docs/source/llm_examples/tao_agent.py | 15 +++--- docs/source/llm_examples/text2sql.py | 22 +++++---- docs/source/llm_examples/thinking.py | 22 +++++---- 18 files changed, 181 insertions(+), 180 deletions(-) diff --git a/docs/source/llm_examples/async_concurrency.py b/docs/source/llm_examples/async_concurrency.py index 32ec20c77..b47170610 100644 --- a/docs/source/llm_examples/async_concurrency.py +++ b/docs/source/llm_examples/async_concurrency.py @@ -52,17 +52,10 @@ async def main(provider: LiteLLMProvider): parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) asyncio.run(main(provider)) diff --git a/docs/source/llm_examples/batch_translate.py b/docs/source/llm_examples/batch_translate.py index bcdb343d3..66b4999f8 100644 --- a/docs/source/llm_examples/batch_translate.py +++ b/docs/source/llm_examples/batch_translate.py @@ -7,8 +7,10 @@ import argparse import os +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.handlers.llm.evaluation import RestrictedEvalProvider from effectful.ops.semantics import handler from effectful.ops.types import NotHandled @@ -38,7 +40,7 @@ def translate(target_language: str, instructions: str = "") -> Template[[str], s parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -55,16 +57,13 @@ def translate(target_language: str, instructions: str = "") -> Template[[str], s ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RestrictedEvalProvider()): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + handler(RestrictedEvalProvider()), + ): translator = translate( target_language="french", instructions="Use formal language." ) diff --git a/docs/source/llm_examples/chat_memory.py b/docs/source/llm_examples/chat_memory.py index 42c8b46ac..b926cdff5 100644 --- a/docs/source/llm_examples/chat_memory.py +++ b/docs/source/llm_examples/chat_memory.py @@ -109,18 +109,11 @@ def chat(self, user_input: str): parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - agent = ChatAgent() provider = LiteLLMProvider(model=args.model) diff --git a/docs/source/llm_examples/chat_search.py b/docs/source/llm_examples/chat_search.py index 8dcdd1691..a2d7cecd5 100644 --- a/docs/source/llm_examples/chat_search.py +++ b/docs/source/llm_examples/chat_search.py @@ -4,6 +4,7 @@ import urllib.parse import requests +from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler @@ -77,7 +78,7 @@ def send(self, user_input: str) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -91,19 +92,21 @@ def send(self, user_input: str) -> str: action="store_true", help="Run in interactive mode, allowing multiple back-and-forth messages", ) + parser.add_argument( + "--num-retries", + type=int, + default=4, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - chatbot = ChatBot(bot_name=args.name) provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): if args.interactive: while True: print(chatbot.send(input("You: "))) diff --git a/docs/source/llm_examples/flight_booking.py b/docs/source/llm_examples/flight_booking.py index 35cdec35d..d4de2e97a 100644 --- a/docs/source/llm_examples/flight_booking.py +++ b/docs/source/llm_examples/flight_booking.py @@ -15,6 +15,8 @@ import os from typing import Literal +from tenacity import stop_after_attempt + from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -227,7 +229,7 @@ def book_flight( parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -235,18 +237,20 @@ def book_flight( action="store_true", help="Run in interactive mode with user prompts", ) + parser.add_argument( + "--num-retries", + type=int, + default=4, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=5)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): book_flight( origin=Airport.SFO, destination=Airport.ANC, diff --git a/docs/source/llm_examples/guardrails.py b/docs/source/llm_examples/guardrails.py index 304818ffb..6ac800572 100644 --- a/docs/source/llm_examples/guardrails.py +++ b/docs/source/llm_examples/guardrails.py @@ -8,6 +8,8 @@ import argparse import os +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -26,14 +28,6 @@ def travel_query(user_query: str) -> str: raise NotHandled -@Template.define -def is_safe_query(user_query: str) -> bool: - """ - Determine whether the user's query is purely related to travel advice: {user_query} - """ - raise NotHandled - - # --------------------------------------------------------------------------- # Guarded agent # --------------------------------------------------------------------------- @@ -41,6 +35,14 @@ def is_safe_query(user_query: str) -> bool: def answer_travel_query(user_query: str) -> str: """Only answer travel-related queries; reject everything else.""" + + @Template.define + def is_safe_query(user_query: str) -> bool: + """ + Determine whether the user's query is purely related to travel advice: {user_query} + """ + raise NotHandled + if is_safe_query(user_query): return travel_query(user_query) else: @@ -56,19 +58,21 @@ def answer_travel_query(user_query: str) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) + parser.add_argument( + "--num-retries", + type=int, + default=4, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=5)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): print(answer_travel_query("What are great places to check out in NYC?")) print(answer_travel_query("Should I buy apple stocks?")) diff --git a/docs/source/llm_examples/hanoi_solver_iterative.py b/docs/source/llm_examples/hanoi_solver_iterative.py index 8733c64ac..0a5ecdbab 100644 --- a/docs/source/llm_examples/hanoi_solver_iterative.py +++ b/docs/source/llm_examples/hanoi_solver_iterative.py @@ -14,6 +14,8 @@ import os from dataclasses import dataclass, field +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -173,7 +175,7 @@ def solve_hanoi(state: GameState, max_steps: int = 30): parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -196,14 +198,10 @@ def solve_hanoi(state: GameState, max_steps: int = 30): ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=args.num_retries)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): solve_hanoi(GameState(size=args.game_size), max_steps=args.max_steps) diff --git a/docs/source/llm_examples/hanoi_solver_recursive.py b/docs/source/llm_examples/hanoi_solver_recursive.py index b5da9107a..b84387729 100644 --- a/docs/source/llm_examples/hanoi_solver_recursive.py +++ b/docs/source/llm_examples/hanoi_solver_recursive.py @@ -29,6 +29,8 @@ import typing from dataclasses import dataclass, field +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.handlers.llm.template import IsRecursive @@ -161,7 +163,7 @@ def validate_solution(size: int, steps: list[Step]) -> bool: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -178,16 +180,12 @@ def validate_solution(size: int, steps: list[Step]) -> bool: ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=args.num_retries)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): n = args.game_size print(f"Solving Tower of Hanoi with {n} disks...") steps = solve(n_disks=n, source=0, target=n - 1, auxiliary=1) diff --git a/docs/source/llm_examples/hitl.py b/docs/source/llm_examples/hitl.py index 5b2ebe17c..540fc27c6 100644 --- a/docs/source/llm_examples/hitl.py +++ b/docs/source/llm_examples/hitl.py @@ -13,6 +13,8 @@ import enum import os +from tenacity import stop_after_attempt + from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -130,7 +132,7 @@ def run_with_approval( parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -144,15 +146,14 @@ def run_with_approval( default=5, help="Maximum number of action steps", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) task = ( @@ -161,7 +162,10 @@ def run_with_approval( "restaurant suggestions, and schedule a meeting to finalize plans." ) - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): print(f"Task: {task}\n") log = run_with_approval( task, diff --git a/docs/source/llm_examples/majority_vote.py b/docs/source/llm_examples/majority_vote.py index 1ad8c6296..25696ddcb 100644 --- a/docs/source/llm_examples/majority_vote.py +++ b/docs/source/llm_examples/majority_vote.py @@ -59,7 +59,7 @@ def majority_vote[Q]( parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -73,13 +73,6 @@ def majority_vote[Q]( ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) with handler(provider): answer, count = majority_vote(yes_or_no, args.question, voters=args.num_voters) diff --git a/docs/source/llm_examples/map_reduce.py b/docs/source/llm_examples/map_reduce.py index f7e00ac1d..70a79c595 100644 --- a/docs/source/llm_examples/map_reduce.py +++ b/docs/source/llm_examples/map_reduce.py @@ -9,10 +9,13 @@ import argparse import asyncio +import collections.abc import dataclasses import functools import os +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -53,13 +56,16 @@ def evaluate_resume(resume: str, job_description: str) -> Evaluation: @Template.define -def summarize_evaluations(job_description: str, evaluations_text: str) -> str: +def summarize_evaluations( + job_description: str, + evaluations: collections.abc.Sequence[Evaluation], +) -> str: """You are a hiring manager summarizing candidate evaluations. Job description: {job_description} Individual evaluations: - {evaluations_text} + {evaluations} Provide a brief summary: rank the candidates from best to worst, highlight the top candidate, and note any concerns. @@ -103,7 +109,11 @@ async def map_reduce_evaluate( # Map: evaluate each resume concurrently evaluate = functools.partial( asyncio.to_thread, - handler(provider)(handler(RetryLLMHandler(num_retries=3))(evaluate_resume)), + handler(provider)( + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries)))( + evaluate_resume + ) + ), ) evaluations: list[Evaluation] = list( await asyncio.gather(*(evaluate(resume, job_description) for resume in resumes)) @@ -116,16 +126,11 @@ async def map_reduce_evaluate( print(f" - {ev.weaknesses}") # Reduce: summarize all evaluations - evaluations_text = "\n\n".join( - f"Candidate: {ev.name}\n" - f"Score: {ev.score}/10\n" - f"Qualified: {ev.qualified}\n" - f"Strengths: {ev.strengths}\n" - f"Weaknesses: {ev.weaknesses}" - for ev in evaluations - ) - with handler(provider), handler(RetryLLMHandler(num_retries=3)): - return summarize_evaluations(job_description, evaluations_text) + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): + return summarize_evaluations(job_description, evaluations) # --------------------------------------------------------------------------- @@ -137,18 +142,17 @@ async def map_reduce_evaluate( parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) print(f"Evaluating {len(RESUMES)} resumes for: {JOB_DESCRIPTION}\n") diff --git a/docs/source/llm_examples/multi_agent.py b/docs/source/llm_examples/multi_agent.py index 448cf7a4f..0389c6c87 100644 --- a/docs/source/llm_examples/multi_agent.py +++ b/docs/source/llm_examples/multi_agent.py @@ -12,6 +12,8 @@ import enum import os +from tenacity import stop_after_attempt + from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -133,7 +135,7 @@ def play_taboo( parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -142,15 +144,14 @@ def play_taboo( default=5, help="Maximum rounds per game", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - games = [ ("piano", ["music", "keys", "instrument", "play"]), ("volcano", ["lava", "eruption", "mountain", "hot"]), @@ -158,7 +159,10 @@ def play_taboo( provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): for secret, taboo in games: print(f"\nGame: '{secret}' (taboo: {taboo})") play_taboo(secret, taboo, max_rounds=args.max_rounds) diff --git a/docs/source/llm_examples/rag.py b/docs/source/llm_examples/rag.py index 166f1dbb4..eca2b4507 100644 --- a/docs/source/llm_examples/rag.py +++ b/docs/source/llm_examples/rag.py @@ -14,6 +14,7 @@ import litellm import numpy as np +from tenacity import stop_after_attempt from effectful.handlers.llm import Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler @@ -153,24 +154,23 @@ def answer_question(question: str) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( "--embedding-model", type=str, - default="lm_studio/nomic-ai/nomic-embed-text-v1.5-GGUF", + default="lm_studio/text-embedding-embeddinggemma-300m-qat", help="Embedding model to use", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - # Offline: build the index index = build_index(DOCUMENTS, embedding_model=args.embedding_model) @@ -186,7 +186,10 @@ def answer_question(question: str) -> str: provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): for question in questions: print(f"\nQ: {question}") answer = answer_question(question) diff --git a/docs/source/llm_examples/research_agent.py b/docs/source/llm_examples/research_agent.py index bc193db1b..308d2df23 100644 --- a/docs/source/llm_examples/research_agent.py +++ b/docs/source/llm_examples/research_agent.py @@ -119,7 +119,7 @@ def research_agent(question: str, max_attempts: int = 3) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -130,13 +130,6 @@ def research_agent(question: str, max_attempts: int = 3) -> str: ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) with handler(provider): diff --git a/docs/source/llm_examples/supervisor.py b/docs/source/llm_examples/supervisor.py index 1009d6a62..4c07a9f83 100644 --- a/docs/source/llm_examples/supervisor.py +++ b/docs/source/llm_examples/supervisor.py @@ -12,6 +12,7 @@ import urllib.parse import requests +from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler @@ -150,7 +151,7 @@ def supervised_research(question: str, max_retries: int = 3) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -159,18 +160,20 @@ def supervised_research(question: str, max_retries: int = 3) -> str: default=3, help="Maximum number of supervisor rejections before accepting", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): result = supervised_research( "What year was the Eiffel Tower completed and how tall is it?", max_retries=args.max_retries, diff --git a/docs/source/llm_examples/tao_agent.py b/docs/source/llm_examples/tao_agent.py index 8f9fbb0c1..2a8a14717 100644 --- a/docs/source/llm_examples/tao_agent.py +++ b/docs/source/llm_examples/tao_agent.py @@ -14,6 +14,7 @@ import urllib.parse import requests +from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import ( @@ -149,7 +150,7 @@ def _act(self, action: AgentAction, action_input: str) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -166,18 +167,14 @@ def _act(self, action: AgentAction, action_input: str) -> str: ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) agent = TAOAgent() - with handler(provider), handler(RetryLLMHandler(num_retries=args.num_retries)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): answer = agent.run( "How many tennis balls would fill an Olympic swimming pool?", max_steps=args.max_steps, diff --git a/docs/source/llm_examples/text2sql.py b/docs/source/llm_examples/text2sql.py index 5d511b5f8..a3e36f933 100644 --- a/docs/source/llm_examples/text2sql.py +++ b/docs/source/llm_examples/text2sql.py @@ -12,6 +12,8 @@ import sqlite3 import textwrap +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -138,18 +140,17 @@ def text_to_sql( parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - conn = create_sample_db() provider = LiteLLMProvider(model=args.model) @@ -159,7 +160,10 @@ def text_to_sql( "How many employees were hired after 2021?", ] - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): for question in questions: print(f"\nQ: {question}") try: diff --git a/docs/source/llm_examples/thinking.py b/docs/source/llm_examples/thinking.py index 101d5cb2e..058de61ef 100644 --- a/docs/source/llm_examples/thinking.py +++ b/docs/source/llm_examples/thinking.py @@ -10,6 +10,8 @@ import dataclasses import os +from tenacity import stop_after_attempt + from effectful.handlers.llm import Agent, Template from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.ops.semantics import handler @@ -68,7 +70,7 @@ def solve(self, problem: str, max_steps: int = 10) -> str: parser.add_argument( "--model", type=str, - default="lm_studio/zai-org/glm-4.7-flash", + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) parser.add_argument( @@ -86,15 +88,14 @@ def solve(self, problem: str, max_steps: int = 10) -> str: ), help="The problem to solve", ) + parser.add_argument( + "--num-retries", + type=int, + default=3, + help="Number of retries for malformed LLM output", + ) args = parser.parse_args() - if args.model.startswith("lm_studio/"): - assert os.environ.get("LM_STUDIO_API_BASE") - elif args.model.startswith("gpt-"): - assert os.environ.get("OPENAI_API_KEY") - elif args.model.startswith("claude-"): - assert os.environ.get("ANTHROPIC_API_KEY") - provider = LiteLLMProvider(model=args.model) problems = [ @@ -105,7 +106,10 @@ def solve(self, problem: str, max_steps: int = 10) -> str: ), ] - with handler(provider), handler(RetryLLMHandler(num_retries=3)): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): for problem in problems: thinker = Thinker() print(f"\nProblem: {problem}") From 2cbb0e0f3cbb1caa639834c94353bef860b74f5c Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 15:10:01 -0400 Subject: [PATCH 007/155] notebook sections as scripts --- docs/source/llm_examples/decode_callable.py | 78 +++++++++++++ .../llm_examples/higher_order_function.py | 105 +++++++++++++++++ docs/source/llm_examples/image_input.py | 63 ++++++++++ docs/source/llm_examples/prompt_templates.py | 88 ++++++++++++++ docs/source/llm_examples/retry_tool_errors.py | 91 +++++++++++++++ docs/source/llm_examples/retry_validation.py | 109 ++++++++++++++++++ docs/source/llm_examples/structured_output.py | 82 +++++++++++++ .../llm_examples/template_composition.py | 76 ++++++++++++ docs/source/llm_examples/tool_calling.py | 65 +++++++++++ 9 files changed, 757 insertions(+) create mode 100644 docs/source/llm_examples/decode_callable.py create mode 100644 docs/source/llm_examples/higher_order_function.py create mode 100644 docs/source/llm_examples/image_input.py create mode 100644 docs/source/llm_examples/prompt_templates.py create mode 100644 docs/source/llm_examples/retry_tool_errors.py create mode 100644 docs/source/llm_examples/retry_validation.py create mode 100644 docs/source/llm_examples/structured_output.py create mode 100644 docs/source/llm_examples/template_composition.py create mode 100644 docs/source/llm_examples/tool_calling.py diff --git a/docs/source/llm_examples/decode_callable.py b/docs/source/llm_examples/decode_callable.py new file mode 100644 index 000000000..05a09212a --- /dev/null +++ b/docs/source/llm_examples/decode_callable.py @@ -0,0 +1,78 @@ +"""Decoding LLM responses into Python objects, including callables. + +Demonstrates: +- Primitive type decoding (``int``) from a template that returns a number +- Synthesizing a Python ``Callable`` from a template, executed via + ``UnsafeEvalProvider`` from ``effectful.handlers.llm.evaluation`` +- ``inspect.getsource`` on the synthesized function +""" + +import argparse +import inspect +import os +from collections.abc import Callable + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def primes(first_digit: int) -> int: + """Give a prime number with {first_digit} as the first digit. Do not use any tools.""" + raise NotHandled + + +@Template.define +def count_char(char: str) -> Callable[[str], int]: + """Write a function which takes a string and counts the occurrances of '{char}'. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Decode LLM responses to Python objects (incl. callables)" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--first-digit", + type=int, + default=6, + help="First digit of the prime to request", + ) + parser.add_argument( + "--char", + type=str, + default="a", + help="Character whose occurrences the synthesized function will count", + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + + with handler(provider), handler(UnsafeEvalProvider()): + prime = primes(args.first_digit) + assert type(prime) is int + print(f"Prime starting with {args.first_digit}: {prime}") + + counter = count_char(args.char) + assert callable(counter) + print("\nGenerated function:") + print(inspect.getsource(counter)) + print(f'counter("banana") == {counter("banana")}') + print(f'counter("cherry") == {counter("cherry")}') diff --git a/docs/source/llm_examples/higher_order_function.py b/docs/source/llm_examples/higher_order_function.py new file mode 100644 index 000000000..da2410959 --- /dev/null +++ b/docs/source/llm_examples/higher_order_function.py @@ -0,0 +1,105 @@ +"""Generating higher-order functions that call other templates. + +Demonstrates: +- A template returning a ``Callable``, evaluated via ``UnsafeEvalProvider`` +- The synthesized function calling sub-templates (``write_chapter``, + ``judge_chapter``) at runtime +- ``RetryLLMHandler`` to recover from transient validation/runtime errors +- ``inspect.getsource`` on the generated function +""" + +import argparse +import inspect +import os +from collections.abc import Callable +from typing import Literal + +from tenacity import stop_after_attempt + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Sub-templates the generated function may call +# --------------------------------------------------------------------------- + + +@Template.define +def write_chapter(chapter_number: int, chapter_name: str) -> str: + """Write a short story about {chapter_number}. Do not use any tools.""" + raise NotHandled + + +@Template.define +def judge_chapter(story_so_far: str, chapter_number: int) -> bool: + """Decide if the new chapter is coherent with the story so far. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Orchestrator template returning a callable +# --------------------------------------------------------------------------- + + +@Template.define +def write_multi_chapter_story(style: Literal["moral", "funny"]) -> Callable[[str], str]: + """Generate a function that writes a story in style: {style} about the given topic. + + If you raise an exception, handle it yourself. + The program can use helper functions defined elsewhere (DO NOT REDEFINE THEM): + - write_chapter(chapter_number: int, chapter_name: str) -> str + - judge_chapter(story_so_far: str, chapter_number: int) -> bool + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate a higher-order function that calls sub-templates" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--topic", type=str, default="a curious cat", help="Story topic" + ) + parser.add_argument( + "--style", + type=str, + choices=["moral", "funny"], + default="moral", + help="Story style", + ) + parser.add_argument( + "--num-retries", + type=int, + default=4, + help="Number of retries for malformed LLM output", + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + + print("Sub-templates available to write_multi_chapter_story:") + print(list(write_multi_chapter_story.tools.keys())) + + with ( + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + handler(provider), + handler(UnsafeEvalProvider()), + ): + print(f"\n=== Generating story function (style={args.style}) ===") + story_fn = write_multi_chapter_story(args.style) + print(inspect.getsource(story_fn)) + print(f"\n=== Running generated function on {args.topic!r} ===") + print(story_fn(args.topic)) diff --git a/docs/source/llm_examples/image_input.py b/docs/source/llm_examples/image_input.py new file mode 100644 index 000000000..14375b294 --- /dev/null +++ b/docs/source/llm_examples/image_input.py @@ -0,0 +1,63 @@ +"""Passing PIL images directly to a template. + +Demonstrates: +- Templates accepting ``PIL.Image.Image`` arguments +- Inline base64 image data so the script is self-contained +""" + +import argparse +import base64 +import io +import os + +from PIL import Image + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Inline image (32x32 yellow smiley face) +# --------------------------------------------------------------------------- + +IMAGE_BASE64 = ( + "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAhElEQVR4nO2W4QqA" + "MAiEVXr/VzYWDGoMdk7Cgrt/sUs/DqZTd3EplFU2JwATYAJMoOlAB4bq89s95+Mg" + "+gyAchsKAYplBBBA43hFhfxnUixDjdEUUL8hpr7R0KLdt9qElzcyiu8As+Kr8zQA" + "mgLavAl+kIzFZyCRxtsAmWb/voZvqRzgBE1sIDuVFX4eAAAAAElFTkSuQmCC" +) + + +# --------------------------------------------------------------------------- +# Template +# --------------------------------------------------------------------------- + + +@Template.define +def describe_image(image: Image.Image) -> str: + """Return a short description of the following image. + {image} + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Pass a PIL image to a template") + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use (must support image inputs)", + ) + args = parser.parse_args() + + image = Image.open(io.BytesIO(base64.b64decode(IMAGE_BASE64))) + + provider = LiteLLMProvider(model=args.model) + with handler(provider): + print(describe_image(image)) diff --git a/docs/source/llm_examples/prompt_templates.py b/docs/source/llm_examples/prompt_templates.py new file mode 100644 index 000000000..56369056a --- /dev/null +++ b/docs/source/llm_examples/prompt_templates.py @@ -0,0 +1,88 @@ +"""Basic prompt templates and deterministic caching. + +Demonstrates: +- ``@Template.define`` for declaring an LLM-backed function +- Non-determinism: calling the same template twice yields different results +- ``functools.cache`` to make a template call deterministic in-process +- ``LiteLLMProvider(caching=True)`` with ``litellm.cache`` for cross-process caching +""" + +import argparse +import functools +import os + +import litellm +from litellm.caching.caching import Cache + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def limerick(theme: str) -> str: + """Write a limerick on the theme of {theme}. Do not use any tools.""" + raise NotHandled + + +@functools.cache +@Template.define +def haiku(theme: str) -> str: + """Write a haiku on the theme of {theme}. Do not use any tools.""" + raise NotHandled + + +@Template.define +def haiku_no_cache(theme: str) -> str: + """Write a haiku on the theme of {theme}. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Basic prompt templates and deterministic caching" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--theme", type=str, default="fish", help="Theme for the poem" + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + + print("=== Non-deterministic limerick (two independent calls) ===") + with handler(provider): + print(limerick(args.theme)) + print("-" * 40) + print(limerick(args.theme)) + + print("\n=== functools.cache: same result on second call ===") + with handler(provider): + print(haiku(args.theme)) + print("-" * 40) + print(haiku(args.theme)) + + print("\n=== LiteLLMProvider(caching=True): backed by litellm.cache ===") + litellm.cache = Cache() + provider_cached = LiteLLMProvider(model=args.model, caching=True) + try: + with handler(provider_cached): + print(haiku_no_cache(args.theme)) + print("-" * 40) + print(haiku_no_cache(args.theme)) + finally: + litellm.cache = None diff --git a/docs/source/llm_examples/retry_tool_errors.py b/docs/source/llm_examples/retry_tool_errors.py new file mode 100644 index 000000000..7ce0b7b11 --- /dev/null +++ b/docs/source/llm_examples/retry_tool_errors.py @@ -0,0 +1,91 @@ +"""Retrying tool execution failures. + +Demonstrates: +- ``RetryLLMHandler`` surfacing tool exceptions back to the LLM as tool messages +- A flaky tool (``unstable_service``) that succeeds only after multiple attempts +- The contrast between an unhandled failure and a retry-handled success +""" + +import argparse +import os + +from tenacity import stop_after_attempt + +from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Flaky tool +# --------------------------------------------------------------------------- + +call_count = 0 +REQUIRED_RETRIES = 3 + + +@Tool.define +def unstable_service() -> str: + """Fetch data from an unstable external service. May require retries.""" + global call_count + call_count += 1 + if call_count < REQUIRED_RETRIES: + raise ConnectionError( + f"Service unavailable! Attempt {call_count}/{REQUIRED_RETRIES}. Please retry." + ) + return "{ 'status': 'ok', 'data': [1, 2, 3] }" + + +# --------------------------------------------------------------------------- +# Template (unstable_service auto-captured from lexical scope) +# --------------------------------------------------------------------------- + + +@Template.define +def fetch_data() -> str: + """Use the unstable_service tool to fetch data.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Retry LLM template calls when tools raise exceptions" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=4, + help="Number of retries for tool/decode failures", + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + + print("=== Without RetryLLMHandler ===") + with handler(provider): + try: + result = fetch_data() + print(f"Result: {result}") + except Exception as e: + print(f"Error: {e}") + + # Reset for the retry-enabled run. + call_count = 0 + + print("\n=== With RetryLLMHandler ===") + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): + result = fetch_data() + print(f"Result: {result} (after {call_count} tool attempts)") diff --git a/docs/source/llm_examples/retry_validation.py b/docs/source/llm_examples/retry_validation.py new file mode 100644 index 000000000..960b3f3fc --- /dev/null +++ b/docs/source/llm_examples/retry_validation.py @@ -0,0 +1,109 @@ +"""Retrying when structured-output validation fails. + +Demonstrates: +- A pydantic dataclass with ``field_validator`` constraints +- ``RetryLLMHandler`` feeding ``PydanticCustomError`` messages back to the LLM + so it can correct its output on a subsequent attempt +""" + +import argparse +import os + +import pydantic +from pydantic import field_validator +from pydantic_core import PydanticCustomError +from tenacity import stop_after_attempt + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Validated structured output +# --------------------------------------------------------------------------- + + +@pydantic.dataclasses.dataclass +class Rating: + score: int + explanation: str + + @field_validator("score") + @classmethod + def check_score(cls, v): + if v < 1 or v > 5: + raise PydanticCustomError( + "invalid_score", + "score must be 1–5, got {v}", + {"v": v}, + ) + return v + + @field_validator("explanation") + @classmethod + def check_explanation_contains_score(cls, v, info): + score = info.data.get("score", None) + if score is not None and str(score) not in v: + raise PydanticCustomError( + "invalid_explanation", + "explanation must mention the score {score}, got '{explanation}'", + {"score": score, "explanation": v}, + ) + return v + + +# --------------------------------------------------------------------------- +# Template +# --------------------------------------------------------------------------- + + +@Template.define +def give_rating_for_movie(movie_name: str) -> Rating: + """Give a rating for {movie_name}. The explanation MUST include the numeric score. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Retry on pydantic validation errors in LLM responses" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--movie", type=str, default="Die Hard", help="Movie to rate" + ) + parser.add_argument( + "--num-retries", + type=int, + default=4, + help="Number of retries for malformed LLM output", + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + + print("=== Without RetryLLMHandler ===") + with handler(provider): + try: + rating = give_rating_for_movie(args.movie) + print(f"Score: {rating.score}/5\nExplanation: {rating.explanation}") + except Exception as e: + print(f"Error: {e}") + + print("\n=== With RetryLLMHandler ===") + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): + rating = give_rating_for_movie(args.movie) + print(f"Score: {rating.score}/5") + print(f"Explanation: {rating.explanation}") diff --git a/docs/source/llm_examples/structured_output.py b/docs/source/llm_examples/structured_output.py new file mode 100644 index 000000000..0f6c85f88 --- /dev/null +++ b/docs/source/llm_examples/structured_output.py @@ -0,0 +1,82 @@ +"""Structured output via dataclasses. + +Demonstrates: +- Dataclass return types decoded from constrained LLM generation +- Round-tripping a dataclass: one template produces it, another consumes it +""" + +import argparse +import dataclasses +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Structured output +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class KnockKnockJoke: + whos_there: str + punchline: str + + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def write_joke(theme: str) -> KnockKnockJoke: + """Write a knock-knock joke on the theme of {theme}. Do not use any tools.""" + raise NotHandled + + +@Template.define +def rate_joke(joke: KnockKnockJoke) -> bool: + """Decide if {joke} is funny or not. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + + +def do_comedy(theme: str) -> None: + joke = write_joke(theme) + print("> You are onstage at a comedy club. You tell the following joke:") + print( + f"Knock knock.\nWho's there?\n{joke.whos_there}.\n" + f"{joke.whos_there} who?\n{joke.punchline}" + ) + if rate_joke(joke): + print("> The crowd laughs politely.") + else: + print("> The crowd stares in stony silence.") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Structured output via dataclasses") + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--theme", type=str, default="lizards", help="Theme for the joke" + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + with handler(provider): + do_comedy(args.theme) diff --git a/docs/source/llm_examples/template_composition.py b/docs/source/llm_examples/template_composition.py new file mode 100644 index 000000000..5f8803078 --- /dev/null +++ b/docs/source/llm_examples/template_composition.py @@ -0,0 +1,76 @@ +"""Template composition: templates can call other templates. + +Demonstrates: +- Sub-templates auto-captured into an orchestrator template's lexical scope +- Inspecting ``write_story.tools`` to confirm sub-templates are exposed to the LLM +- The orchestrator dispatches to the right sub-template based on a style argument +""" + +import argparse +import os + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Sub-templates +# --------------------------------------------------------------------------- + + +@Template.define +def story_with_moral(topic: str) -> str: + """Write a short story about {topic} and end with a moral lesson. Do not use any tools.""" + raise NotHandled + + +@Template.define +def story_funny(topic: str) -> str: + """Write a funny, humorous story about {topic}. Do not use any tools.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Orchestrator template +# --------------------------------------------------------------------------- + + +@Template.define +def write_story(topic: str, style: str) -> str: + """Write a story about {topic} in the style: {style}. + Available styles: 'moral' for a story with a lesson, 'funny' for humor. + Use story_funny for humor, story_with_moral for a story with a lesson. + """ + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Template composition with auto-captured sub-templates" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--topic", type=str, default="a curious cat", help="Story topic" + ) + args = parser.parse_args() + + assert story_with_moral in write_story.tools.values() + assert story_funny in write_story.tools.values() + print("Sub-templates available to write_story:", list(write_story.tools.keys())) + + provider = LiteLLMProvider(model=args.model) + with handler(provider): + print("\n=== Story with moral ===") + print(write_story(args.topic, "moral")) + print("\n=== Funny story ===") + print(write_story(args.topic, "funny")) diff --git a/docs/source/llm_examples/tool_calling.py b/docs/source/llm_examples/tool_calling.py new file mode 100644 index 000000000..f7d9b9f28 --- /dev/null +++ b/docs/source/llm_examples/tool_calling.py @@ -0,0 +1,65 @@ +"""Tool calling: templates invoke Python callables exposed via ``@Tool.define``. + +Demonstrates: +- ``@Tool.define`` for exposing a Python function to the model +- Lexical-scope auto-capture: tools defined alongside a template are made + available to the LLM without explicit registration +- The model chains multiple tool calls to answer a multi-step query +""" + +import argparse +import os + +from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + +# --------------------------------------------------------------------------- +# Tools +# --------------------------------------------------------------------------- + + +@Tool.define +def cities() -> list[str]: + """Return a list of cities that can be passed to `weather`.""" + return ["Chicago", "New York", "Barcelona"] + + +@Tool.define +def weather(city: str) -> str: + """Given a city name, return a description of the weather in that city.""" + status = {"Chicago": "cold", "New York": "wet", "Barcelona": "sunny"} + return status.get(city, "unknown") + + +# --------------------------------------------------------------------------- +# Template (cities and weather are auto-captured from lexical scope) +# --------------------------------------------------------------------------- + + +@Template.define +def vacation() -> str: + """Use the provided tools to suggest a city that has good weather. Use only the `cities` and `weather` tools provided.""" + raise NotHandled + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Tool calling with auto-captured lexical scope" + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + args = parser.parse_args() + + provider = LiteLLMProvider(model=args.model) + with handler(provider): + print(vacation()) From ed3d16ba31dc015ffe0e913cd7f65bd345c4b6a2 Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 15:10:36 -0400 Subject: [PATCH 008/155] lint --- docs/source/llm_examples/prompt_templates.py | 4 +--- docs/source/llm_examples/retry_validation.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/source/llm_examples/prompt_templates.py b/docs/source/llm_examples/prompt_templates.py index 56369056a..74c1cf0d3 100644 --- a/docs/source/llm_examples/prompt_templates.py +++ b/docs/source/llm_examples/prompt_templates.py @@ -57,9 +57,7 @@ def haiku_no_cache(theme: str) -> str: default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) - parser.add_argument( - "--theme", type=str, default="fish", help="Theme for the poem" - ) + parser.add_argument("--theme", type=str, default="fish", help="Theme for the poem") args = parser.parse_args() provider = LiteLLMProvider(model=args.model) diff --git a/docs/source/llm_examples/retry_validation.py b/docs/source/llm_examples/retry_validation.py index 960b3f3fc..aab80df9e 100644 --- a/docs/source/llm_examples/retry_validation.py +++ b/docs/source/llm_examples/retry_validation.py @@ -78,9 +78,7 @@ def give_rating_for_movie(movie_name: str) -> Rating: default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) - parser.add_argument( - "--movie", type=str, default="Die Hard", help="Movie to rate" - ) + parser.add_argument("--movie", type=str, default="Die Hard", help="Movie to rate") parser.add_argument( "--num-retries", type=int, From 3cac33d7b3ff6b1a6bdbfe91d231ec6e1de834a2 Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 15:18:14 -0400 Subject: [PATCH 009/155] nit --- docs/source/llm_examples/supervisor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/llm_examples/supervisor.py b/docs/source/llm_examples/supervisor.py index 4c07a9f83..29f258fe0 100644 --- a/docs/source/llm_examples/supervisor.py +++ b/docs/source/llm_examples/supervisor.py @@ -154,6 +154,12 @@ def supervised_research(question: str, max_retries: int = 3) -> str: default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) + parser.add_argument( + "--question", + type=str, + default="What year was the Eiffel Tower completed and how tall is it?", + help="Research question to answer", + ) parser.add_argument( "--max-retries", type=int, @@ -175,7 +181,7 @@ def supervised_research(question: str, max_retries: int = 3) -> str: handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), ): result = supervised_research( - "What year was the Eiffel Tower completed and how tall is it?", + args.question, max_retries=args.max_retries, ) print(f"\nFinal answer: {result}") From ee6fc1f27eb53d659e854b4f6e5b13a4cf2d289b Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 25 Apr 2026 15:31:18 -0400 Subject: [PATCH 010/155] retry --- docs/source/llm_examples/decode_callable.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/source/llm_examples/decode_callable.py b/docs/source/llm_examples/decode_callable.py index 05a09212a..195167bce 100644 --- a/docs/source/llm_examples/decode_callable.py +++ b/docs/source/llm_examples/decode_callable.py @@ -12,8 +12,10 @@ import os from collections.abc import Callable +from tenacity import stop_after_attempt + from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider +from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.handlers.llm.evaluation import UnsafeEvalProvider from effectful.ops.semantics import handler from effectful.ops.types import NotHandled @@ -49,6 +51,12 @@ def count_char(char: str) -> Callable[[str], int]: default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), help="LLM model to use", ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed LLM output", + ) parser.add_argument( "--first-digit", type=int, @@ -65,7 +73,11 @@ def count_char(char: str) -> Callable[[str], int]: provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(UnsafeEvalProvider()): + with ( + handler(provider), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + handler(UnsafeEvalProvider()), + ): prime = primes(args.first_digit) assert type(prime) is int print(f"Prime starting with {args.first_digit}: {prime}") From c2f6bbcfe5af68ba967dd3f10fdd123ff198768e Mon Sep 17 00:00:00 2001 From: Jack Feser Date: Tue, 2 Jun 2026 15:01:09 -0400 Subject: [PATCH 011/155] Add example using object handles (#597) * add example using object handles * coerce into the same format --------- Co-authored-by: Eli --- docs/source/llm_examples/image_tool.py | 90 ++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/source/llm_examples/image_tool.py diff --git a/docs/source/llm_examples/image_tool.py b/docs/source/llm_examples/image_tool.py new file mode 100644 index 000000000..813058494 --- /dev/null +++ b/docs/source/llm_examples/image_tool.py @@ -0,0 +1,90 @@ +import argparse +import os + +from PIL import Image + +from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm.completions import ( + LiteLLMProvider, + RetryLLMHandler, +) +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + + +class ImageTools(Agent): + """You are an image processing agent.""" + + _image_to_handle: dict[int, int] + _handle_to_image: dict[int, Image.Image] + + def __init__(self): + self._image_to_handle = {} + self._handle_to_image = {} + + def _encode(self, image: Image.Image) -> int: + image_id = id(image) + handle = self._image_to_handle.get(image_id, None) + if handle is not None: + return handle + + handle = len(self._image_to_handle) + self._image_to_handle[image_id] = handle + + assert handle not in self._handle_to_image + self._handle_to_image[handle] = image + return handle + + def _decode(self, image_handle: int) -> Image.Image: + return self._handle_to_image[image_handle] + + @Tool.define + def rotate(self, image: int, angle: float) -> int: + """Returns a rotated copy of this image. The copy is rotated by `angle` + degrees counterclockwise around the image center. + + """ + return self._encode(self._decode(image).rotate(angle)) + + @Tool.define + def concat_horiz(self, i1_h: int, i2_h: int) -> int: + """Concatenates two images horizontally. The larger image will be + cropped to the height of the smaller image. + + """ + i1 = self._decode(i1_h) + i2 = self._decode(i2_h) + i3 = Image.new("RGB", (i1.width + i2.width, min(i1.height, i2.height))) + i3.paste(i1, (0, 0)) + i3.paste(i2, (i1.width, 0)) + return self._encode(i3) + + @Template.define + def _rotate_and_concat(self, i: int) -> int: + """Create an image consisting of four copies of the image {i} + concatenated horizontally. Each copy should be rotated 90 degrees from + the previous. + + """ + raise NotHandled + + def rotate_and_concat(self, i: Image.Image) -> Image.Image: + return self._decode(self._rotate_and_concat(self._encode(i))) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use (must support image inputs)", + ) + args = parser.parse_args() + + image_agent = ImageTools() + img = Image.open("../_static/img/chirho_logo_wide.png") + + provider = LiteLLMProvider(model=args.model) + with handler(provider), handler(RetryLLMHandler()): + image_agent.rotate_and_concat(img).show() From c3555c435c7cd652c27a1c9fe375e9dc3af38f85 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 19 Jun 2026 14:49:37 -0400 Subject: [PATCH 012/155] Synthesis-based template handling --- effectful/handlers/llm/completions.py | 218 ++++++++++++++++++-- effectful/handlers/llm/encoding.py | 4 + effectful/handlers/llm/template.py | 31 +++ tests/test_handlers_llm_provider.py | 282 +++++++++++++++++++++++++- 4 files changed, 513 insertions(+), 22 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index d69d827f1..2bb9e085d 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -33,6 +33,7 @@ from effectful.handlers.llm.evaluation import ReplSession from effectful.handlers.llm.template import ( Agent, + FinalTool, Template, Tool, _is_recursive_signature, @@ -176,9 +177,6 @@ def to_feedback_message(self, include_traceback: bool) -> Message: ) -type MessageResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] - - class _LexicalVariableTool[T](Tool[[], T]): """A zero-arg `Tool` that returns the captured value of a variable from a `Template`'s lexical context. @@ -358,6 +356,166 @@ def _collect( return tools +@Operation.define +def _synthesis_signature() -> inspect.Signature | None: + """Return the signature of the in-flight Template call, or ``None``. + + `SynthesizeAndCall` installs a fresh handler for this inside each + `Template.__apply__` (mirroring `_repl_session`), giving it a lifetime of + exactly one Template call. Outside such a scope there is no synthesis + target, so this falls back to ``None`` -- e.g. when tools are listed outside + a Template call -- and no synthesis tool is injected. + """ + return None + + +def _synthesis_final_tool( + signature: inspect.Signature, + env: collections.abc.Mapping[str, typing.Any], + name: str = "submit_solution", +) -> FinalTool: + """Build a :class:`FinalTool` that finalizes a Template by code synthesis. + + The tool takes one argument -- a function with the Template's signature, + synthesized from the model's code by the existing ``Encodable[Callable[...]]`` + machinery -- and applies it to the original inputs (recovered from ``env``), + returning the value. Because it is a :class:`FinalTool`, calling it + terminates the completion loop and its return value is the Template's result. + """ + param_types = [] + for pname, param in signature.parameters.items(): + if param.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): + raise TypeError( + f"SynthesizeAndCall cannot synthesize a function for parameter " + f"'{pname}' of kind {param.kind.description}: variadic parameters " + "cannot be expressed as a Callable type signature." + ) + param_types.append( + param.annotation + if param.annotation is not inspect.Parameter.empty + else typing.Any + ) + return_type = signature.return_annotation + if return_type is inspect.Signature.empty: + raise TypeError( + "SynthesizeAndCall requires a return annotation on the Template's " + "signature to construct the synthesis tool's Callable type." + ) + + callable_type = collections.abc.Callable[param_types, return_type] # type: ignore[valid-type] + + # Recover the original arguments from `env` by name, respecting each + # parameter's kind so positional-only and keyword-only parameters bind + # correctly (variadic kinds were rejected above). + pos_names = [ + pname + for pname, param in signature.parameters.items() + if param.kind + in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + ] + kw_names = [ + pname + for pname, param in signature.parameters.items() + if param.kind is inspect.Parameter.KEYWORD_ONLY + ] + + def submit_solution(implementation): + bound = signature.bind( + *(env[pname] for pname in pos_names), + **{pname: env[pname] for pname in kw_names}, + ) + return implementation(*bound.args, **bound.kwargs) + + submit_solution.__name__ = name + submit_solution.__qualname__ = name + submit_solution.__module__ = __name__ + submit_solution.__doc__ = ( + "Submit your final answer as a Python function implementing the task. " + "The function must have the required signature; it is applied to the " + "original inputs and its return value is your final answer." + ) + submit_solution.__annotations__ = { + "implementation": callable_type, + "return": signature.return_annotation, + } + return FinalTool.define(submit_solution) + + +class SynthesizeAndCall(ObjectInterpretation): + """Answer a Template by synthesizing a function and calling it. + + Instead of asking the LLM to generate an instance of the Template's return + type directly, this handler exposes a :class:`FinalTool` that lets the model + "answer" by writing a Python function with the Template's signature. The + harness applies that function to the original arguments and its return value + becomes the Template's result. This is the declarative "CodeAdapt" workflow: + the LLM writes code implementing the body of the Template rather than + reasoning out the answer itself. + + The synthesis tool is offered *alongside* the Template's normal completion + paths rather than replacing them: across turns the model may freely call any + other tool in scope (their results are fed back as usual), and it may still + answer the return type directly via structured output. The loop terminates + when it either answers directly or calls the synthesis :class:`FinalTool`. + To force the synthesis path, pass ``tool_choice="required"`` (handler config + is forwarded to the model request). The function is synthesized by reusing + the existing ``Callable`` synthesis machinery: the tool's argument is typed + as ``Callable[[params], ret]``, so :func:`call_assistant`'s tool-call + decoding parses, type-checks, compiles and executes the model's code into a + real function before it is applied. + + Scoping mirrors :class:`PythonRepl`: this handles `Template.__apply__` to + introduce a fresh `_synthesis_signature` handler bound to that call's + signature, and handles `collect_tools` to inject the synthesis tool built + from it. The synthesis target is therefore introduced and eliminated by its + own handler, bounded to the Template call by construction -- nested Template + calls get their own target. + + Failures compose with :class:`RetryLLMHandler`: a function that fails to + synthesize surfaces as a :class:`ToolCallDecodingError`, and one that raises + when applied to the inputs as a :class:`ToolCallExecutionError`; both are fed + back to the model as a tool message and the loop continues so it can revise:: + + with ( + handler(LiteLLMProvider(model="gpt-5-mini")), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler()), + ): + ... + + Requires an eval provider (e.g. :class:`UnsafeEvalProvider` or + :class:`RestrictedEvalProvider`) to be installed so the synthesized code can + be compiled and executed. + """ + + @implements(Template.__apply__) + def _apply[**P, T]( + self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs + ) -> T: + # Bind the synthesis target to this Template call's signature for the + # duration of the call, so `collect_tools` can build the tool from it. + signature = template.__signature__ + with handler({_synthesis_signature: lambda: signature}): + return fwd() + + @implements(collect_tools) + def _collect( + self, env: collections.abc.Mapping[str, typing.Any] + ) -> collections.abc.Mapping[str, Tool]: + tools = dict(fwd()) + signature = _synthesis_signature() + if signature is not None: + final_tool = _synthesis_final_tool(signature, env) + tools[final_tool.__name__] = final_tool + return tools + + @Operation.define @functools.wraps(litellm.completion) def completion(*args, **kwargs) -> typing.Any: @@ -374,13 +532,16 @@ class _BoxedResponse[T](pydantic.BaseModel): value: T +type AssistantResult[T] = tuple[Message, typing.Sequence[DecodedToolCall], T | None] + + @Operation.define def call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], model: str, **kwargs, -) -> MessageResult[T]: +) -> AssistantResult[T]: """Low-level LLM request. Handlers may log/modify requests and delegate via fwd(). This effect is emitted for model request/response rounds so handlers can @@ -426,13 +587,29 @@ def call_assistant[T]( raw_message = _make_message({**message.model_dump(mode="json")}) append_message(raw_message) + raw_tool_calls = message.get("tool_calls") or [] tool_calls: list[DecodedToolCall] = [] encoding: pydantic.TypeAdapter[DecodedToolCall] = pydantic.TypeAdapter( Encodable[DecodedToolCall] ) - for raw_tool_call in message.get("tool_calls") or []: + for raw_tool_call in raw_tool_calls: try: tool_calls += [encoding.validate_python(raw_tool_call, context=tools)] + if isinstance(tool_calls[-1].tool, FinalTool): + if not ( + tool_calls[-1].result_type == response_type + or issubclass(tool_calls[-1].result_type, response_type) + ): + raise TypeError( + f"FinalTool '{tool_calls[-1].name}' returns {tool_calls[-1].result_type!r}, " + f"which does not match the Template's result type {response_type!r}." + ) + if len(raw_tool_calls) > 1: + raise TypeError( + f"A FinalTool call must be the only tool call in its turn, but " + f"{len(raw_tool_calls)} tool calls were requested " + f"({sum(isinstance(encoding.validate_python(tc, context=tools).tool, FinalTool) for tc in raw_tool_calls)} of them final). Call the final tool alone." + ) except Exception as e: raise ToolCallDecodingError( raw_tool_call=raw_tool_call, @@ -460,12 +637,18 @@ def call_assistant[T]( return (raw_message, tool_calls, result) +type ToolResult[T] = tuple[Message, T | None, bool] + + @Operation.define -def call_tool(tool_call: DecodedToolCall) -> Message: +def call_tool[T](tool_call: DecodedToolCall[T]) -> ToolResult[T]: """Implements a roundtrip call to a python function. Input is a json string representing an LLM tool call request parameters. The output is the serialised response to the model. + Returns the appended tool message, the tool's return value, and whether the + call was a finalizing one (a :class:`FinalTool` call, whose value becomes the + Template's result and terminates the completion loop). """ # call tool with python types try: @@ -485,7 +668,7 @@ def call_tool(tool_call: DecodedToolCall) -> Message: dict(role="tool", content=encoded_result, tool_call_id=tool_call.id), ) append_message(message) - return message + return (message, result, isinstance(tool_call.tool, FinalTool)) @Operation.define @@ -616,7 +799,7 @@ def _call_assistant[T]( response_type: type[T], model: str, **kwargs, - ) -> MessageResult[T]: + ) -> AssistantResult[T]: _message_sequence = _get_history().copy() with handler({_get_history: lambda: _message_sequence}): @@ -626,12 +809,16 @@ def _call_assistant[T]( return (message, tool_calls, result) @implements(call_tool) - def _call_tool(self, tool_call: DecodedToolCall) -> Message: + def _call_tool[T](self, tool_call: DecodedToolCall[T]) -> ToolResult[T]: """Handle tool execution with runtime error capture. Runtime errors from tool execution are captured and returned as error messages to the LLM. Only exceptions matching `catch_tool_errors` are caught; others propagate up. + + A captured failure is reported as ``is_final=False`` so that the + completion loop continues even when a :class:`FinalTool` call raised: + the model sees the error message and gets another turn to retry. """ try: return fwd(tool_call) @@ -639,7 +826,7 @@ def _call_tool(self, tool_call: DecodedToolCall) -> Message: if isinstance(e.original_error, self.catch_tool_errors): message = e.to_feedback_message(self.include_traceback) append_message(message) - return message + return (message, None, False) else: raise @@ -682,14 +869,17 @@ def _call[**P, T]( message: Message = call_user(template.__prompt_template__, env) # loop based on: https://cookbook.openai.com/examples/reasoning_function_calls - tool_calls: list[DecodedToolCall] = [] result: T | None = None - while message["role"] != "assistant" or tool_calls: + is_final: bool = False + while not is_final: message, tool_calls, result = call_assistant( env, template.__signature__.return_annotation, **self.config ) - for tool_call in tool_calls: - message = call_tool(tool_call) + if tool_calls: + for tool_call in tool_calls: + message, result, is_final = call_tool(tool_call) + else: + is_final = True try: _get_history() diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 18daa48ab..7878e2a09 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -108,6 +108,10 @@ class DecodedToolCall[T]: id: ToolCallID name: str + @property + def result_type(self) -> type[T]: + return inspect.signature(self.tool).return_annotation + if typing.TYPE_CHECKING: type Encodable[T] = typing.Annotated[T, "encoded"] diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index cffda38cf..8171440e0 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -109,6 +109,37 @@ def define(cls, *args, **kwargs) -> "Tool[P, T]": return typing.cast("Tool[P, T]", super().define(*args, **kwargs)) +class FinalTool[**P, T](Tool[P, T]): + """A :class:`Tool` whose invocation *finalizes* a :class:`Template` call. + + During completion a :class:`Template` lets the LLM freely call any tool in + scope, feeding each tool's result back for another turn. When the LLM + instead calls a :class:`FinalTool`, that tool's return value becomes the + Template's result and the completion loop terminates -- no further model + turn is taken, so the value is attributed to executing the tool rather than + generated by the model. + + This is the mechanism behind code-synthesis completion (see + :class:`effectful.handlers.llm.completions.SynthesizeAndCall`): the LLM + "answers" by calling a final tool with a function it wrote, the harness + applies that function to the original inputs, and the resulting value is the + answer. + + A finalizing call that *fails* does not terminate the loop -- the error is + fed back as a tool message and the model is given another turn (see + :class:`effectful.handlers.llm.completions.RetryLLMHandler`). + """ + + @classmethod + def define(cls, *args, **kwargs) -> "FinalTool[P, T]": + """Define a final tool. + + See :func:`effectful.ops.types.Operation.define` for more information on + the use of :func:`FinalTool.define`. + """ + return typing.cast("FinalTool[P, T]", super().define(*args, **kwargs)) + + class Template[**P, T](Tool[P, T]): """A :class:`Template` is a function that is implemented by a large language model. diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index d6325041a..5380c46a1 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -29,15 +29,18 @@ from effectful.handlers.llm import Agent, Template from effectful.handlers.llm.completions import ( DecodedToolCall, + FinalTool, LexicalReaders, LiteLLMProvider, PythonRepl, ResultDecodingError, RetryLLMHandler, + SynthesizeAndCall, Tool, ToolCallDecodingError, ToolCallExecutionError, _get_history, + _synthesis_final_tool, call_assistant, call_tool, collect_tools, @@ -479,6 +482,37 @@ def make_tool_call_response( ) +def make_multi_tool_call_response( + calls: list[tuple[str, str, str]], +) -> ModelResponse: + """Create a ModelResponse with several tool calls in one assistant turn. + + Each entry is ``(tool_name, arguments_json, tool_call_id)``. + """ + return ModelResponse( + id="test", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": cid, + "type": "function", + "function": {"name": name, "arguments": args}, + } + for name, args, cid in calls + ], + }, + "finish_reason": "tool_calls", + } + ], + model="test-model", + ) + + def make_text_response(content: str) -> ModelResponse: """Create a ModelResponse with text content.""" return ModelResponse( @@ -983,7 +1017,7 @@ def test_retry_handler_catches_tool_runtime_error(self): tool_call = DecodedToolCall(failing_tool, bound_args, "call_1", "failing_tool") with handler(RetryLLMHandler()): - result = call_tool(tool_call) + result, _, _ = call_tool(tool_call) # The result should be an error message, not an exception assert result["role"] == "tool" @@ -1000,7 +1034,7 @@ def test_retry_handler_catches_division_by_zero(self): tool_call = DecodedToolCall(divide_tool, bound_args, "call_div", "divide_tool") with handler(RetryLLMHandler()): - result = call_tool(tool_call) + result, _, _ = call_tool(tool_call) assert result["role"] == "tool" assert result["tool_call_id"] == "call_div" @@ -1015,7 +1049,7 @@ def test_successful_tool_execution_returns_result(self): tool_call = DecodedToolCall(add_numbers, bound_args, "call_add", "add_numbers") with handler(RetryLLMHandler()): - result = call_tool(tool_call) + result, _, _ = call_tool(tool_call) assert result["role"] == "tool" assert result["tool_call_id"] == "call_add" @@ -1302,6 +1336,238 @@ def test_synthesize_three_params(self, request): assert multiply_three(5, 0, 10) == 0 +@FinalTool.define +def final_int(x: int) -> int: + """Finalize the answer with an int.""" + return x + + +@FinalTool.define +def final_str(x: int) -> str: + """Finalize the answer with a str (deliberately mismatched return type).""" + return str(x) + + +class TestFinalToolInvariants: + """`call_assistant` enforces that a FinalTool call is unambiguous: it must be + the only call this turn and its return type must match `response_type`. + Violations fail like any malformed tool call (`ToolCallDecodingError`).""" + + def _run(self, response: ModelResponse, env, response_type): + mock = MockCompletionHandler([response]) + message_sequence = collections.OrderedDict( + id1={"id": "id1", "role": "user", "content": "go"}, + ) + with ( + handler(mock), + handler({_get_history: lambda: message_sequence}), + ): + return call_assistant( + env=env, response_type=response_type, model="test-model" + ) + + def test_lone_matching_final_call_is_accepted(self): + _, tool_calls, result = self._run( + make_tool_call_response("final_int", '{"x": 5}', "c1"), + env={"final_int": final_int}, + response_type=int, + ) + assert len(tool_calls) == 1 + assert isinstance(tool_calls[0].tool, FinalTool) + assert result is None # call_assistant does not execute tools + + def test_final_mixed_with_normal_is_rejected(self): + with pytest.raises(ToolCallDecodingError): + self._run( + make_multi_tool_call_response( + [ + ("add_numbers", '{"a": 1, "b": 2}', "c1"), + ("final_int", '{"x": 5}', "c2"), + ] + ), + env={"add_numbers": add_numbers, "final_int": final_int}, + response_type=int, + ) + + def test_multiple_final_calls_are_rejected(self): + with pytest.raises(ToolCallDecodingError): + self._run( + make_multi_tool_call_response( + [("final_int", '{"x": 5}', "c1"), ("final_int", '{"x": 6}', "c2")] + ), + env={"final_int": final_int}, + response_type=int, + ) + + def test_final_return_type_mismatch_is_rejected(self): + with pytest.raises(ToolCallDecodingError): + self._run( + make_tool_call_response("final_str", '{"x": 5}', "c1"), + env={"final_str": final_str}, + response_type=int, + ) + + +def make_submit_solution_response( + module_code: str, tool_call_id: str = "call_1" +) -> ModelResponse: + """A tool-call response in which the model finalizes by calling the + synthesis ``submit_solution`` FinalTool with a function it wrote.""" + return make_tool_call_response( + "submit_solution", + json.dumps({"implementation": {"module_code": module_code}}), + tool_call_id=tool_call_id, + ) + + +@Template.define +def double_it(x: int) -> int: + """Return double the integer {x}.""" + raise NotHandled + + +class _Doubler(Agent): + @Template.define + def double(self, x: int) -> int: + """Return double the integer {x}.""" + raise NotHandled + + +class TestSynthesizeAndCall: + """Tests for the SynthesizeAndCall handler, which answers a Template by + exposing a FinalTool that the model calls with a synthesized function; the + function is applied to the original arguments and its value is the result.""" + + def test_returns_called_result(self): + """The Template result is the value of applying the synthesized function + to the original arguments, not the function itself.""" + mock = MockCompletionHandler( + [ + make_submit_solution_response( + "def double_it(x: int) -> int:\n return x * 2\n" + ) + ] + ) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + result = double_it(21) + + assert result == 42 + assert mock.call_count == 1 + + def test_value_recorded_as_tool_message(self): + """The computed value enters history as a tool result, and is never + fabricated as assistant content.""" + agent = _Doubler() + mock = MockCompletionHandler( + [ + make_submit_solution_response( + "def double(x: int) -> int:\n return x * 2\n" + ) + ] + ) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + result = agent.double(21) + + assert result == 42 + messages = list(agent.__history__.values()) + tool_messages = [m for m in messages if m["role"] == "tool"] + assert tool_messages, "computed value should be recorded as a tool result" + assert "42" in str(tool_messages[-1]["content"]) + # The model never generated the value itself. + assistant_messages = [m for m in messages if m["role"] == "assistant"] + assert all("42" not in str(m.get("content") or "") for m in assistant_messages) + + def test_direct_structured_answer_is_allowed(self): + """The synthesis tool is offered alongside, not instead of, direct + structured output: the model may answer the return type directly.""" + mock = MockCompletionHandler([make_text_response(json.dumps({"value": 99}))]) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + result = double_it(21) + + assert result == 99 + assert mock.call_count == 1 + + def test_retries_on_runtime_error(self): + """A synthesized function that raises when applied to the inputs surfaces + as a ToolCallExecutionError; RetryLLMHandler feeds the error back and the + loop continues so the model can revise.""" + mock = MockCompletionHandler( + [ + make_submit_solution_response( + "def double_it(x: int) -> int:\n return x // 0\n", + tool_call_id="call_bad", + ), + make_submit_solution_response( + "def double_it(x: int) -> int:\n return x * 2\n", + tool_call_id="call_good", + ), + ] + ) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + handler(RetryLLMHandler()), + ): + result = double_it(21) + + assert result == 42 + assert mock.call_count == 2 + + def test_normal_tool_calls_do_not_terminate(self): + """A non-final tool call is fed back and the loop continues; only the + FinalTool call terminates.""" + mock = MockCompletionHandler( + [ + make_tool_call_response("add_numbers", '{"a": 1, "b": 2}'), + make_submit_solution_response( + "def double_it(x: int) -> int:\n return x * 2\n" + ), + ] + ) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + # add_numbers is in scope as a lexical tool + result = double_it(21) + + assert result == 42 + assert mock.call_count == 2 + + def test_rejects_variadic_parameters(self): + """A signature with *args/**kwargs cannot be expressed as a Callable type, + so building the synthesis tool for it is rejected.""" + sig = inspect.Signature( + [ + inspect.Parameter( + "args", inspect.Parameter.VAR_POSITIONAL, annotation=int + ) + ], + return_annotation=int, + ) + with pytest.raises(TypeError, match="variadic"): + _synthesis_final_tool(sig, {}) + + class TestMessageSequence: """Tests for MessageSequence message sequence tracking.""" @@ -1616,7 +1882,7 @@ def test_call_tool_success_does_not_raise(self): bound_args = sig.bind(a=3, b=4) tc = DecodedToolCall(add_numbers, bound_args, "call_ok", "add_numbers") - result = call_tool(tc) + result, _, _ = call_tool(tc) assert result["role"] == "tool" assert result["tool_call_id"] == "call_ok" @@ -1630,7 +1896,7 @@ def body(exec_code): pydantic.TypeAdapter(Encodable[CodeType]).validate_python("1 / 0") ) tc = DecodedToolCall(exec_code, bound_args, "call_exec", "exec_code") - return call_tool(tc) + return call_tool(tc)[0] msg = _drive_repl(body) assert msg["role"] == "tool" @@ -1648,7 +1914,7 @@ def test_matching_error_returns_feedback_message(self): tc = DecodedToolCall(flaky_tool, bound_args, "call_match", "flaky_tool") with handler(RetryLLMHandler(catch_tool_errors=ConnectionError)): - result = call_tool(tc) + result, _, _ = call_tool(tc) assert result["role"] == "tool" assert result["tool_call_id"] == "call_match" @@ -1677,7 +1943,7 @@ def test_default_catch_all_catches_everything(self): ) with handler(RetryLLMHandler()): - result = call_tool(tc) + result, _, _ = call_tool(tc) assert result["role"] == "tool" assert "Tool execution failed" in result["content"] @@ -1693,7 +1959,7 @@ def test_tuple_of_error_types(self): catch_tool_errors=(ConnectionError, ValueError), ) ): - result = call_tool(tc) + result, _, _ = call_tool(tc) assert result["role"] == "tool" assert "Tool execution failed" in result["content"] From 1eaa68503bb2b6d0eec444481acb489de8013fcf Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 19 Jun 2026 17:15:27 -0400 Subject: [PATCH 013/155] doctest --- effectful/handlers/llm/completions.py | 56 +++++----- effectful/handlers/llm/encoding.py | 42 ++++++-- effectful/handlers/llm/evaluation.py | 50 ++++++++- effectful/handlers/llm/template.py | 55 +++++++++- tests/test_handlers_llm_evaluation.py | 135 +++++++++++++++++++++++ tests/test_handlers_llm_provider.py | 148 ++++++++++++++++++++++++-- tests/test_handlers_llm_template.py | 100 +++++++++++++++++ 7 files changed, 540 insertions(+), 46 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 2bb9e085d..3b4e3c166 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -28,6 +28,7 @@ from effectful.handlers.llm.encoding import ( DecodedToolCall, Encodable, + _SynthesisSpec, to_content_blocks, ) from effectful.handlers.llm.evaluation import ReplSession @@ -272,17 +273,10 @@ def _collect( continue try: result[name] = _LexicalVariableTool.define(obj, name=name) - # `TypeError` joins the three Pydantic errors because the - # `Encodable[T]` registry raises `TypeError` to signal - # "no schema possible" — e.g. `_pydantic_type_operation`, - # `_pydantic_type_term`, and `_pydantic_callable`'s - # incomplete-signature path. Same intent as the Pydantic - # cases, different exception class. except ( pydantic.errors.PydanticSchemaGenerationError, pydantic.errors.PydanticInvalidForJsonSchema, pydantic.errors.PydanticUserError, - TypeError, ): continue return result @@ -357,20 +351,24 @@ def _collect( @Operation.define -def _synthesis_signature() -> inspect.Signature | None: - """Return the signature of the in-flight Template call, or ``None``. +def _synthesis_template() -> Template | None: + """Return the in-flight Template being answered by synthesis, or ``None``. `SynthesizeAndCall` installs a fresh handler for this inside each `Template.__apply__` (mirroring `_repl_session`), giving it a lifetime of exactly one Template call. Outside such a scope there is no synthesis target, so this falls back to ``None`` -- e.g. when tools are listed outside a Template call -- and no synthesis tool is injected. + + `collect_tools` uses it to build the synthesis tool from the Template's + signature, and to bind the Template onto the synthesized function's type (see + :class:`~effectful.handlers.llm.encoding._SynthesisSpec`). """ return None def _synthesis_final_tool( - signature: inspect.Signature, + template: Template, env: collections.abc.Mapping[str, typing.Any], name: str = "submit_solution", ) -> FinalTool: @@ -382,6 +380,7 @@ def _synthesis_final_tool( returning the value. Because it is a :class:`FinalTool`, calling it terminates the completion loop and its return value is the Template's result. """ + signature = template.__signature__ param_types = [] for pname, param in signature.parameters.items(): if param.kind in ( @@ -406,6 +405,7 @@ def _synthesis_final_tool( ) callable_type = collections.abc.Callable[param_types, return_type] # type: ignore[valid-type] + callable_type = typing.Annotated[callable_type, _SynthesisSpec(template)] # type: ignore # Recover the original arguments from `env` by name, respecting each # parameter's kind so positional-only and keyword-only parameters bind @@ -442,7 +442,7 @@ def submit_solution(implementation): ) submit_solution.__annotations__ = { "implementation": callable_type, - "return": signature.return_annotation, + "return": return_type, } return FinalTool.define(submit_solution) @@ -471,11 +471,11 @@ class SynthesizeAndCall(ObjectInterpretation): real function before it is applied. Scoping mirrors :class:`PythonRepl`: this handles `Template.__apply__` to - introduce a fresh `_synthesis_signature` handler bound to that call's - signature, and handles `collect_tools` to inject the synthesis tool built - from it. The synthesis target is therefore introduced and eliminated by its - own handler, bounded to the Template call by construction -- nested Template - calls get their own target. + introduce a fresh `_synthesis_template` handler bound to that call's Template, + and handles `collect_tools` to inject the synthesis tool built from it. The + synthesis target is therefore introduced and eliminated by its own handler, + bounded to the Template call by construction -- nested Template calls get + their own target. Failures compose with :class:`RetryLLMHandler`: a function that fails to synthesize surfaces as a :class:`ToolCallDecodingError`, and one that raises @@ -498,10 +498,7 @@ class SynthesizeAndCall(ObjectInterpretation): def _apply[**P, T]( self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs ) -> T: - # Bind the synthesis target to this Template call's signature for the - # duration of the call, so `collect_tools` can build the tool from it. - signature = template.__signature__ - with handler({_synthesis_signature: lambda: signature}): + with handler({_synthesis_template: lambda: template}): return fwd() @implements(collect_tools) @@ -509,9 +506,9 @@ def _collect( self, env: collections.abc.Mapping[str, typing.Any] ) -> collections.abc.Mapping[str, Tool]: tools = dict(fwd()) - signature = _synthesis_signature() - if signature is not None: - final_tool = _synthesis_final_tool(signature, env) + template = _synthesis_template() + if template is not None: + final_tool = _synthesis_final_tool(template, env) tools[final_tool.__name__] = final_tool return tools @@ -553,7 +550,13 @@ def call_assistant[T]( ResultDecodingError: If the result cannot be decoded. The error includes the raw assistant message for retry handling. """ - tools = dict(collect_tools(env)) + tools = collect_tools(env) + # Decode tool calls (and the code synthesized for them) against the lexical + # context, so they resolve names from the Template's scope. + env = collections.ChainMap( + typing.cast("collections.abc.MutableMapping[str, typing.Any]", env), + typing.cast("collections.abc.MutableMapping[str, typing.Any]", tools), + ) tool_specs = { k: typing.cast( pydantic.TypeAdapter[typing.Any], @@ -594,7 +597,7 @@ def call_assistant[T]( ) for raw_tool_call in raw_tool_calls: try: - tool_calls += [encoding.validate_python(raw_tool_call, context=tools)] + tool_calls += [encoding.validate_python(raw_tool_call, context=env)] if isinstance(tool_calls[-1].tool, FinalTool): if not ( tool_calls[-1].result_type == response_type @@ -607,8 +610,7 @@ def call_assistant[T]( if len(raw_tool_calls) > 1: raise TypeError( f"A FinalTool call must be the only tool call in its turn, but " - f"{len(raw_tool_calls)} tool calls were requested " - f"({sum(isinstance(encoding.validate_python(tc, context=tools).tool, FinalTool) for tc in raw_tool_calls)} of them final). Call the final tool alone." + f"{len(raw_tool_calls)} tool calls were requested." ) except Exception as e: raise ToolCallDecodingError( diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 7878e2a09..892068112 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -33,8 +33,9 @@ from PIL import Image import effectful.handlers.llm.evaluation as evaluation -from effectful.handlers.llm.template import Tool +from effectful.handlers.llm.template import Template, Tool from effectful.internals.unification import GenericAlias, TypeEvaluator, nested_type +from effectful.ops.semantics import handler from effectful.ops.types import Operation, Term type ToolCallID = str @@ -122,6 +123,11 @@ def __class_getitem__(cls, item): return TypeToPydanticType().evaluate(item) +@dataclasses.dataclass(frozen=True) +class _SynthesisSpec[T]: + template: Template[..., T] + + class TypeToPydanticType(TypeEvaluator): """Substitute custom types with their Pydantic Annotated equivalents. @@ -143,6 +149,14 @@ def register(cls, *args, **kwargs): return cls._registry.register(*args, **kwargs) def evaluate(self, ty): + if typing.get_origin(ty) is typing.Annotated and any( + isinstance(m, _SynthesisSpec) for m in ty.__metadata__ + ): + inner, *meta = typing.get_args(ty) + return self._registry.dispatch(typing.get_origin(inner) or inner)( + inner, *meta + ) + app = super().evaluate(ty) origin = typing.get_origin(app) # Only dispatch on regular types. Special forms (Literal, Annotated, @@ -351,12 +365,16 @@ def _serialize(value, info: pydantic.SerializationInfo): @TypeToPydanticType.register(Term) def _pydantic_type_term(ty: type[Term]): - raise TypeError("Terms cannot be converted to Pydantic types.") + raise pydantic.errors.PydanticSchemaGenerationError( + "Terms cannot be converted to Pydantic types." + ) @TypeToPydanticType.register(Operation) def _pydantic_type_operation(ty: type[Operation]): - raise TypeError("Operations cannot be converted to Pydantic types.") + raise pydantic.errors.PydanticSchemaGenerationError( + "Operations cannot be converted to Pydantic types." + ) @pydantic.validate_call(validate_return=False) @@ -437,7 +455,10 @@ def _create_typed_synthesized_function( 1. Produce one block of Python code. 2. The function MUST have type annotations for all parameters and the return type. 3. The function definition must be the LAST statement - do not add any code after it. -4. Do not include usage examples or function calls. +4. You may include doctest examples (lines starting with >>>) inside the function's + docstring to demonstrate and verify its behavior; these examples are run as tests. +5. Do not add any executable code after the function definition (the doctest examples + in the docstring are the only usage examples allowed). """ @@ -492,7 +513,9 @@ def _validate_signature_callable( @TypeToPydanticType.register(Callable) -def _pydantic_callable(callable_type: Any) -> Any: +def _pydantic_callable( + callable_type: Any, metadata: _SynthesisSpec | None = None +) -> Any: """Create a Pydantic-compatible Annotated type for a parameterized Callable. Usage: PydanticCallable(Callable[[int, str], bool]) @@ -505,7 +528,7 @@ def _pydantic_callable(callable_type: Any) -> Any: expected_return = None else: if len(type_args) < 2: - raise TypeError( + raise pydantic.errors.PydanticSchemaGenerationError( f"Callable type signature incomplete: {callable_type}. " "Expected Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType]." ) @@ -575,6 +598,13 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: ) _validate_signature_callable(result, expected_params, expected_return) + + if metadata is not None: + result = functools.wraps(metadata.template)( + handler({metadata.template: result})(result) + ) + g.update({metadata.template.__name__: result}) + evaluation.run_doctests(result, g) return result def _serialize(value: Callable) -> dict: diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index 17129729d..93c59e1ce 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -5,6 +5,7 @@ import collections.abc import contextlib import copy +import doctest import inspect import io import keyword @@ -71,6 +72,26 @@ def type_check( ) +@defop +def run_doctests( + obj: collections.abc.Callable | type | types.ModuleType, + globs: typing.Mapping[str, Any], +) -> None: + """Run the doctests found in a synthesized object's docstring. + + obj: The synthesized object (typically a function) whose docstring may + contain interactive ``>>>`` examples. + globs: The namespace the examples execute in (typically the exec namespace, + which already contains the function plus its lexical context). + + Returns None, raises TypeError if any doctest example fails. A docstring + with no examples is a no-op (passes trivially). + """ + raise NotImplementedError( + "An eval provider must be installed in order to run doctests." + ) + + @defop def compile(module: ast.Module, filename: str) -> CodeType: """ @@ -686,9 +707,6 @@ def mypy_type_check( return None -# Eval Providers - - class UnsafeEvalProvider(ObjectInterpretation): """UNSAFE provider that handles parse, comple and exec operations by shelling out to python *without* any further checks. Only use for testing.""" @@ -732,6 +750,32 @@ def exec( # Execute module-style so top-level defs land in `env`. builtins.exec(bytecode, env, env) + @implements(run_doctests) + def run_doctests( + self, + obj: collections.abc.Callable | type | types.ModuleType, + globs: typing.Mapping[str, Any], + ) -> None: + assert hasattr(obj, "__name__") + name = obj.__name__ + finder = doctest.DocTestFinder(recurse=False) + runner = doctest.DocTestRunner(verbose=False) + # Collect each example's want/got report via `out=...` and read failure + # counts from `run`'s return value, avoiding `summarize`, which would print + # to stdout instead of returning the report. + output: list[str] = [] + failed = attempted = 0 + for test in finder.find(obj, name=name, globs=dict(globs)): + results = runner.run(test, out=output.append) + failed += results.failed + attempted += results.attempted + if failed: + report = "".join(output).strip() + if not report: + report = f"{failed} doctest(s) failed out of {attempted} attempted." + raise TypeError(f"doctest failed:\n{report}") + return None + class _StdoutPrintCollector(PrintCollector): """`_print_` factory whose `print(...)` writes to the real `sys.stdout` diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 8171440e0..dc4533b0f 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -1,4 +1,5 @@ import abc +import doctest import functools import inspect import re @@ -202,6 +203,52 @@ class Template[**P, T](Tool[P, T]): __context__: ChainMap[str, Any] __system_prompt__: str + @classmethod + def _validate_doctests_constant(cls, template: "Template", doc: str) -> None: + """Validate that no format string variables are spliced into doctests. + + The whole docstring is ``str.format``-ed into the prompt at call time, + so an active replacement field inside a ``>>>`` example would be + substituted, breaking the example. Doctests must therefore be constant: + the example source, expected output and exception message may contain + only escaped braces (``{{``/``}}``), never active fields. + + :raises TypeError: If any doctest example contains an active field. + """ + try: + parts = doctest.DocTestParser().parse(doc, template.__name__) + except ValueError: + # Malformed doctest -- not a prompt-field concern; it surfaces when + # the doctests are actually run, so skip the constancy check here. + return + + formatter = string.Formatter() + spliced: list[str] = [] + for part in parts: + if not isinstance(part, doctest.Example): + continue + for text in (part.source, part.want, part.exc_msg or ""): + try: + spliced.extend( + field_name + for _, field_name, _, _ in formatter.parse(text) + if field_name is not None + ) + except ValueError: + # An unbalanced brace (e.g. a bare ``{`` or ``}``) is also + # non-constant: ``str.format`` would reject it at call time. + spliced.append("") + + if spliced: + # Render the auto-numbered empty field ``{}`` readably. + shown = sorted({f or "{}" for f in spliced}) + raise TypeError( + f"Template '{template.__name__}' splices {shown} " + f"into a doctest example. Doctests must be constant -- they are " + f"formatted into the prompt at call time, so they may not contain " + f"format fields. Escape literal braces as '{{{{' and '}}}}'." + ) + @classmethod def _validate_prompt( cls, @@ -212,11 +259,15 @@ def _validate_prompt( refer to names resolvable at call time. Each variable must be either a parameter in the signature - or a name captured in the lexical context. + or a name captured in the lexical context. Additionally, doctest + examples in the docstring must be constant (see + :meth:`_validate_doctests_constant`). - :raises TypeError: If any format string variable cannot be resolved. + :raises TypeError: If any format string variable cannot be resolved, or + a format field is spliced into a doctest example. """ doc = template.__prompt_template__ + cls._validate_doctests_constant(template, doc) formatter = string.Formatter() param_names = set(template.__signature__.parameters.keys()) context_keys = set(context.keys()) diff --git a/tests/test_handlers_llm_evaluation.py b/tests/test_handlers_llm_evaluation.py index deac39e97..0165cce4c 100644 --- a/tests/test_handlers_llm_evaluation.py +++ b/tests/test_handlers_llm_evaluation.py @@ -27,6 +27,7 @@ collect_runtime_type_stubs, collect_variable_declarations, mypy_type_check, + run_doctests, type_to_ast, ) from effectful.handlers.llm.evaluation import compile as compile_op @@ -1755,3 +1756,137 @@ def test_restricted_exec_print_captured_to_stdout(): output-capturing callers see it (rather than NameError on `_print_`).""" out = _restricted_run("print('hi')", {}, capture=True) assert out == "hi\n" + + +class TestRunDoctests: + """Doctest validation stage for synthesized functions (#433).""" + + def test_passing_doctests_pass(self): + def count_char(s: str, c: str) -> int: + """Count occurrences of ``c`` in ``s``. + + >>> count_char("hello", "l") + 2 + >>> count_char("", "x") + 0 + """ + return s.count(c) + + # No exception means the doctests passed. + run_doctests(count_char, {"count_char": count_char}) + + def test_failing_doctest_raises_with_report(self): + def count_char(s: str, c: str) -> int: + """Wrong expected output. + + >>> count_char("hello", "l") + 99 + """ + return s.count(c) + + with pytest.raises(TypeError) as exc: + run_doctests(count_char, {"count_char": count_char}) + msg = str(exc.value) + assert "doctest failed" in msg + assert "Expected:" in msg and "99" in msg + + def test_no_doctests_is_noop(self): + def plain(x: int) -> int: + """No interactive examples here.""" + return x + + run_doctests(plain, {"plain": plain}) + + def test_no_docstring_is_noop(self): + def nodoc(x: int) -> int: + return x + + run_doctests(nodoc, {"nodoc": nodoc}) + + def test_doctest_uses_globs(self): + offset = 10 + + def add_offset(x: int) -> int: + """Add the captured ``offset`` to ``x``. + + >>> add_offset(5) + 15 + """ + return x + offset + + # `offset` resolves from globs, not the example's literals. + run_doctests(add_offset, {"add_offset": add_offset, "offset": offset}) + + def test_op_runs_via_default_rule_without_provider(self): + # `run_doctests` carries its mechanics in its default rule, so it works + # with no eval provider installed (wrappers like SynthesizeAndCall still + # always enclose it). + def f(x: int) -> int: + """Identity. + + >>> f(1) + 1 + """ + return x + + run_doctests(f, {"f": f}) # passes + + def g(x: int) -> int: + """Wrong. + + >>> g(1) + 2 + """ + return x + + with pytest.raises(TypeError, match="doctest failed"): + run_doctests(g, {"g": g}) + + +class TestRunDoctestsThroughCallableDecode: + """`Encodable[Callable[...]]` runs the synthesized function's doctests.""" + + def _decode(self, module_code: str, provider=None): + provider = provider or UnsafeEvalProvider() + with handler(provider): + return pydantic.TypeAdapter( + Encodable[Callable[[str, str], int]] + ).validate_python(SynthesizedFunction(module_code=module_code), context={}) + + def test_decode_runs_passing_doctests(self): + fn = self._decode( + "def count_char(s: str, c: str) -> int:\n" + ' """Count occurrences.\n' + "\n" + ' >>> count_char("hello", "l")\n' + " 2\n" + ' """\n' + " return s.count(c)\n" + ) + assert fn("banana", "a") == 3 + + def test_decode_rejects_failing_doctests(self): + with pytest.raises(Exception) as exc: + self._decode( + "def count_char(s: str, c: str) -> int:\n" + ' """Count occurrences.\n' + "\n" + ' >>> count_char("hello", "l")\n' + " 99\n" + ' """\n' + " return s.count(c)\n" + ) + assert "doctest failed" in str(exc.value) + + def test_decode_restricted_provider(self): + fn = self._decode( + "def count_char(s: str, c: str) -> int:\n" + ' """Count occurrences.\n' + "\n" + ' >>> count_char("hello", "l")\n' + " 2\n" + ' """\n' + " return s.count(c)\n", + provider=RestrictedEvalProvider(), + ) + assert fn("hello", "l") == 2 diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index 5380c46a1..bee5cc644 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -41,6 +41,7 @@ ToolCallExecutionError, _get_history, _synthesis_final_tool, + _synthesis_template, call_assistant, call_tool, collect_tools, @@ -1556,16 +1557,147 @@ def test_normal_tool_calls_do_not_terminate(self): def test_rejects_variadic_parameters(self): """A signature with *args/**kwargs cannot be expressed as a Callable type, so building the synthesis tool for it is rejected.""" - sig = inspect.Signature( + + @Template.define + def variadic(*args: int) -> int: + """Sum the arguments.""" + raise NotHandled + + with pytest.raises(TypeError, match="variadic"): + _synthesis_final_tool(variadic, {}) + + +class TestSynthesizeAndCallDoctests: + """SynthesizeAndCall validates the synthesized function against the + Template's own docstring doctests (#433), rerouting Template calls in the + doctests to the synthesized function so they never re-synthesize. + + The doctest-bearing Templates are defined *inside* each test rather than at + module scope so pytest's ``--doctest-modules`` collection does not try to run + them (they reference a Template that needs an LLM to resolve).""" + + def test_passes_when_synthesized_function_meets_template_doctests(self): + # The Template's docstring carries the doctests; the synthesized + # function's OWN docstring is deliberately wrong to prove the Template's + # docstring is what gets run. + @Template.define + def triple_it(x: int) -> int: + """Return triple the integer {x}. + + >>> triple_it(2) + 6 + >>> triple_it(0) + 0 + """ + raise NotHandled + + good = ( + "def impl(x: int) -> int:\n" + ' """>>> triple_it(2)\n' + " 999\n" + ' """\n' + " return x * 3\n" + ) + mock = MockCompletionHandler([make_submit_solution_response(good)]) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + result = triple_it(2) + + assert result == 6 + # A single completion: the doctest's `triple_it(...)` calls dispatched to + # the synthesized function, never re-entering synthesis. + assert mock.call_count == 1 + + def test_rejects_then_retries_when_doctests_fail(self): + @Template.define + def triple_it(x: int) -> int: + """Return triple the integer {x}. + + >>> triple_it(2) + 6 + """ + raise NotHandled + + bad = "def impl(x: int) -> int:\n return x * 2\n" # doubles, not triples + good = "def impl(x: int) -> int:\n return x * 3\n" + mock = MockCompletionHandler( [ - inspect.Parameter( - "args", inspect.Parameter.VAR_POSITIONAL, annotation=int - ) - ], - return_annotation=int, + make_submit_solution_response(bad, tool_call_id="bad"), + make_submit_solution_response(good, tool_call_id="good"), + ] ) - with pytest.raises(TypeError, match="variadic"): - _synthesis_final_tool(sig, {}) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + handler(RetryLLMHandler()), + ): + result = triple_it(2) + + assert result == 6 + assert mock.call_count == 2 + + def test_template_without_doctests_synthesizes_normally(self): + @Template.define + def triple_it(x: int) -> int: + """Return triple the integer {x}.""" + raise NotHandled + + good = "def impl(x: int) -> int:\n return x * 3\n" + mock = MockCompletionHandler([make_submit_solution_response(good)]) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + result = triple_it(2) + + assert result == 6 + assert mock.call_count == 1 + + def test_run_doctests_not_globally_hijacked_during_synthesis(self): + """The name/docstring/doctest behavior is local to the synthesis argument + (carried by _SynthesisSpec on its type), not a global run_doctests + override. So a *separate* Encodable[Callable] decode validates against its + OWN docstring even while a doctest-bearing Template is the synthesis + target -- under the old override it would have had the Template's + docstring spliced in and failed.""" + + @Template.define + def triple_it(x: int) -> int: + """Return triple {x}. + + >>> triple_it(2) + 6 + """ + raise NotHandled + + # A plain synthesized function whose OWN doctest passes but which doubles + # (so it would fail triple_it's spliced-in doctest under the old code). + module_code = ( + "def double(x: int) -> int:\n" + ' """Double x.\n' + "\n" + " >>> double(2)\n" + " 4\n" + ' """\n' + " return x * 2\n" + ) + with ( + handler(UnsafeEvalProvider()), + handler({_synthesis_template: lambda: triple_it}), + ): + f = pydantic.TypeAdapter(Encodable[Callable[[int], int]]).validate_python( + {"module_code": module_code}, context={} + ) + + assert f(3) == 6 class TestMessageSequence: diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index e714cf7db..cc16b35e9 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1515,6 +1515,106 @@ def bad(x: int) -> str: raise NotHandled +# --------------------------------------------------------------------------- +# Doctests in a Template docstring must be constant (no spliced arguments). +# Templates are defined *inside* each test so pytest's --doctest-modules does +# not try to collect/run these docstring examples. +# --------------------------------------------------------------------------- + + +def test_validate_constant_doctest_ok(): + """A doctest with no format fields is accepted.""" + + @Template.define + def dbl(x: int) -> int: + """Double {x}. + + >>> dbl(2) + 4 + """ + raise NotHandled + + assert "dbl(2)" in dbl.__prompt_template__ + + +def test_validate_param_spliced_into_doctest_source_rejected(): + """A parameter spliced into the doctest source is rejected at define time.""" + with pytest.raises(TypeError, match="constant") as exc: + + @Template.define + def dbl(x: int) -> int: + """Double {x}. + + >>> dbl({x}) + 4 + """ + raise NotHandled + + assert "'x'" in str(exc.value) + + +def test_validate_field_spliced_into_doctest_want_rejected(): + """A field spliced into the expected output is rejected.""" + with pytest.raises(TypeError, match="constant"): + + @Template.define + def dbl(x: int) -> int: + """Double {x}. + + >>> dbl(2) + {x} + """ + raise NotHandled + + +def test_validate_bare_braces_in_doctest_rejected(): + """A bare ``{}`` in a doctest is non-constant (str.format treats it as a + positional field) and is rejected.""" + with pytest.raises(TypeError, match="constant"): + + @Template.define + def dbl(x: int) -> int: + """Double {x}. + + >>> d = {} + >>> dbl(2) + 4 + """ + raise NotHandled + + +def test_validate_escaped_braces_in_doctest_ok(): + """Escaped braces ``{{``/``}}`` format to literal braces, so they are + constant and accepted.""" + + @Template.define + def make_dict(x: int) -> dict: + """Build a dict from {x}. + + >>> d = {{}} + >>> make_dict(2) + {{'k': 2}} + """ + raise NotHandled + + assert "make_dict(2)" in make_dict.__prompt_template__ + + +def test_validate_field_in_prose_with_constant_doctest_ok(): + """Format fields are still allowed in the prose around constant doctests.""" + + @Template.define + def about(theme: str) -> int: + """Count words about {theme}. + + >>> about("cats") + 1 + """ + raise NotHandled + + assert "{theme}" in about.__prompt_template__ + + # Forward ref through Tool subclass of Operation. # Use types Pydantic can serialize (not arbitrary classes) to avoid # PydanticSchemaGenerationError when other tests build tool schemas. From d5129c8db9a4f8130ca50e9a232ab960cbff26f4 Mon Sep 17 00:00:00 2001 From: eb8680 Date: Sat, 20 Jun 2026 13:13:13 -0400 Subject: [PATCH 014/155] Add idiomatic codeadapt example (#695) --- docs/source/codeadapt.py | 272 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 docs/source/codeadapt.py diff --git a/docs/source/codeadapt.py b/docs/source/codeadapt.py new file mode 100644 index 000000000..c2799005c --- /dev/null +++ b/docs/source/codeadapt.py @@ -0,0 +1,272 @@ +"""CodeAdapt: solving hard problems by writing and running Python. + +You are a careful problem solver and an expert Python programmer. You answer by +writing code, not by reasoning in prose alone: problems that are error-prone to +work out by hand are often easy to brute-force or verify with a short program. + +You MUST use the ``submit_solution`` tool to give your final answer, +the harness will not accept a final answer in direct text. + +You can use whatever other tools are available to develop your solution, +and refine incorrect attempts given feedback from failures of ``submit_solution``. +""" + +import argparse +import os +from typing import Literal, NamedTuple + +from tenacity import stop_after_attempt + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import ( + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + + +@Template.define +def least_beautiful_base(threshold: int) -> int: + r"""Find the least integer base b >= 2 for which there are more than + {threshold} ``b``-eautiful integers. + + A positive integer n is ``b``-eautiful if it has exactly two digits when + written in base b and those two digits sum to ``sqrt(n)``. For example, 81 + is 13-eautiful because 81 = 6_3 in base 13 and 6 + 3 = sqrt(81). + + >>> least_beautiful_base(0) + 3 + >>> least_beautiful_base(1) + 7 + >>> least_beautiful_base(5) + 31 + >>> least_beautiful_base(7) + 211 + """ + raise NotHandled + + +class LineupClue(NamedTuple): + """ + A clue about the relative ordering of n people, numbered 0 to n - 1, in a line. + Used to describe puzzles like the classic "zebra puzzle" represented in `solve_lineup`. + Each `LineupClue` corresponds to a single ordering constraint, ``(kind, a, b)``. + + The meaning of ``a`` and ``b`` depends on ``kind``: + + - ``("at", a, k)`` -- person ``a`` is at position ``k`` + - ``("left", a, b)`` -- person ``a`` is somewhere left of person ``b`` + - ``("imm_left", a, b)`` -- person ``a`` is immediately left of person ``b`` + - ``("adj", a, b)`` -- persons ``a`` and ``b`` are in adjacent positions + """ + + kind: Literal["at", "left", "imm_left", "adj"] + a: int + b: int + + +@Template.define +def solve_lineup(n: int, clues: list[LineupClue]) -> tuple[int, ...]: + """Solve a 'zebra'-style ordering puzzle: place n={n} people, numbered 0 to + n - 1, in a line in positions 1 to n (each position used once) so that every + `LineupClue` in the following list holds: + + {clues} + + Every puzzle has exactly one consistent arrangement. Return the tuple of + positions ``(position of 0, position of 1, ..., position of n - 1)``, + as shown in the following worked examples: + + >>> solve_lineup(3, [LineupClue("at", 0, 1), LineupClue("left", 1, 2)]) + (1, 2, 3) + >>> solve_lineup(4, [LineupClue("left", 0, 1), LineupClue("left", 1, 2), LineupClue("left", 2, 3)]) + (1, 2, 3, 4) + >>> solve_lineup(4, [LineupClue("imm_left", 0, 1), LineupClue("at", 2, 4), LineupClue("left", 3, 0)]) + (2, 3, 4, 1) + >>> solve_lineup(5, [LineupClue("at", 0, 3), LineupClue("imm_left", 1, 2), LineupClue("left", 3, 4), LineupClue("at", 4, 5)]) + (3, 1, 2, 4, 5) + """ + raise NotHandled + + +@Template.define +def countdown_reachable(numbers: list[int], target: int) -> bool: + """In the Countdown numbers game, decide whether {target} can be made from + {numbers}, using each number exactly once and combining them with + - * / + (every intermediate division must come out exact). + + >>> countdown_reachable([2, 3, 5], 11) + True + >>> countdown_reachable([1, 1], 5) + False + >>> countdown_reachable([4, 7, 8, 9], 100) + True + >>> countdown_reachable([5, 5, 5], 3) + False + """ + raise NotHandled + + +@Template.define +def constrained_paragraph(endings: list[str]) -> str: + r"""Write a short paragraph whose sentences end, in order, with the words in + {endings}: one sentence per word, each ending with that exact word. + + The examples below split the returned paragraph into sentences and compare the + last word of each (lowercased, punctuation stripped) against the requested + endings -- so a synthesized function must build text with the right shape: + + >>> import re + >>> def endings_of(paragraph): + ... sents = [s for s in re.split(r"(?<=[.!?])\s+", paragraph.strip()) if s] + ... return [re.findall(r"[A-Za-z']+", s)[-1].lower() for s in sents] + >>> endings_of(constrained_paragraph(["walk", "tumbling", "another", "lunatic"])) + ['walk', 'tumbling', 'another', 'lunatic'] + >>> endings_of(constrained_paragraph(["dawn", "river"])) + ['dawn', 'river'] + """ + raise NotHandled + + +@Template.define +def fix_typos(text: str) -> str: + """Output the following text exactly, with no changes at all except for fixing + the misspellings. Leave every other stylistic decision -- commas, US vs British + spellings, capitalization, line breaks -- exactly as in the original: + + {text} + + Only misspelled words may change; every correctly spelled word and all + punctuation and whitespace must be preserved verbatim. Identify the typos, then + apply the corrections with code so that nothing else can drift. + + >>> fix_typos("We inctroduce a probablistic method in the presense of noise.") + 'We introduce a probabilistic method in the presence of noise.' + >>> fix_typos("Teh quick borwn fox jumpps over the lazy dog.") + 'The quick brown fox jumps over the lazy dog.' + """ + raise NotHandled + + +@Template.define +def musr_object_placement( + story: str, person: str, item: str, locations: list[str] +) -> str: + """A MuSR object-placement question: a theory-of-mind puzzle. Read the story + and decide, from {locations}, where {person} would look for the {item}. + + The answer is the last place {person} *saw* the {item}: the last move they + watched, or any later moment they directly saw it somewhere; or its original + location if they never saw it after that. A person's belief does not change + while they are not watching, so where the {item} actually ends up and where + {person} believes it is can differ. + + {story} + + >>> musr_object_placement( + ... "Danny set the earphones in the recording booth, then stepped out for a " + ... "call. While he was gone, Emma quietly moved them to the producer's desk.", + ... "Danny", + ... "earphones", + ... ["recording booth", "producer's desk"], + ... ) + 'recording booth' + """ + raise NotHandled + + +def main( + task: Literal["beautiful", "lineup", "countdown", "paragraph", "typos", "musr"], +) -> None: + if task == "beautiful": + threshold = 10 + print(f"Least b with > {threshold} b-eautiful integers") + print(f"Answer: {least_beautiful_base(threshold)}") + elif task == "lineup": + puzzle = [ + LineupClue("imm_left", 0, 1), + LineupClue("imm_left", 1, 2), + LineupClue("at", 3, 5), + LineupClue("left", 4, 0), + ] + print(f"Zebra-style ordering puzzle: n=5, clues={puzzle}") + print(f"Answer: {solve_lineup(5, puzzle)}") + elif task == "countdown": + numbers, target = [3, 6, 25, 50], 147 + print(f"Countdown: reach {target} from {numbers}") + print(f"Answer: {countdown_reachable(numbers, target)}") + elif task == "paragraph": + endings = ["mountain", "whisper", "thunder"] + print(f"Paragraph with sentences ending in {endings}") + print(f"Answer: {constrained_paragraph(endings)}") + elif task == "typos": + text = ( + "We inctroduce a probablistic algorithm that estimates the " + "timne-varying location in the presense of measurment noise." + ) + print(f"Fix only the typos in:\n{text}") + print(f"Answer: {fix_typos(text)}") + elif task == "musr": + STUDIO_STORY = """\ +In the heart of the bustling studio, Ricky, Emma, and Danny readied themselves \ +for a day of creating magic. Ricky, the gifted singer-songwriter, had his \ +precious notebook of lyrics on the producer's desk. Emma, their producer, was \ +cognizant of the notebook's place at her desk. Across the room, Danny, the studio \ +assistant, kept the earphones in the recording booth. They were all aware of the \ +arrangement -- the notebook on the producer's desk, the earphones in the \ +recording booth. + +Ricky gently places his notebook onto the piano, then becomes engrossed in \ +perfecting his song. Emma, engrossed in her thoughts, deftly moves the earphones \ +to the producer's desk. At that moment Danny was in a stirring conversation with a \ +visiting sound engineer; the visitor stood blocking Danny's general overview of \ +the studio space. + +Later, delicately lifting Ricky's notebook, Danny orchestrates its move to the \ +producer's desk. At the desk, he glimpses a pair of earphones indirectly drawing \ +his attention amidst his routine of tidying up. Meanwhile Emma, from inside a \ +sound-proofed booth, was lost in reviewing already-recorded tracks, out of \ +Danny's view.""" + person = "Danny" + item = "earphones" + locations = ["piano", "producer's desk", "recording booth"] + answer = musr_object_placement(STUDIO_STORY, person, item, locations) + print(f"MuSR: where would {person} look for the {item}?") + print(f"Answer: {answer}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="CodeAdapt: solve hard problems by writing and running code" + ) + parser.add_argument( + "--task", + choices=("beautiful", "lineup", "countdown", "paragraph", "typos", "musr"), + default="beautiful", + help="Which problem to solve", + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required")), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + ): + main(args.task) From 70afc43ac33a56a1e0da76f0703438565a938a29 Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 22 Jun 2026 10:40:34 -0400 Subject: [PATCH 015/155] observability --- docs/source/codeadapt.py | 28 +++++--- effectful/handlers/llm/completions.py | 99 ++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/docs/source/codeadapt.py b/docs/source/codeadapt.py index c2799005c..70046acb4 100644 --- a/docs/source/codeadapt.py +++ b/docs/source/codeadapt.py @@ -12,13 +12,16 @@ """ import argparse +import contextlib import os from typing import Literal, NamedTuple -from tenacity import stop_after_attempt +import tenacity from effectful.handlers.llm import Template from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, LiteLLMProvider, PythonRepl, RetryLLMHandler, @@ -70,25 +73,25 @@ class LineupClue(NamedTuple): @Template.define -def solve_lineup(n: int, clues: list[LineupClue]) -> tuple[int, ...]: +def solve_lineup(n: int, clues: list[LineupClue]) -> list[int]: """Solve a 'zebra'-style ordering puzzle: place n={n} people, numbered 0 to n - 1, in a line in positions 1 to n (each position used once) so that every `LineupClue` in the following list holds: {clues} - Every puzzle has exactly one consistent arrangement. Return the tuple of - positions ``(position of 0, position of 1, ..., position of n - 1)``, + Every puzzle has exactly one consistent arrangement. Return the list of + positions ``[position of 0, position of 1, ..., position of n - 1]``, as shown in the following worked examples: >>> solve_lineup(3, [LineupClue("at", 0, 1), LineupClue("left", 1, 2)]) - (1, 2, 3) + [1, 2, 3] >>> solve_lineup(4, [LineupClue("left", 0, 1), LineupClue("left", 1, 2), LineupClue("left", 2, 3)]) - (1, 2, 3, 4) + [1, 2, 3, 4] >>> solve_lineup(4, [LineupClue("imm_left", 0, 1), LineupClue("at", 2, 4), LineupClue("left", 3, 0)]) - (2, 3, 4, 1) + [2, 3, 4, 1] >>> solve_lineup(5, [LineupClue("at", 0, 3), LineupClue("imm_left", 1, 2), LineupClue("left", 3, 4), LineupClue("at", 4, 5)]) - (3, 1, 2, 4, 5) + [3, 1, 2, 4, 5] """ raise NotHandled @@ -261,12 +264,19 @@ def main( default=5, help="Number of retries for malformed/failing LLM output", ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) args = parser.parse_args() with ( handler(LiteLLMProvider(model=args.model, tool_choice="required")), handler(UnsafeEvalProvider()), handler(PythonRepl()), handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), ): main(args.task) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 3b4e3c166..eb869aac1 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -11,6 +11,7 @@ import typing import uuid +import langfuse import litellm import pydantic import tenacity @@ -269,7 +270,7 @@ def _collect( ) -> collections.abc.Mapping[str, Tool]: result = dict(fwd()) for name, obj in env.items(): - if name in result or not name.isidentifier(): + if name in result or not name.isidentifier() or isinstance(obj, Tool): continue try: result[name] = _LexicalVariableTool.define(obj, name=name) @@ -889,3 +890,99 @@ def _call[**P, T]( history.clear() history.update(history_copy) return typing.cast(T, result) + + +@dataclasses.dataclass(frozen=True) +class LangfuseTracer(ObjectInterpretation): + """Traces Tool, Template, and completion calls with Langfuse. + + Compose with a provider via :func:`~effectful.ops.semantics.handler` + to add tracing:: + + with handler(provider), handler(LangfuseTracer()): + print(limerick(theme)) + """ + + client: langfuse.Langfuse = dataclasses.field(default_factory=langfuse.get_client) + + @implements(completion) + def completion(self, model, *args, **kwargs): + messages = kwargs.get("messages") + if kwargs.get("tools") is not None: + gen_input = {"tools": kwargs["tools"], "messages": messages} + else: + gen_input = messages + + model_parameters = { + k: kwargs[k] + for k in ("tool_choice", "temperature", "max_tokens", "top_p") + if kwargs.get(k) is not None + } + metadata = {} + response_format = kwargs.get("response_format") + if response_format is not None: + metadata["response_format"] = ( + response_format.model_json_schema() + if isinstance(response_format, type) + and issubclass(response_format, pydantic.BaseModel) + else response_format + ) + with self.client.start_as_current_observation( + as_type="generation", + name="completion", + model=model, + input=gen_input, + model_parameters=model_parameters or None, + metadata=metadata or None, + ) as gen: + response = fwd() + usage = getattr(response, "usage", None) + if usage is not None: + gen.update( + usage_details={ + "input": usage.prompt_tokens, + "output": usage.completion_tokens, + "total": usage.total_tokens, + } + ) + gen.update(output=response.choices[0].message) + return response + + @implements(call_tool) + def call_tool(self, tool_call: DecodedToolCall): + input = { + name: pydantic.TypeAdapter( + Encodable[nested_type(value).value] # type: ignore[misc] + ).dump_python(value, mode="json", context={}) + for name, value in tool_call.bound_args.arguments.items() + } + with self.client.start_as_current_observation( + as_type="tool", + name=tool_call.name, + input=input, + metadata={"tool_call_id": tool_call.id}, + ) as obs: + message, result, is_final = fwd() + obs.update(output=message["content"], metadata={"is_final": is_final}) + return message, result, is_final + + @implements(Template.__apply__) + def call_template(self, template: Template, *args, **kwargs): + bound = inspect.signature(template).bind(*args, **kwargs) + bound.apply_defaults() + agent_input = { + name: pydantic.TypeAdapter( + Encodable[nested_type(value).value] # type: ignore[misc] + ).dump_python(value, mode="json", context={}) + for name, value in bound.arguments.items() + } + with self.client.start_as_current_observation( + as_type="agent", name=template.__name__, input=agent_input + ) as obs: + result = fwd() + obs.update( + output=pydantic.TypeAdapter( + Encodable[nested_type(result).value] # type: ignore[misc] + ).dump_python(result, mode="json", context={}) + ) + return result diff --git a/pyproject.toml b/pyproject.toml index d565403f2..7a0a2cfe6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ numpyro = [ "jax<0.10" ] llm = [ + "langfuse", "litellm", "tenacity", "mypy", From 598c6879dc7217ac7e8ab543fb2ba6fb90bde9e7 Mon Sep 17 00:00:00 2001 From: Eli Date: Tue, 23 Jun 2026 17:09:25 -0400 Subject: [PATCH 016/155] Support bound method Templates --- docs/source/codeadapt_agent.py | 128 ++++++++++++++++++++++++++ effectful/handlers/llm/completions.py | 10 +- effectful/handlers/llm/encoding.py | 59 ++++++++++-- tests/test_handlers_llm_provider.py | 50 +++++++++- 4 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 docs/source/codeadapt_agent.py diff --git a/docs/source/codeadapt_agent.py b/docs/source/codeadapt_agent.py new file mode 100644 index 000000000..cdac82623 --- /dev/null +++ b/docs/source/codeadapt_agent.py @@ -0,0 +1,128 @@ +"""CodeAdapt as an Agent: in-context learning across a conversation. + +This is the Agent-method variant of ``codeadapt.py``. Instead of a free +function, the task is a :class:`~effectful.handlers.llm.Template` method on an +:class:`~effectful.handlers.llm.Agent` subclass, so each call accumulates +message history on the instance and the model can take advantage of in-context +learning across calls. + +The synthesized function is a drop-in syntactic replacement for the method body +-- it keeps ``self`` in its signature -- and the worked examples in the method's +docstring are run as doctests against that synthesized function (calls on +freshly constructed agents are rerouted to it rather than re-invoking the model). +""" + +import argparse +import contextlib +import os + +import tenacity + +from effectful.handlers.llm import Agent, Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + + +class CodeAdaptAgent(Agent): + """ + You are a careful problem solver and an expert Python programmer. You answer by + writing code, not by reasoning in prose alone: problems that are error-prone to + work out by hand are often easy to brute-force or verify with a short program. + + You MUST use the ``submit_solution`` tool to give your final answer, + the harness will not accept a final answer in direct text. + + You can use whatever other tools are available to develop your solution, + and refine incorrect attempts given feedback from failures of ``submit_solution``. + """ + + @Template.define + def countdown_reachable(self, numbers: list[int], target: int) -> bool: + """In the Countdown numbers game, decide whether {target} can be made from + {numbers}, using each number exactly once and combining them with + - * / + (every intermediate division must come out exact). + + >>> agent = CodeAdaptAgent() + >>> agent.countdown_reachable([2, 3, 5], 11) + True + >>> agent.countdown_reachable([1, 1], 5) + False + >>> agent.countdown_reachable([4, 7, 8, 9], 100) + True + >>> agent.countdown_reachable([5, 5, 5], 3) + False + """ + raise NotHandled + + +def main(args: argparse.Namespace) -> None: + if args.task == "countdown": + agent = CodeAdaptAgent() + # Fresh examples (none appear in the docstring doctests), each paired with its + # known-correct answer so we can validate the agent's output. + test_examples: list[tuple[list[int], int, bool]] = [ + ([3, 6, 25, 50], 147, True), # (50 - 25) * 6 - 3 + ([1, 2, 3, 4], 24, True), # 1 * 2 * 3 * 4 + ([2, 4, 8], 9, False), # all-even operands can never reach an odd target + ] + for numbers, target, expected in test_examples: + print(f"Testing countdown_reachable({numbers}, {target})...") + answer = agent.countdown_reachable(numbers, target) + status = "OK" if answer == expected else "WRONG" + print( + f"[{status}] reach {target} from {numbers}: {answer} (expected {expected})" + ) + assert answer == expected, ( + f"countdown_reachable({numbers}, {target}) = {answer}, expected {expected}" + ) + else: + raise ValueError(f"Unknown task {args.task}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="CodeAdapt agent: solve reasoning tasks by writing code" + ) + parser.add_argument( + "--task", + choices=("countdown",), + default="countdown", + help="Which problem to solve", + ) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required")), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index eb869aac1..87df479e6 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -8,6 +8,7 @@ import string import textwrap import traceback +import types import typing import uuid @@ -381,7 +382,14 @@ def _synthesis_final_tool( returning the value. Because it is a :class:`FinalTool`, calling it terminates the completion loop and its return value is the Template's result. """ - signature = template.__signature__ + # Synthesize a drop-in syntactic replacement for the Template body, so the + # function carries the Template's full signature -- including `self` for + # Agent-method Templates (whose `__default__` is a bound method). + if isinstance(template.__default__, types.MethodType): + signature = inspect.signature(template.__default__.__func__) + else: + signature = template.__signature__ + param_types = [] for pname, param in signature.parameters.items(): if param.kind in ( diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 892068112..d6be14459 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -35,7 +35,7 @@ import effectful.handlers.llm.evaluation as evaluation from effectful.handlers.llm.template import Template, Tool from effectful.internals.unification import GenericAlias, TypeEvaluator, nested_type -from effectful.ops.semantics import handler +from effectful.ops.semantics import fwd, handler from effectful.ops.types import Operation, Term type ToolCallID = str @@ -127,6 +127,25 @@ def __class_getitem__(cls, item): class _SynthesisSpec[T]: template: Template[..., T] + @property + def _class_template(self) -> Template[..., T] | None: + if isinstance(self.template.__default__, types.MethodType): + return self.template.__default__.__func__.__wrapped__ # type: ignore[attr-defined] + else: + return None + + def _method_instance(self, other: Template) -> Any | None: + """The instance ``op`` is bound to, if ``op`` is this synthesized + Agent-method on *some* instance; otherwise ``None``. + """ + if ( + self._class_template is not None + and _SynthesisSpec(other)._class_template is self._class_template + ): + return other.__default__.__self__ # type: ignore[attr-defined] + else: + return None + class TypeToPydanticType(TypeEvaluator): """Substitute custom types with their Pydantic Annotated equivalents. @@ -600,12 +619,38 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: _validate_signature_callable(result, expected_params, expected_return) if metadata is not None: - result = functools.wraps(metadata.template)( - handler({metadata.template: result})(result) - ) - g.update({metadata.template.__name__: result}) - evaluation.run_doctests(result, g) - return result + if metadata._class_template is not None: + # Agent-method template: doctests build their own instances, so the + # method must route to `synth` on *any* instance (not just the one + # that triggered synthesis). A fresh instance's call dispatches + # through `Template.__apply__`, which we intercept here. + result = functools.wraps(metadata._class_template)(result) + + def _doctest_apply(op, *args, **kwargs): + instance = metadata._method_instance(op) + if instance is None: + return fwd() + return metadata._class_template(instance, *args, **kwargs) + + with handler( + { + Template.__apply__: _doctest_apply, + metadata._class_template: result, + } + ): + evaluation.run_doctests(result, g) + return result + else: + # Free-function template: shadow the global name the doctest calls, + # and route the template op back into `synth` for recursion. + result = functools.wraps(metadata.template)(result) + g.update({metadata.template.__name__: result}) + with handler({metadata.template: result}): + evaluation.run_doctests(result, g) + return result + else: + evaluation.run_doctests(result, g) + return result def _serialize(value: Callable) -> dict: if not callable(value): diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index bee5cc644..13ee9abc3 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -1467,7 +1467,7 @@ def test_value_recorded_as_tool_message(self): mock = MockCompletionHandler( [ make_submit_solution_response( - "def double(x: int) -> int:\n return x * 2\n" + "def double(self, x: int) -> int:\n return x * 2\n" ) ] ) @@ -1699,6 +1699,54 @@ def triple_it(x: int) -> int: assert f(3) == 6 + def test_agent_method_doctests_route_to_synthesized_function(self): + """An Agent-method Template's doctests build their own instances + (``agent = Doubler()``), so each ``agent.double(...)`` call dispatches a + *fresh* per-instance op -- distinct from the one that triggered + synthesis. Matching on the shared class-level template reroutes every + such call to the synthesized function (with the instance passed as + ``self``), so the doctests validate the synthesized code instead of + re-synthesizing or hitting the LLM.""" + + class Doubler(Agent): + @Template.define + def double(self, x: int) -> int: + """Return double the integer {x}. + + >>> agent = Doubler() + >>> agent.double(2) + 4 + >>> agent.double(0) + 0 + """ + raise NotHandled + + # A drop-in syntactic replacement for the method body keeps `self` in the + # signature; the synthesized function's OWN docstring is deliberately + # wrong to prove the Template's docstring is what gets run. + good = ( + "def double(self, x: int) -> int:\n" + ' """>>> never\n' + " run\n" + ' """\n' + " return x * 2\n" + ) + agent = Doubler() + mock = MockCompletionHandler([make_submit_solution_response(good)]) + with ( + handler(LiteLLMProvider(model="test-model")), + handler(SynthesizeAndCall()), + handler(UnsafeEvalProvider()), + handler(mock), + ): + result = agent.double(21) + + assert result == 42 + # A single completion: the doctest's `agent.double(...)` calls on a fresh + # instance were answered by the synthesized function, never re-entering + # synthesis. + assert mock.call_count == 1 + class TestMessageSequence: """Tests for MessageSequence message sequence tracking.""" From ad2e5854103ebb10504cc3dbbb973b7a5b63d3b0 Mon Sep 17 00:00:00 2001 From: Eli Date: Tue, 23 Jun 2026 23:01:01 -0400 Subject: [PATCH 017/155] cleanup --- effectful/handlers/llm/completions.py | 580 ++++++++++---------------- effectful/handlers/llm/encoding.py | 177 +++++--- effectful/handlers/llm/evaluation.py | 2 - 3 files changed, 358 insertions(+), 401 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 87df479e6..0dcaa413a 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -5,8 +5,6 @@ import functools import inspect import json -import string -import textwrap import traceback import types import typing @@ -19,18 +17,18 @@ from litellm import ( ChatCompletionFunctionMessage, ChatCompletionMessageToolCall, - ChatCompletionTextObject, ChatCompletionToolMessage, OpenAIChatCompletionAssistantMessage, OpenAIChatCompletionSystemMessage, OpenAIChatCompletionUserMessage, - OpenAIMessageContentListBlock, ) from effectful.handlers.llm.encoding import ( DecodedToolCall, Encodable, + _callable_type_from_signature, _SynthesisSpec, + format_as_content_blocks, to_content_blocks, ) from effectful.handlers.llm.evaluation import ReplSession @@ -180,47 +178,6 @@ def to_feedback_message(self, include_traceback: bool) -> Message: ) -class _LexicalVariableTool[T](Tool[[], T]): - """A zero-arg `Tool` that returns the captured value of a variable - from a `Template`'s lexical context. - - Tools are constructed fresh each `call_assistant` invocation, so - the reader closes over the snapshot `value` rather than the - surrounding `env` — in-place mutation of a mutable value is still - visible (same object reference), but rebinding the source name is - not. - """ - - @classmethod - def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": - """Construct a synthetic reader Tool that returns `value`. - - Raises if `Encodable[nested_type(value)]` cannot be generated. - The caller is responsible for catching the failure and deciding - whether to skip the symbol. - """ - assert not isinstance(value, Tool), ( - "Tools are real tools and must not be re-wrapped as lexical readers." - ) - typ: typing.Any = nested_type(value).value - # Probe schema generation; raises if `Encodable[typ]` is not implemented. - pydantic.TypeAdapter(Encodable[typ]).json_schema() - - def tool_fn(): - return value - - tool_fn.__name__ = name - tool_fn.__qualname__ = name - tool_fn.__module__ = type(value).__module__ - tool_fn.__doc__ = ( - f"Reads the value of lexical variable `{name}` from the " - f"enclosing scope where this Template was defined. Takes " - f"no arguments; returns the current value." - ) - tool_fn.__annotations__ = {"return": typ} - return super().define(tool_fn) - - @Operation.define def collect_tools( env: collections.abc.Mapping[str, typing.Any], @@ -257,271 +214,6 @@ def collect_tools( return result -class LexicalReaders(ObjectInterpretation): - """Override `collect_tools` to also expose plain values from the - lexical context as zero-argument read-only Tools. Each non-Tool, - non-Template, non-Agent value bound to a valid identifier is - wrapped via `_LexicalVariableTool` if `Encodable[T]` accepts it; - schema-generation failures cause the symbol to be skipped. - """ - - @implements(collect_tools) - def _collect( - self, env: collections.abc.Mapping[str, typing.Any] - ) -> collections.abc.Mapping[str, Tool]: - result = dict(fwd()) - for name, obj in env.items(): - if name in result or not name.isidentifier() or isinstance(obj, Tool): - continue - try: - result[name] = _LexicalVariableTool.define(obj, name=name) - except ( - pydantic.errors.PydanticSchemaGenerationError, - pydantic.errors.PydanticInvalidForJsonSchema, - pydantic.errors.PydanticUserError, - ): - continue - return result - - -@Operation.define -def _repl_session(env: collections.abc.MutableMapping[str, typing.Any]) -> ReplSession: - """Return the REPL session for the current Template call, seeded from `env`. - - `PythonRepl` installs a fresh handler for this inside each `Template.__apply__` - (mirroring how `__history__` is managed), giving the session a lifetime of - exactly one Template call. Outside such a scope there is no managed session, - so this falls back to a fresh one -- e.g. when tools are listed outside a - Template call. - """ - return ReplSession(env) - - -class PythonRepl(ObjectInterpretation): - """Expose a persistent Python session to the LLM as an `exec_code` Tool. - - Off by default; install it where the LLM should be able to run code whose - state (variables, imports, definitions) survives across tool calls within a - single Template invocation. - - Scoping mirrors how `__history__` is managed for Template calls: `PythonRepl` - handles `Template.__apply__` to introduce a fresh `_repl_session` handler for - the duration of the call, and handles `collect_tools` to inject an `exec_code` - Tool routed to that session. The session is therefore introduced and - eliminated by its own handler, bounded to the Template call by construction -- - there is no global registry of sessions, and nested Template calls get their - own isolated sessions. - - The session is seeded from the Template's lexical context and routes execution - through the `parse`/`compile`/`exec` effect operations, so it works under any - installed eval provider (`UnsafeEvalProvider` or `RestrictedEvalProvider`). - """ - - @implements(Template.__apply__) - def _apply[**P, T]( - self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs - ) -> T: - # One session per Template call, created lazily on first use (the call's - # `env`, supplied by `collect_tools`/`exec_code`, seeds it). The - # enclosing `handler(...)` bounds the session's lifetime to this call, so - # nested Template calls introduce their own fresh session. - session: ReplSession | None = None - - def session_for( - env: collections.abc.MutableMapping[str, typing.Any], - ) -> ReplSession: - nonlocal session - if session is None: - session = ReplSession(env) - return session - - with handler({_repl_session: session_for}): - return fwd() - - @implements(collect_tools) - def _collect( - self, env: collections.abc.Mapping[str, typing.Any] - ) -> collections.abc.Mapping[str, Tool]: - tools = dict(fwd()) - # `collect_tools` only promises a `Mapping`, but the per-call `env` is the - # writable `ChainMap` the session splices its shared scope layer into, so - # narrow it for `_repl_session`/`ReplSession`. - tools["exec_code"] = _repl_session( - typing.cast(collections.abc.MutableMapping[str, typing.Any], env) - ).exec_code - return tools - - -@Operation.define -def _synthesis_template() -> Template | None: - """Return the in-flight Template being answered by synthesis, or ``None``. - - `SynthesizeAndCall` installs a fresh handler for this inside each - `Template.__apply__` (mirroring `_repl_session`), giving it a lifetime of - exactly one Template call. Outside such a scope there is no synthesis - target, so this falls back to ``None`` -- e.g. when tools are listed outside - a Template call -- and no synthesis tool is injected. - - `collect_tools` uses it to build the synthesis tool from the Template's - signature, and to bind the Template onto the synthesized function's type (see - :class:`~effectful.handlers.llm.encoding._SynthesisSpec`). - """ - return None - - -def _synthesis_final_tool( - template: Template, - env: collections.abc.Mapping[str, typing.Any], - name: str = "submit_solution", -) -> FinalTool: - """Build a :class:`FinalTool` that finalizes a Template by code synthesis. - - The tool takes one argument -- a function with the Template's signature, - synthesized from the model's code by the existing ``Encodable[Callable[...]]`` - machinery -- and applies it to the original inputs (recovered from ``env``), - returning the value. Because it is a :class:`FinalTool`, calling it - terminates the completion loop and its return value is the Template's result. - """ - # Synthesize a drop-in syntactic replacement for the Template body, so the - # function carries the Template's full signature -- including `self` for - # Agent-method Templates (whose `__default__` is a bound method). - if isinstance(template.__default__, types.MethodType): - signature = inspect.signature(template.__default__.__func__) - else: - signature = template.__signature__ - - param_types = [] - for pname, param in signature.parameters.items(): - if param.kind in ( - inspect.Parameter.VAR_POSITIONAL, - inspect.Parameter.VAR_KEYWORD, - ): - raise TypeError( - f"SynthesizeAndCall cannot synthesize a function for parameter " - f"'{pname}' of kind {param.kind.description}: variadic parameters " - "cannot be expressed as a Callable type signature." - ) - param_types.append( - param.annotation - if param.annotation is not inspect.Parameter.empty - else typing.Any - ) - return_type = signature.return_annotation - if return_type is inspect.Signature.empty: - raise TypeError( - "SynthesizeAndCall requires a return annotation on the Template's " - "signature to construct the synthesis tool's Callable type." - ) - - callable_type = collections.abc.Callable[param_types, return_type] # type: ignore[valid-type] - callable_type = typing.Annotated[callable_type, _SynthesisSpec(template)] # type: ignore - - # Recover the original arguments from `env` by name, respecting each - # parameter's kind so positional-only and keyword-only parameters bind - # correctly (variadic kinds were rejected above). - pos_names = [ - pname - for pname, param in signature.parameters.items() - if param.kind - in ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - ) - ] - kw_names = [ - pname - for pname, param in signature.parameters.items() - if param.kind is inspect.Parameter.KEYWORD_ONLY - ] - - def submit_solution(implementation): - bound = signature.bind( - *(env[pname] for pname in pos_names), - **{pname: env[pname] for pname in kw_names}, - ) - return implementation(*bound.args, **bound.kwargs) - - submit_solution.__name__ = name - submit_solution.__qualname__ = name - submit_solution.__module__ = __name__ - submit_solution.__doc__ = ( - "Submit your final answer as a Python function implementing the task. " - "The function must have the required signature; it is applied to the " - "original inputs and its return value is your final answer." - ) - submit_solution.__annotations__ = { - "implementation": callable_type, - "return": return_type, - } - return FinalTool.define(submit_solution) - - -class SynthesizeAndCall(ObjectInterpretation): - """Answer a Template by synthesizing a function and calling it. - - Instead of asking the LLM to generate an instance of the Template's return - type directly, this handler exposes a :class:`FinalTool` that lets the model - "answer" by writing a Python function with the Template's signature. The - harness applies that function to the original arguments and its return value - becomes the Template's result. This is the declarative "CodeAdapt" workflow: - the LLM writes code implementing the body of the Template rather than - reasoning out the answer itself. - - The synthesis tool is offered *alongside* the Template's normal completion - paths rather than replacing them: across turns the model may freely call any - other tool in scope (their results are fed back as usual), and it may still - answer the return type directly via structured output. The loop terminates - when it either answers directly or calls the synthesis :class:`FinalTool`. - To force the synthesis path, pass ``tool_choice="required"`` (handler config - is forwarded to the model request). The function is synthesized by reusing - the existing ``Callable`` synthesis machinery: the tool's argument is typed - as ``Callable[[params], ret]``, so :func:`call_assistant`'s tool-call - decoding parses, type-checks, compiles and executes the model's code into a - real function before it is applied. - - Scoping mirrors :class:`PythonRepl`: this handles `Template.__apply__` to - introduce a fresh `_synthesis_template` handler bound to that call's Template, - and handles `collect_tools` to inject the synthesis tool built from it. The - synthesis target is therefore introduced and eliminated by its own handler, - bounded to the Template call by construction -- nested Template calls get - their own target. - - Failures compose with :class:`RetryLLMHandler`: a function that fails to - synthesize surfaces as a :class:`ToolCallDecodingError`, and one that raises - when applied to the inputs as a :class:`ToolCallExecutionError`; both are fed - back to the model as a tool message and the loop continues so it can revise:: - - with ( - handler(LiteLLMProvider(model="gpt-5-mini")), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler()), - ): - ... - - Requires an eval provider (e.g. :class:`UnsafeEvalProvider` or - :class:`RestrictedEvalProvider`) to be installed so the synthesized code can - be compiled and executed. - """ - - @implements(Template.__apply__) - def _apply[**P, T]( - self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs - ) -> T: - with handler({_synthesis_template: lambda: template}): - return fwd() - - @implements(collect_tools) - def _collect( - self, env: collections.abc.Mapping[str, typing.Any] - ) -> collections.abc.Mapping[str, Tool]: - tools = dict(fwd()) - template = _synthesis_template() - if template is not None: - final_tool = _synthesis_final_tool(template, env) - tools[final_tool.__name__] = final_tool - return tools - - @Operation.define @functools.wraps(litellm.completion) def completion(*args, **kwargs) -> typing.Any: @@ -690,46 +382,7 @@ def call_user( """ Format a template applied to arguments into a user message. """ - formatter = string.Formatter() - parts: list[OpenAIMessageContentListBlock] = [] - - buf: list[str] = [] - - def flush_text() -> None: - if buf: - parts.append(ChatCompletionTextObject(type="text", text="".join(buf))) - buf.clear() - - for literal, field_name, format_spec, conversion in formatter.parse( - textwrap.dedent(template) - ): - if literal: - buf.append(literal) - - if field_name is None: - continue - - obj, _ = formatter.get_field(field_name, (), env) - encoder: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( - Encodable[nested_type(obj).value] # type: ignore[misc] - ) - encoded_obj = encoder.dump_python(obj, mode="json", context=env) - for part in to_content_blocks(encoded_obj): - if part["type"] == "text": - text = ( - formatter.convert_field(part["text"], conversion) - if conversion - else part["text"] - ) - buf.append(formatter.format_field(text, format_spec or "")) - else: - flush_text() - parts.append(part) - - flush_text() - - # Note: The OpenAI api only seems to accept images in the 'user' role. The - # effect of different roles on the model's response is currently unclear. + parts = format_as_content_blocks(template, env) message = _make_message(dict(role="user", content=parts)) append_message(message) return message @@ -737,13 +390,238 @@ def flush_text() -> None: @Operation.define def call_system(template: Template) -> Message: - """Get system instruction message(s) to prepend to all LLM prompts.""" + """ + Get system instruction message(s) to prepend to all LLM prompts. + """ system_prompt = template.__system_prompt__ or DEFAULT_SYSTEM_PROMPT message = _make_message(dict(role="system", content=system_prompt)) append_message(message, last=False) return message +class _LexicalVariableTool[T](Tool[[], T]): + """A zero-arg `Tool` that returns the captured value of a variable + from a `Template`'s lexical context. + + Tools are constructed fresh each `call_assistant` invocation, so + the reader closes over the snapshot `value` rather than the + surrounding `env` — in-place mutation of a mutable value is still + visible (same object reference), but rebinding the source name is + not. + """ + + @classmethod + def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": + """Construct a synthetic reader Tool that returns `value`. + + Raises if `Encodable[nested_type(value)]` cannot be generated. + The caller is responsible for catching the failure and deciding + whether to skip the symbol. + """ + assert not isinstance(value, Tool), ( + "Tools are real tools and must not be re-wrapped as lexical readers." + ) + typ: typing.Any = nested_type(value).value + # Probe schema generation; raises if `Encodable[typ]` is not implemented. + pydantic.TypeAdapter(Encodable[typ]).json_schema() + + def tool_fn(): + return value + + tool_fn.__name__ = name + tool_fn.__qualname__ = name + tool_fn.__module__ = type(value).__module__ + tool_fn.__doc__ = ( + f"Reads the value of lexical variable `{name}` from the " + f"enclosing scope where this Template was defined. Takes " + f"no arguments; returns the current value." + ) + tool_fn.__annotations__ = {"return": typ} + return super().define(tool_fn) + + +class LexicalReaders(ObjectInterpretation): + """Override `collect_tools` to also expose plain values from the + lexical context as zero-argument read-only Tools. Each non-Tool, + non-Template, non-Agent value bound to a valid identifier is + wrapped via `_LexicalVariableTool` if `Encodable[T]` accepts it; + schema-generation failures cause the symbol to be skipped. + """ + + @implements(collect_tools) + def _collect( + self, env: collections.abc.Mapping[str, typing.Any] + ) -> collections.abc.Mapping[str, Tool]: + result = dict(fwd()) + for name, obj in env.items(): + if name in result or not name.isidentifier() or isinstance(obj, Tool): + continue + try: + result[name] = _LexicalVariableTool.define(obj, name=name) + except ( + pydantic.errors.PydanticSchemaGenerationError, + pydantic.errors.PydanticInvalidForJsonSchema, + pydantic.errors.PydanticUserError, + ): + continue + return result + + +class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): + """A :class:`FinalTool` that finalizes a Template by synthesizing a function. + + The tool takes one argument -- a function with the Template's signature, + synthesized from the model's code by the existing ``Encodable[Callable[...]]`` + machinery -- and applies it to the original inputs (recovered from ``env``), + returning the value. Because it is a :class:`FinalTool`, calling it + terminates the completion loop and its return value is the Template's result. + """ + + @classmethod + def define( + cls, + template: Template[..., T], + bound_args: inspect.BoundArguments, + *, + name: str, + ) -> FinalTool[[collections.abc.Callable[..., T]], T]: + # Synthesize a drop-in syntactic replacement for the Template body, so the + # function carries the Template's full signature -- including `self` for + # Agent-method Templates (whose `__default__` is a bound method). + if isinstance(template.__default__, types.MethodType): + signature = inspect.signature(template.__default__.__func__) + args, kwargs = ( + (template.__default__.__self__,) + bound_args.args, + bound_args.kwargs, + ) + else: + signature = inspect.signature(template) + args, kwargs = bound_args.args, bound_args.kwargs + + callable_type = _callable_type_from_signature(signature) + callable_type = typing.Annotated[callable_type, _SynthesisSpec(template)] # type: ignore + return_type = signature.return_annotation + + def submit_solution(implementation: callable_type) -> return_type: # type: ignore + """ + Submit your final answer as a Python function implementing the task. + The function must have the required signature; it is applied to the + original inputs and its return value is your final answer. + """ + return implementation(*args, **kwargs) # type: ignore + + return super().define(submit_solution, name=name) + + +class SynthesizeAndCall(ObjectInterpretation): + """Answer a Template by synthesizing a function and calling it. + + Instead of asking the LLM to generate an instance of the Template's return + type directly, this handler exposes a :class:`FinalTool` that lets the model + "answer" by writing a Python function with the Template's signature. The + harness applies that function to the original arguments and its return value + becomes the Template's result. This is the declarative "CodeAdapt" workflow: + the LLM writes code implementing the body of the Template rather than + reasoning out the answer itself. + + The synthesis tool is offered *alongside* the Template's normal completion + paths rather than replacing them: across turns the model may freely call any + other tool in scope (their results are fed back as usual), and it may still + answer the return type directly via structured output. The loop terminates + when it either answers directly or calls the synthesis :class:`FinalTool`. + To force the synthesis path, pass ``tool_choice="required"`` (handler config + is forwarded to the model request). The function is synthesized by reusing + the existing ``Callable`` synthesis machinery: the tool's argument is typed + as ``Callable[[params], ret]``, so :func:`call_assistant`'s tool-call + decoding parses, type-checks, compiles and executes the model's code into a + real function before it is applied. + + Failures compose with :class:`RetryLLMHandler`: a function that fails to + synthesize surfaces as a :class:`ToolCallDecodingError`, and one that raises + when applied to the inputs as a :class:`ToolCallExecutionError`; both are fed + back to the model as a tool message and the loop continues so it can revise:: + + with ( + handler(LiteLLMProvider(model="gpt-5-mini")), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler()), + ): + ... + + Requires an eval provider (e.g. :class:`UnsafeEvalProvider` or + :class:`RestrictedEvalProvider`) to be installed so the synthesized code can + be compiled and executed. + """ + + @implements(Template.__apply__) + def _apply[**P, T]( + self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs + ) -> T: + bound_args = template.__signature__.bind(*args, **kwargs) + bound_args.apply_defaults() + tool = _SynthesisFinalTool.define(template, bound_args, name="submit_solution") + with handler({collect_tools: lambda _: {**fwd(), tool.__name__: tool}}): # type: ignore + return fwd() + + +class PythonRepl(ObjectInterpretation): + """Expose a persistent Python session to the LLM as an `exec_code` Tool. + + Off by default; install it where the LLM should be able to run code whose + state (variables, imports, definitions) survives across tool calls within a + single Template invocation. + + Scoping mirrors how `__history__` is managed for Template calls: `PythonRepl` + handles `Template.__apply__` to introduce a fresh `_repl_session` handler for + the duration of the call, and handles `collect_tools` to inject an `exec_code` + Tool routed to that session. The session is therefore introduced and + eliminated by its own handler, bounded to the Template call by construction -- + there is no global registry of sessions, and nested Template calls get their + own isolated sessions. + + The session is seeded from the Template's lexical context and routes execution + through the `parse`/`compile`/`exec` effect operations, so it works under any + installed eval provider (`UnsafeEvalProvider` or `RestrictedEvalProvider`). + """ + + @Tool.define + @functools.wraps(ReplSession.exec_code) + def exec_code(self, code: types.CodeType) -> str: + raise NotImplementedError("No handler") + + @Tool.define + def read_lexical_variable(self, name: str) -> typing.Any: + """ + Read the value of lexical variable ``name`` into the LLM context. + """ + raise NotImplementedError("No handler") + + @implements(Template.__apply__) + def _apply[**P, T]( + self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs + ) -> T: + bound_args = template.__signature__.bind(*args, **kwargs) + bound_args.apply_defaults() + env = collections.ChainMap(bound_args.arguments, template.__context__) + session = ReplSession(env=env) + with handler( + { + self.exec_code: session.exec_code, + self.read_lexical_variable: env.get, + } + ): + return fwd() + + @implements(collect_tools) + def _collect( + self, env: collections.abc.Mapping[str, typing.Any] + ) -> collections.abc.Mapping[str, Tool]: + tools = dict(fwd()) + tools[self.exec_code.__name__] = self.exec_code + tools[self.read_lexical_variable.__name__] = self.read_lexical_variable + return tools + + class RetryLLMHandler(ObjectInterpretation): """Retries LLM requests if tool call or result decoding fails. diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index d6be14459..1a3384e4f 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -1,11 +1,13 @@ import ast import base64 +import collections.abc import dataclasses import functools import inspect import io import json import linecache +import string import textwrap import types import typing @@ -98,6 +100,80 @@ def walk(v: typing.Any) -> None: return blocks +def format_as_content_blocks( + template: str, + env: collections.abc.Mapping[str, typing.Any], +) -> list[OpenAIMessageContentListBlock]: + """ + Format a template applied to arguments into a list of content blocks. + This is similar to str.format() but produces a list of content blocks + instead of a single string, so that non-text content is preserved. + """ + formatter = string.Formatter() + parts: list[OpenAIMessageContentListBlock] = [] + + buf: list[str] = [] + + def flush_text() -> None: + if buf: + parts.append(ChatCompletionTextObject(type="text", text="".join(buf))) + buf.clear() + + for literal, field_name, format_spec, conversion in formatter.parse( + textwrap.dedent(template) + ): + if literal: + buf.append(literal) + + if field_name is None: + continue + + obj, _ = formatter.get_field(field_name, (), env) + encoder: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( + Encodable[nested_type(obj).value] # type: ignore[misc] + ) + encoded_obj = encoder.dump_python(obj, mode="json", context=env) + for part in to_content_blocks(encoded_obj): + if part["type"] == "text": + text = ( + formatter.convert_field(part["text"], conversion) + if conversion + else part["text"] + ) + buf.append(formatter.format_field(text, format_spec or "")) + else: + flush_text() + parts.append(part) + + flush_text() + + return parts + + +def _inline_refs(schema: dict) -> dict: + """Inline ``$ref`` pointers so ``WithJsonSchema`` never emits orphan refs. + + Workaround for https://github.com/pydantic/pydantic/issues/12145 — + Pydantic's ``GenerateJsonSchema`` does not merge user-provided ``$defs`` + into its internal ref map, so any ``$ref`` in a ``WithJsonSchema`` value + causes a ``KeyError`` when the annotated type is composed into a model. + """ + defs = schema.get("$defs", {}) + + def _resolve(obj): + if isinstance(obj, dict): + if "$ref" in obj: + ref_name = obj["$ref"].split("/")[-1] + if ref_name in defs: + return _resolve(defs[ref_name]) + return {k: _resolve(v) for k, v in obj.items() if k != "$defs"} + if isinstance(obj, list): + return [_resolve(item) for item in obj] + return obj + + return _resolve(schema) + + @dataclasses.dataclass(frozen=True, eq=True) class DecodedToolCall[T]: """ @@ -123,30 +199,6 @@ def __class_getitem__(cls, item): return TypeToPydanticType().evaluate(item) -@dataclasses.dataclass(frozen=True) -class _SynthesisSpec[T]: - template: Template[..., T] - - @property - def _class_template(self) -> Template[..., T] | None: - if isinstance(self.template.__default__, types.MethodType): - return self.template.__default__.__func__.__wrapped__ # type: ignore[attr-defined] - else: - return None - - def _method_instance(self, other: Template) -> Any | None: - """The instance ``op`` is bound to, if ``op`` is this synthesized - Agent-method on *some* instance; otherwise ``None``. - """ - if ( - self._class_template is not None - and _SynthesisSpec(other)._class_template is self._class_template - ): - return other.__default__.__self__ # type: ignore[attr-defined] - else: - return None - - class TypeToPydanticType(TypeEvaluator): """Substitute custom types with their Pydantic Annotated equivalents. @@ -268,30 +320,6 @@ def validate(value: object) -> types.CodeType: ] -def _inline_refs(schema: dict) -> dict: - """Inline ``$ref`` pointers so ``WithJsonSchema`` never emits orphan refs. - - Workaround for https://github.com/pydantic/pydantic/issues/12145 — - Pydantic's ``GenerateJsonSchema`` does not merge user-provided ``$defs`` - into its internal ref map, so any ``$ref`` in a ``WithJsonSchema`` value - causes a ``KeyError`` when the annotated type is composed into a model. - """ - defs = schema.get("$defs", {}) - - def _resolve(obj): - if isinstance(obj, dict): - if "$ref" in obj: - ref_name = obj["$ref"].split("/")[-1] - if ref_name in defs: - return _resolve(defs[ref_name]) - return {k: _resolve(v) for k, v in obj.items() if k != "$defs"} - if isinstance(obj, list): - return [_resolve(item) for item in obj] - return obj - - return _resolve(schema) - - @TypeToPydanticType.register(tuple) def _pydantic_type_tuple(ty): """Convert finitary tuples to object-based schemas (``properties/required``). @@ -427,6 +455,59 @@ def _pydantic_type_image(ty: type[Image.Image]): ] +def _callable_type_from_signature( + signature: inspect.Signature, +) -> type[types.FunctionType]: + """Construct a `Callable` type from a signature. + + Raises if the signature is recursive (e.g. a Template that returns itself) + or contains variadic parameters (which cannot be expressed in a `Callable` + type). + """ + param_types = [] + for pname, param in signature.parameters.items(): + if param.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): + raise NotImplementedError( + f"Cannot synthesize a function for parameter " + f"'{pname}' of kind {param.kind.description}: variadic parameters " + "cannot be expressed as a Callable type signature." + ) + param_types.append( + param.annotation + if param.annotation is not inspect.Parameter.empty + else typing.Any + ) + return_type = signature.return_annotation + return collections.abc.Callable[param_types, return_type] # type: ignore + + +@dataclasses.dataclass(frozen=True) +class _SynthesisSpec[T]: + template: Template[..., T] + + @property + def _class_template(self) -> Template[..., T] | None: + if isinstance(self.template.__default__, types.MethodType): + return self.template.__default__.__func__.__wrapped__ # type: ignore[attr-defined] + else: + return None + + def _method_instance(self, other: Template) -> Any | None: + """The instance ``op`` is bound to, if ``op`` is this synthesized + Agent-method on *some* instance; otherwise ``None``. + """ + if ( + self._class_template is not None + and _SynthesisSpec(other)._class_template is self._class_template + ): + return other.__default__.__self__ # type: ignore[attr-defined] + else: + return None + + class SynthesizedFunction(pydantic.BaseModel): """Structured output for function synthesis. diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index 93c59e1ce..fc4e9b79c 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -30,7 +30,6 @@ ) from RestrictedPython.PrintCollector import PrintCollector -from effectful.handlers.llm.template import Tool from effectful.internals.unification import nested_type from effectful.ops.syntax import ObjectInterpretation, defop, implements from effectful.ops.types import Operation @@ -961,7 +960,6 @@ def runcode(self, code: CodeType) -> None: except: self.showtraceback() - @Tool.define def exec_code(self, code: CodeType) -> str: """Run Python in a persistent, stateful session and return its output. From ce03b3069684389fdd2a00271bf02c492b58ade6 Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 29 Jun 2026 18:14:52 -0400 Subject: [PATCH 018/155] add name and signature to prompt_template and tool --- effectful/handlers/llm/encoding.py | 4 +-- effectful/handlers/llm/template.py | 5 +++- tests/test_handlers_llm.py | 43 ----------------------------- tests/test_handlers_llm_template.py | 30 ++++++++++---------- 4 files changed, 22 insertions(+), 60 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 1a3384e4f..1b8da70c8 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -797,13 +797,13 @@ def _serialize_tool(value: Tool) -> ChatCompletionToolParam: ) response_format = litellm.utils.type_to_response_format_param(sig_model) assert response_format is not None - assert value.__default__.__doc__ is not None + description = f"{getattr(value, '__qualname__', value.__name__)} : {value.__signature__}\n\n{textwrap.dedent(value.__doc__ or '')}" return pydantic.TypeAdapter(ChatCompletionToolParam).validate_python( { "type": "function", "function": { "name": value.__name__, - "description": textwrap.dedent(value.__default__.__doc__), + "description": description, "parameters": response_format["json_schema"]["schema"], "strict": True, }, diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index dc4533b0f..b190ddde9 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -293,7 +293,10 @@ def _validate_prompt( @property def __prompt_template__(self) -> str: assert self.__default__.__doc__ is not None - return self.__default__.__doc__ + header = f"{self.__name__}{self.__signature__}".replace("{", "{{").replace( + "}", "}}" + ) + return f"{header}\n\n{self.__default__.__doc__}" @property def tools(self) -> Mapping[str, Tool]: diff --git a/tests/test_handlers_llm.py b/tests/test_handlers_llm.py index c4c8be2cd..9c9f2e7f1 100644 --- a/tests/test_handlers_llm.py +++ b/tests/test_handlers_llm.py @@ -7,35 +7,6 @@ from effectful.ops.syntax import ObjectInterpretation, implements -class MockLLMProvider[T](ObjectInterpretation): - """Mock provider for testing. - - Initialized with prompts and responses. Raises if an unexpected prompt is given. - """ - - def __init__(self, prompt_responses: dict[str, T]): - """Initialize with a dictionary mapping prompts to expected responses. - - Args: - prompt_responses: Dict mapping prompt strings to their expected responses - """ - self.prompt_responses = prompt_responses - - @implements(Template.__apply__) - def _call[**P]( - self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs - ) -> T: - bound_args = template.__signature__.bind(*args, **kwargs) - bound_args.apply_defaults() - prompt = template.__prompt_template__.format(**bound_args.arguments) - - if prompt not in self.prompt_responses: - raise ValueError(f"Unexpected prompt: {prompt!r}") - - response = self.prompt_responses[prompt] - return response - - class SingleResponseLLMProvider[T](ObjectInterpretation): """Simplified mock provider that returns a single response for any prompt.""" @@ -92,20 +63,6 @@ def mutual_b() -> Annotated[str, IsRecursive]: raise NotHandled -# Unit tests -def test_limerick(): - """Test the limerick template returns a string.""" - mock_response = "There once was a fish from the sea" - mock_provider = MockLLMProvider( - {"Write a limerick on the theme of fish.": mock_response} - ) - - with handler(mock_provider): - result = limerick("fish") - assert result == mock_response - assert isinstance(result, str) - - def test_primes_decode_int(): """Test the primes template correctly decodes integer response.""" mock_provider = SingleResponseLLMProvider(61) diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index cc16b35e9..382e76c51 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -48,7 +48,7 @@ def rhyme(a: str, b: str) -> str: raise NotHandled with handler(TemplateStringIntp()): - assert rhyme("cat", "hat") == "The cat sat in the hat." + assert rhyme("cat", "hat").endswith("The cat sat in the hat.") def test_template_formatting_method(): @@ -63,8 +63,8 @@ def greet(self, day: str) -> float: with handler(TemplateStringIntp()): user = User("Bob") - assert ( - user.greet("Monday") == "Greet the user 'Bob' and wish them a good Monday." + assert user.greet("Monday").endswith( + "Greet the user 'Bob' and wish them a good Monday." ) @@ -1219,9 +1219,8 @@ def convert(feet: int) -> float: raise NotHandled with handler(TemplateStringIntp()): - assert ( - convert(7920) - == "How many miles is 7920 feet? There are 5280 feet per mile." + assert convert(7920).endswith( + "How many miles is 7920 feet? There are 5280 feet per mile." ) @@ -1233,7 +1232,7 @@ def poem(topic: str, style: str) -> str: """Write a {style} poem about {topic}.""" raise NotHandled - assert poem.__prompt_template__ == "Write a {style} poem about {topic}." + assert poem.__prompt_template__.endswith("Write a {style} poem about {topic}.") def test_validate_no_vars(): @@ -1244,7 +1243,7 @@ def simple() -> str: """Just a plain prompt with no variables.""" raise NotHandled - assert simple.__prompt_template__ == "Just a plain prompt with no variables." + assert simple.__prompt_template__.endswith("Just a plain prompt with no variables.") def test_validate_undefined_var(): @@ -1281,7 +1280,9 @@ def greet(self, day: str) -> str: """Agent '{self.name}' says hello on {day}.""" raise NotHandled - assert Agent.greet.__prompt_template__ == "Agent '{self.name}' says hello on {day}." + assert Agent.greet.__prompt_template__.endswith( + "Agent '{self.name}' says hello on {day}." + ) def test_validate_staticmethod(): @@ -1294,7 +1295,7 @@ def ok(a: str, b: str) -> str: raise NotHandled # The underlying Template should exist - assert ok.__func__.__prompt_template__ == "Combine {a} and {b}." + assert ok.__func__.__prompt_template__.endswith("Combine {a} and {b}.") def test_validate_staticmethod_undefined(): @@ -1334,9 +1335,8 @@ def convert(feet: int) -> str: raise NotHandled with handler(TemplateStringIntp()): - assert ( - convert(7920) - == "How many miles is 7920 feet? There are 5280 feet per mile." + assert convert(7920).endswith( + "How many miles is 7920 feet? There are 5280 feet per mile." ) @@ -1361,7 +1361,9 @@ def write_poem(topic: str) -> str: """Write a poem about {topic} by {author}.""" raise NotHandled - assert write_poem.__prompt_template__ == "Write a poem about {topic} by {author}." + assert write_poem.__prompt_template__.endswith( + "Write a poem about {topic} by {author}." + ) def test_validate_undefined_with_lexical_still_fails(): From 9c775f31a6a84eb4f043081ff283ae923f96b6ee Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 29 Jun 2026 18:46:27 -0400 Subject: [PATCH 019/155] add serialization-only schema for callable and add return schema to tool description --- effectful/handlers/llm/encoding.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 1b8da70c8..70b21acdb 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -768,8 +768,27 @@ def _serialize(value: Callable) -> dict: callable_type, pydantic.PlainValidator(_validate), pydantic.PlainSerializer(_serialize), + # Distinct schemas per direction. Validation (the model *produces* a + # function -- tool arguments, response_format) carries the synthesis + # instructions. Serialization (the model *reads* an encoded function -- + # e.g. a tool's output) shows only the shape `_serialize` emits, with no + # synthesis prose. pydantic.WithJsonSchema( - _inline_refs(pydantic.TypeAdapter(typed_enc).json_schema()) + _inline_refs(pydantic.TypeAdapter(typed_enc).json_schema()), + mode="validation", + ), + pydantic.WithJsonSchema( + { + "type": "object", + "required": ["module_code"], + "properties": { + "module_code": { + "type": "string", + "description": "Python source defining the function.", + } + }, + }, + mode="serialization", ), ] @@ -797,7 +816,14 @@ def _serialize_tool(value: Tool) -> ChatCompletionToolParam: ) response_format = litellm.utils.type_to_response_format_param(sig_model) assert response_format is not None - description = f"{getattr(value, '__qualname__', value.__name__)} : {value.__signature__}\n\n{textwrap.dedent(value.__doc__ or '')}" + ret_schema = pydantic.TypeAdapter( + Encodable[value.__signature__.return_annotation] + ).json_schema(mode="serialization") + description = ( + f"{getattr(value, '__qualname__', value.__name__)} : {value.__signature__}" + ) + description += f"\n\n{textwrap.dedent(value.__doc__ or '')}" + description += f"\n\nAnnotated JSON schema of return type: {json.dumps(ret_schema)}" return pydantic.TypeAdapter(ChatCompletionToolParam).validate_python( { "type": "function", From 00afd9662d327a4683678837022c859df2741e00 Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 29 Jun 2026 22:47:30 -0400 Subject: [PATCH 020/155] Assemble system prompt --- effectful/handlers/llm/__init__.py | 47 ++++ effectful/handlers/llm/completions.py | 363 +++++++++++++++++++------- effectful/handlers/llm/encoding.py | 25 ++ effectful/handlers/llm/template.py | 163 ++++++------ tests/test_handlers_llm_template.py | 48 ++-- 5 files changed, 443 insertions(+), 203 deletions(-) diff --git a/effectful/handlers/llm/__init__.py b/effectful/handlers/llm/__init__.py index cdda93479..b446eb310 100644 --- a/effectful/handlers/llm/__init__.py +++ b/effectful/handlers/llm/__init__.py @@ -1,3 +1,50 @@ +"""LLM-implemented functions via algebraic effects. + +`effectful.handlers.llm` lets you write Python functions whose bodies are +implemented by a large language model, and call them like ordinary code. + +## Core concepts + +- **`Template`** — a fully type-annotated Python function whose body is `raise + NotHandled` and whose docstring is a [format + string](https://docs.python.org/3/library/string.html#format-string-syntax) + prompt. Calling a template (under a provider) formats its arguments into the + prompt, invokes the model, and decodes the response to the template's declared + return type. Define one with the `Template.define` decorator. + +- **`Tool`** — a normal Python callable exposed to the model. Its signature and + docstring become the schema the model sees; the model calls it by name with + JSON arguments and receives the encoded result. Tools in a template's lexical + scope are offered to the model automatically. Define one with `Tool.define`. + +- **`Agent`** — a class mixin giving each instance a persistent message history, + so its `Template` methods accumulate conversation context across calls. + Instance attributes are available in prompts via `{self.attr}`. + +- **`Encodable`** — the type-driven JSON bridge used internally to encode Python + values into the model's context and decode the model's output (structured + return values and tool-call arguments) back into typed Python objects. + +## Tool calling and structured output + +During a template call the model may take multiple turns: on each turn it can +call any `Tool` in scope (results are fed back and the loop continues) or +produce a final answer. The final answer is decoded to the template's return +type via constrained/structured generation, so non-`str` return types (ints, +dataclasses, etc.) come back as real Python values. A `FinalTool` lets the model +"answer" by calling a tool whose return value becomes the result and terminates +the loop. + +## Providers and handlers + +Execution is controlled by composing handlers with +`effectful.ops.semantics.handler(...)`: a provider such as +`effectful.handlers.llm.completions.LiteLLMProvider` implements the model calls, +and helpers like `RetryLLMHandler` add reliability behavior. Because everything +is an algebraic effect, behavior (model requests, tool dispatch, history) can be +observed, logged, or overridden by installing additional handlers. +""" + from .template import Agent, Template, Tool __all__ = ["Agent", "Template", "Tool"] diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 0dcaa413a..e71d215b6 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -1,4 +1,5 @@ import abc +import builtins import collections import collections.abc import dataclasses @@ -67,10 +68,6 @@ class UserMessage(OpenAIChatCompletionUserMessage): Message = AssistantMessage | ToolMessage | FunctionMessage | SystemMessage | UserMessage -DEFAULT_SYSTEM_PROMPT = ( - "You are a helpful assistant, you need to follow user's instruction" -) - class _NoActiveHistoryException(Exception): """Raised when there is no active message history to append to.""" @@ -388,56 +385,149 @@ def call_user( return message -@Operation.define -def call_system(template: Template) -> Message: - """ - Get system instruction message(s) to prepend to all LLM prompts. +def _get_qualname(cls) -> str: + """Module-qualified name of a type, dropping the ``builtins`` prefix.""" + if not isinstance(cls, type): + return str(cls) + module = getattr(cls, "__module__", None) + name = ( + getattr(cls, "__qualname__", None) or getattr(cls, "__name__", None) or str(cls) + ) + return name if module in (None, "builtins") else f"{module}.{name}" + + +def _render_vars_block(env: collections.abc.Mapping[str, typing.Any]) -> str: + """Markdown table of the non-module bindings in scope (name -> type). + + Excludes dunder names (``__main__`` etc.) and names already bound to their + standard builtin (which the model knows). """ - system_prompt = template.__system_prompt__ or DEFAULT_SYSTEM_PROMPT - message = _make_message(dict(role="system", content=system_prompt)) - append_message(message, last=False) - return message + rows = { + name: _get_qualname(type(value)) + for name, value in env.items() + if not (name.startswith("__") and name.endswith("__")) + and value not in vars(builtins).values() + and not isinstance(value, types.ModuleType) + } + if not rows: + return "" + body = "\n".join(f"| `{n}` | `{t}` |" for n, t in sorted(rows.items())) + return f"## Lexical scope\n\n| name | type |\n| --- | --- |\n{body}" -class _LexicalVariableTool[T](Tool[[], T]): - """A zero-arg `Tool` that returns the captured value of a variable - from a `Template`'s lexical context. +def _render_imports_block(env: collections.abc.Mapping[str, typing.Any]) -> str: + """Markdown table of the imported modules in scope (name -> module name). - Tools are constructed fresh each `call_assistant` invocation, so - the reader closes over the snapshot `value` rather than the - surrounding `env` — in-place mutation of a mutable value is still - visible (same object reference), but rebinding the source name is - not. + Excludes dunder names and names already bound to their standard builtin. """ + rows = { + name: value.__name__ + for name, value in env.items() + if not (name.startswith("__") and name.endswith("__")) + and value not in vars(builtins).values() + and isinstance(value, types.ModuleType) + } + if not rows: + return "" + body = "\n".join(f"| `{n}` | `{m}` |" for n, m in sorted(rows.items())) + return f"## Imported modules\n\n| name | module |\n| --- | --- |\n{body}" + + +def _render_template_block(template: Template) -> str: + """Markdown spec for a single `Template`: header, prompt, arg schemas.""" + parts = [f"### `{template.__name__}{template.__signature__}`"] + prompt = inspect.getdoc(template.__default__) or "" + if prompt: + parts.append(prompt) + args = [ + f"- `{name}` — `{_get_qualname(p.annotation)}`\n\n" + f" ```json\n {json.dumps(pydantic.TypeAdapter(Encodable[p.annotation]).json_schema())}\n ```" + for name, p in template.__signature__.parameters.items() + ] + if args: + parts.append("**Arguments**\n\n" + "\n".join(args)) + return "\n\n".join(parts) + + +def _render_agent_block(template: Template) -> str: + """One lexical inventory plus the spec of every Template + sharing the current history (an Agent's methods, or just ``template``).""" + inst = ( + template.__default__.__self__ + if isinstance(template.__default__, types.MethodType) + else None + ) + if isinstance(inst, Agent): + agent_doc = inspect.getdoc(type(inst)) or "" + templates = set() + for cls in type(inst).__mro__: + for attr in vars(cls): + try: + value = getattr(inst, attr) + except Exception: + continue + if isinstance(value, Template): + templates.add(value) + else: + agent_doc = "" + templates = {template} + + # Order by name so the prompt is stable across method reordering in source. + specs = "\n\n".join( + _render_template_block(t) for t in sorted(templates, key=lambda t: t.__name__) + ) + sections = [ + f"## Agent `{_get_qualname(type(inst))}`\n\n{agent_doc}" if agent_doc else "", + f"## Templates\n\n{specs}", + ] + return "\n\n".join(s for s in sections if s) - @classmethod - def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": - """Construct a synthetic reader Tool that returns `value`. - Raises if `Encodable[nested_type(value)]` cannot be generated. - The caller is responsible for catching the failure and deciding - whether to skip the symbol. - """ - assert not isinstance(value, Tool), ( - "Tools are real tools and must not be re-wrapped as lexical readers." - ) - typ: typing.Any = nested_type(value).value - # Probe schema generation; raises if `Encodable[typ]` is not implemented. - pydantic.TypeAdapter(Encodable[typ]).json_schema() - - def tool_fn(): - return value - - tool_fn.__name__ = name - tool_fn.__qualname__ = name - tool_fn.__module__ = type(value).__module__ - tool_fn.__doc__ = ( - f"Reads the value of lexical variable `{name}` from the " - f"enclosing scope where this Template was defined. Takes " - f"no arguments; returns the current value." - ) - tool_fn.__annotations__ = {"return": typ} - return super().define(tool_fn) +def _render_module_block(mod: types.ModuleType | None) -> str: + """Markdown section with the source (or docstring fallback) of a module.""" + if mod is None: + return "" + try: + src = inspect.getsource(mod) + return f"## Module `{mod.__name__}`\n\n```python\n{src}\n```" + except (OSError, TypeError): + doc = inspect.getdoc(mod) + return f"## Module `{mod.__name__}`\n\n{doc}" if doc else "" + + +def _render_global_block(tool_types: collections.abc.Set[type[Tool]]) -> str: + """Constant framework-concept prefix, sourced from real docstrings.""" + import effectful.handlers.llm as _llm + + assert all(issubclass(t, Tool) and t not in {Tool, Template} for t in tool_types) + parts = [inspect.getdoc(_llm) or ""] + for obj in [ + Template, + Tool, + Agent, + Encodable, + *sorted(tool_types, key=_get_qualname), + ]: + parts += [f"## `{obj.__name__}`\n\n{inspect.getdoc(obj)}"] + return "\n\n".join(p for p in parts if p) + + +@Operation.define +def call_system( + template: Template, *, tool_types: collections.abc.Set[type[Tool]] = frozenset() +) -> Message: + """Assemble and install the system message (a Markdown document).""" + sections = [ + _render_global_block(tool_types), + _render_module_block(inspect.getmodule(template)), + _render_agent_block(template), + _render_imports_block(template.__context__), + _render_vars_block(template.__context__), + ] + content = "\n\n".join(s for s in sections if s) + message = _make_message(dict(role="system", content=content)) + append_message(message, last=False) + return message class LexicalReaders(ObjectInterpretation): @@ -448,6 +538,48 @@ class LexicalReaders(ObjectInterpretation): schema-generation failures cause the symbol to be skipped. """ + @typing.final + class _LexicalVariableTool[T](Tool[[], T]): + """## Reading lexical variables + + Some of the tools below take no arguments and simply return the current + value of a named variable from this Template's lexical scope (see the + *Lexical scope* table for the available names and their types). Call such a + reader when your answer depends on the concrete value of an in-scope + variable that has not already been spliced into the prompt — it lets you + fetch that value on demand instead of guessing it. Each reader's description + names the variable it reads. + """ + + @classmethod + def define(cls, value: typing.Any, *, name: str) -> "Tool[[], typing.Any]": + """Construct a synthetic reader Tool that returns `value`. + + Raises if `Encodable[nested_type(value)]` cannot be generated. + The caller is responsible for catching the failure and deciding + whether to skip the symbol. + """ + assert not isinstance(value, Tool), ( + "Tools are real tools and must not be re-wrapped as lexical readers." + ) + typ: typing.Any = nested_type(value).value + # Probe schema generation; raises if `Encodable[typ]` is not implemented. + pydantic.TypeAdapter(Encodable[typ]).json_schema() + + def tool_fn(): + return value + + tool_fn.__name__ = name + tool_fn.__qualname__ = name + tool_fn.__module__ = type(value).__module__ + tool_fn.__doc__ = "Reads lexical variable of the same name" + tool_fn.__annotations__ = {"return": typ} + return super().define(tool_fn) + + @implements(call_system) + def _call_system(self, template, tool_types=frozenset()): + return fwd(template, tool_types=tool_types | {self._LexicalVariableTool}) + @implements(collect_tools) def _collect( self, env: collections.abc.Mapping[str, typing.Any] @@ -457,7 +589,7 @@ def _collect( if name in result or not name.isidentifier() or isinstance(obj, Tool): continue try: - result[name] = _LexicalVariableTool.define(obj, name=name) + result[name] = self._LexicalVariableTool.define(obj, name=name) except ( pydantic.errors.PydanticSchemaGenerationError, pydantic.errors.PydanticInvalidForJsonSchema, @@ -467,52 +599,6 @@ def _collect( return result -class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): - """A :class:`FinalTool` that finalizes a Template by synthesizing a function. - - The tool takes one argument -- a function with the Template's signature, - synthesized from the model's code by the existing ``Encodable[Callable[...]]`` - machinery -- and applies it to the original inputs (recovered from ``env``), - returning the value. Because it is a :class:`FinalTool`, calling it - terminates the completion loop and its return value is the Template's result. - """ - - @classmethod - def define( - cls, - template: Template[..., T], - bound_args: inspect.BoundArguments, - *, - name: str, - ) -> FinalTool[[collections.abc.Callable[..., T]], T]: - # Synthesize a drop-in syntactic replacement for the Template body, so the - # function carries the Template's full signature -- including `self` for - # Agent-method Templates (whose `__default__` is a bound method). - if isinstance(template.__default__, types.MethodType): - signature = inspect.signature(template.__default__.__func__) - args, kwargs = ( - (template.__default__.__self__,) + bound_args.args, - bound_args.kwargs, - ) - else: - signature = inspect.signature(template) - args, kwargs = bound_args.args, bound_args.kwargs - - callable_type = _callable_type_from_signature(signature) - callable_type = typing.Annotated[callable_type, _SynthesisSpec(template)] # type: ignore - return_type = signature.return_annotation - - def submit_solution(implementation: callable_type) -> return_type: # type: ignore - """ - Submit your final answer as a Python function implementing the task. - The function must have the required signature; it is applied to the - original inputs and its return value is your final answer. - """ - return implementation(*args, **kwargs) # type: ignore - - return super().define(submit_solution, name=name) - - class SynthesizeAndCall(ObjectInterpretation): """Answer a Template by synthesizing a function and calling it. @@ -553,13 +639,68 @@ class SynthesizeAndCall(ObjectInterpretation): be compiled and executed. """ + @typing.final + class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): + """## Code synthesis + + You may "answer" a Template by writing code instead of producing the value + directly. A final tool (typically `submit_solution`) accepts a single + argument: a Python function whose signature matches the Template's signature + (see its spec below). The harness applies that function to the original + inputs and its return value becomes the answer, so write the function body + as a drop-in implementation of the Template. The function may reference + names from the lexical scope (see the *Lexical scope* table). Calling this + tool terminates the completion. + """ + + __toolname__: typing.ClassVar[typing.Literal["submit_solution"]] = ( + "submit_solution" + ) + + @classmethod + def define( + cls, + template: Template[..., T], + bound_args: inspect.BoundArguments, + ) -> FinalTool[[collections.abc.Callable[..., T]], T]: + # Synthesize a drop-in syntactic replacement for the Template body, so the + # function carries the Template's full signature -- including `self` for + # Agent-method Templates (whose `__default__` is a bound method). + if isinstance(template.__default__, types.MethodType): + signature = inspect.signature(template.__default__.__func__) + args, kwargs = ( + (template.__default__.__self__,) + bound_args.args, + bound_args.kwargs, + ) + else: + signature = inspect.signature(template) + args, kwargs = bound_args.args, bound_args.kwargs + + callable_type = _callable_type_from_signature(signature) + callable_type = typing.Annotated[callable_type, _SynthesisSpec(template)] # type: ignore + return_type = signature.return_annotation + + def submit_solution(implementation: callable_type) -> return_type: # type: ignore + """ + Submit your final answer as a Python function implementing the task. + The function must have the required signature; it is applied to the + original inputs and its return value is your final answer. + """ + return implementation(*args, **kwargs) # type: ignore + + return super().define(submit_solution, name=cls.__toolname__) + + @implements(call_system) + def _call_system(self, template, tool_types=frozenset()): + return fwd(template, tool_types=tool_types | {self._SynthesisFinalTool}) + @implements(Template.__apply__) def _apply[**P, T]( self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs ) -> T: bound_args = template.__signature__.bind(*args, **kwargs) bound_args.apply_defaults() - tool = _SynthesisFinalTool.define(template, bound_args, name="submit_solution") + tool = self._SynthesisFinalTool.define(template, bound_args) with handler({collect_tools: lambda _: {**fwd(), tool.__name__: tool}}): # type: ignore return fwd() @@ -584,18 +725,38 @@ class PythonRepl(ObjectInterpretation): installed eval provider (`UnsafeEvalProvider` or `RestrictedEvalProvider`). """ - @Tool.define + @typing.final + class _ReplInteractionTool[**P, T](Tool[P, T]): + """## Python REPL + + You may run arbitrary Python code in a persistent session. The code is + executed in the context of this Template's lexical scope (see the *Lexical + scope* table for the available names and their types). The session persists + across turns, so you may define variables, functions, and classes that are + used in later turns. The return value of the code is returned to you as the + result of the tool call. + """ + + @typing.final + @_ReplInteractionTool.define + @classmethod @functools.wraps(ReplSession.exec_code) - def exec_code(self, code: types.CodeType) -> str: + def exec_code(cls, code: types.CodeType) -> str: raise NotImplementedError("No handler") - @Tool.define - def read_lexical_variable(self, name: str) -> typing.Any: + @typing.final + @_ReplInteractionTool.define + @classmethod + def read_lexical_variable(cls, name: str) -> typing.Any: """ Read the value of lexical variable ``name`` into the LLM context. """ raise NotImplementedError("No handler") + @implements(call_system) + def _call_system(self, template, tool_types=frozenset()): + return fwd(template, tool_types=tool_types | {self._ReplInteractionTool}) + @implements(Template.__apply__) def _apply[**P, T]( self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs @@ -753,9 +914,9 @@ def _call[**P, T]( not _get_history() or next(iter(_get_history().values()))["role"] != "system" ): - call_system(template) + message: Message = call_system(template) - message: Message = call_user(template.__prompt_template__, env) + message = call_user(template.__prompt_template__, env) # loop based on: https://cookbook.openai.com/examples/reasoning_function_calls result: T | None = None diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 70b21acdb..72896191c 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -195,6 +195,31 @@ def result_type(self) -> type[T]: else: class Encodable: + """The type-driven JSON bridge between Python values and the LLM. + + `Encodable[T]` maps a Python type `T` to a Pydantic-compatible type + whose JSON schema and (de)serialization the harness uses to move + values across the model boundary in both directions: + + - **Encoding (Python -> model):** argument and tool-result *values* + spliced into prompts are serialized to JSON via `Encodable[type]`, + so the model sees a faithful, schema-shaped rendering of each value + (including non-text values such as images, emitted as content + blocks). + - **Decoding (model -> Python):** a `Template`'s structured return + value and the arguments of every tool call are validated and + decoded from the model's JSON back into real Python objects through + the same `Encodable[type]` schema, so the value handed to your code + already has the declared type. + + Custom types register their JSON representation with + `TypeToPydanticType`; see + `effectful.handlers.llm.encoding.type_to_encodable_type`. Because the + encoding is derived from the *type*, it is the single source of truth + for both the schema shown to the model and the validation applied to + its output. + """ + def __class_getitem__(cls, item): return TypeToPydanticType().evaluate(item) diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index b190ddde9..44809ecb8 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -64,30 +64,38 @@ def _is_recursive_signature(sig: inspect.Signature): class Tool[**P, T](Operation[P, T]): - """A :class:`Tool` is a function that may be called by a :class:`Template`. + """A `Tool` is a function that may be called by a `Template`. - **Example usage:** + A `Tool` wraps a normal Python callable; its signature (parameter types + and return type) and docstring define the schema the model sees, and the + model invokes it by name with JSON arguments. - Templates may call any tool that is in their lexical scope. - In the following example, the LLM suggests a vacation destination using the :code:`cities` and :code:`weather` tools.:: + ## Example usage - @Tool.define - def cities() -> list[str]: - \"\"\"Return a list of cities that can be passed to `weather`.\"\"\" - return ["Chicago", "New York", "Barcelona"] + Templates may call any tool that is in their lexical scope. In the + following example, the LLM suggests a vacation destination using the + `cities` and `weather` tools: - @Tool.define - def weather(city: str) -> str: - \"\"\"Given a city name, return a description of the weather in that city.\"\"\" - status = {"Chicago": "cold", "New York": "wet", "Barcelona": "sunny"} - return status.get(city, "unknown") + ```python + @Tool.define + def cities() -> list[str]: + \"\"\"Return a list of cities that can be passed to `weather`.\"\"\" + return ["Chicago", "New York", "Barcelona"] - @Template.define # cities and weather auto-captured from lexical scope - def vacation() -> str: - \"\"\"Use the `cities` and `weather` tools to suggest a city that has good weather.\"\"\" - raise NotHandled + @Tool.define + def weather(city: str) -> str: + \"\"\"Given a city name, return a description of the weather in that city.\"\"\" + status = {"Chicago": "cold", "New York": "wet", "Barcelona": "sunny"} + return status.get(city, "unknown") + + @Template.define # cities and weather auto-captured from lexical scope + def vacation() -> str: + \"\"\"Use the `cities` and `weather` tools to suggest a city that has good weather.\"\"\" + raise NotHandled + ``` - Class methods may be used as templates, in which case any other methods decorated with :func:`Tool.define` will be provided as tools. + Class methods may be used as templates, in which case any other methods + decorated with `Tool.define` will be provided as tools. """ @@ -104,55 +112,57 @@ def __signature__(self): def define(cls, *args, **kwargs) -> "Tool[P, T]": """Define a tool. - See :func:`effectful.ops.types.Operation.define` for more information on the use of :func:`Tool.define`. + See `effectful.ops.types.Operation.define` for more information on the + use of `Tool.define`. """ return typing.cast("Tool[P, T]", super().define(*args, **kwargs)) class FinalTool[**P, T](Tool[P, T]): - """A :class:`Tool` whose invocation *finalizes* a :class:`Template` call. + """A `Tool` whose invocation *finalizes* a `Template` call. - During completion a :class:`Template` lets the LLM freely call any tool in + During completion a `Template` lets the LLM freely call any tool in scope, feeding each tool's result back for another turn. When the LLM - instead calls a :class:`FinalTool`, that tool's return value becomes the + instead calls a `FinalTool`, that tool's return value becomes the Template's result and the completion loop terminates -- no further model turn is taken, so the value is attributed to executing the tool rather than generated by the model. This is the mechanism behind code-synthesis completion (see - :class:`effectful.handlers.llm.completions.SynthesizeAndCall`): the LLM + `effectful.handlers.llm.completions.SynthesizeAndCall`): the LLM "answers" by calling a final tool with a function it wrote, the harness applies that function to the original inputs, and the resulting value is the answer. A finalizing call that *fails* does not terminate the loop -- the error is fed back as a tool message and the model is given another turn (see - :class:`effectful.handlers.llm.completions.RetryLLMHandler`). + `effectful.handlers.llm.completions.RetryLLMHandler`). """ @classmethod def define(cls, *args, **kwargs) -> "FinalTool[P, T]": """Define a final tool. - See :func:`effectful.ops.types.Operation.define` for more information on - the use of :func:`FinalTool.define`. + See `effectful.ops.types.Operation.define` for more information on + the use of `FinalTool.define`. """ return typing.cast("FinalTool[P, T]", super().define(*args, **kwargs)) class Template[**P, T](Tool[P, T]): - """A :class:`Template` is a function that is implemented by a large language model. + """A `Template` is a function that is implemented by a large language model. - **Constructing Templates:** + ## Constructing Templates - Templates are constructed by calling :func:`Template.define`. + Templates are constructed by calling `Template.define`. `Template.define` should be used as a decorator on a function or method. The function must be fully type-annotated and have a docstring. - The body of the function must contain only :code:`raise NotHandled`. - See :func:`effectful.ops.types.Operation.define` for more information on the use of :func:`Template.define`. + The body of the function must contain only `raise NotHandled`. + See `effectful.ops.types.Operation.define` for more information on the use of `Template.define`. - The template docstring is a `format string `__, which may refer to the template arguments. + The template docstring is a [format string](https://docs.python.org/3/library/string.html#format-string-syntax), + which may refer to the template arguments. When the template is called, the arguments and docstring are formatted into a prompt for the LLM and the LLM's response is returned. The following template writes limericks on a given theme: @@ -162,7 +172,7 @@ class Template[**P, T](Tool[P, T]): ... \"\"\"Write a limerick on the theme of {theme}. Do not use any tools.\"\"\" ... raise NotHandled - **Structured output:** + ## Structured output Templates may return types that are not strings. The output from the LLM is then decoded before being returned to the user. @@ -190,18 +200,17 @@ class Template[**P, T](Tool[P, T]): ... raise NotHandled Many common Python data types are decodable without additional effort. - To register a decoder for a custom type, see :func:`effectful.handlers.llm.encoding.type_to_encodable_type`. + To register a decoder for a custom type, see `effectful.handlers.llm.encoding.type_to_encodable_type`. - **Using tools:** + ## Using tools - Instances of :class:`Tool` that are in the lexical scope of a :class:`Template` may be called by the LLM during template completion. + Instances of `Tool` that are in the lexical scope of a `Template` may be called by the LLM during template completion. Templates are themselves tools which enables the construction of complex agent workflows. - When a method is defined as a template, other methods on the class that are decorated with :func:`Tool.define` or :func:`Template.define` are provided to the template as tools. + When a method is defined as a template, other methods on the class that are decorated with `Tool.define` or `Template.define` are provided to the template as tools. """ __context__: ChainMap[str, Any] - __system_prompt__: str @classmethod def _validate_doctests_constant(cls, template: "Template", doc: str) -> None: @@ -327,14 +336,6 @@ def __get__[S](self, instance: S | None, owner: type[S] | None = None): if isinstance(instance, Agent): assert isinstance(result, Template) and not hasattr(result, "__history__") result.__history__ = instance.__history__ # type: ignore[attr-defined] - result.__system_prompt__ = "\n\n".join( - part - for part in ( - getattr(result, "__system_prompt__", ""), - instance.__system_prompt__, - ) - if part - ) return result @classmethod @@ -343,11 +344,11 @@ def define[**Q, V]( ) -> "Template[Q, V]": """Define a prompt template. - :func:`define` takes a function and can be used as a decorator. + `define` takes a function and can be used as a decorator. The function's docstring should be a prompt, which may be templated in the function arguments. - The prompt will be provided with any instances of :class:`Tool` that exist in the lexical context as callable tools. + The prompt will be provided with any instances of `Tool` that exist in the lexical context as callable tools. - See :func:`effectful.ops.types.Operation.define` for more information on the use of :func:`Template.define`. + See `effectful.ops.types.Operation.define` for more information on the use of `Template.define`. """ frame = inspect.currentframe() @@ -396,8 +397,6 @@ def define[**Q, V]( ) op = super().define(default, *args, **kwargs) op.__context__ = context # type: ignore[attr-defined] - mod = inspect.getmodule(_fn) - op.__system_prompt__ = inspect.getdoc(mod) or "" # type: ignore[attr-defined] # Keep validation on original define-time callables, but skip the bound wrapper path. # to avoid dropping `self` from the signature and falsely rejecting valid prompt fields like `{self.name}`. is_bound_wrapper = ( @@ -412,52 +411,42 @@ def define[**Q, V]( class Agent(abc.ABC): """Mixin that gives each instance a persistent LLM message history. - Subclass and decorate methods with :func:`Template.define`. + Subclass and decorate methods with `Template.define`. Each instance accumulates messages across calls so the LLM sees prior conversation context. - Agents compose freely with :func:`dataclasses.dataclass` and other + Agents compose freely with `dataclasses.dataclass` and other base classes. Instance attributes are available in template - docstrings via ``{self.attr}``. + docstrings via `{self.attr}`. - Example:: + Example: - import dataclasses - from effectful.handlers.llm import Agent, Template - from effectful.handlers.llm.completions import LiteLLMProvider - from effectful.ops.semantics import handler - from effectful.ops.types import NotHandled + ```python + import dataclasses + from effectful.handlers.llm import Agent, Template + from effectful.handlers.llm.completions import LiteLLMProvider + from effectful.ops.semantics import handler + from effectful.ops.types import NotHandled - @dataclasses.dataclass - class ChatBot(Agent): - bot_name: str = dataclasses.field(default="ChatBot") + @dataclasses.dataclass + class ChatBot(Agent): + bot_name: str = dataclasses.field(default="ChatBot") - @Template.define - def send(self, user_input: str) -> str: - \"""Friendly bot named {self.bot_name}. User writes: {user_input}\""" - raise NotHandled + @Template.define + def send(self, user_input: str) -> str: + \"""Friendly bot named {self.bot_name}. User writes: {user_input}\""" + raise NotHandled - provider = LiteLLMProvider() - chatbot = ChatBot() + provider = LiteLLMProvider() + chatbot = ChatBot() - with handler(provider): - chatbot.send("Hi! How are you? I am in France.") - chatbot.send("Remind me again, where am I?") # sees prior context + with handler(provider): + chatbot.send("Hi! How are you? I am in France.") + chatbot.send("Remind me again, where am I?") # sees prior context + ``` """ - __history__: OrderedDict[str, Mapping[str, Any]] - __system_prompt__: str - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - if not hasattr(cls, "__history__"): - prop = functools.cached_property(lambda _: OrderedDict()) - prop.__set_name__(cls, "__history__") - cls.__history__ = prop - if not hasattr(cls, "__system_prompt__"): - sp = functools.cached_property( - lambda self: inspect.getdoc(type(self)) or "" - ) - sp.__set_name__(cls, "__system_prompt__") - cls.__system_prompt__ = sp + @functools.cached_property + def __history__(self) -> OrderedDict[str, Mapping[str, Any]]: + return OrderedDict() diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 382e76c51..fc41a4d68 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -11,8 +11,11 @@ from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import ( DEFAULT_SYSTEM_PROMPT, + LexicalReaders, LiteLLMProvider, RetryLLMHandler, + _get_history, + call_system, call_user, completion, ) @@ -450,36 +453,49 @@ def standalone(topic: str) -> str: assert_single_system_message_first(mock.received_messages[0]) assert_single_system_message_first(mock.received_messages[1]) - def test_empty_system_prompt_uses_default_fallback(self): + def test_system_message_assembled_from_introspection(self): @Template.define def standalone(topic: str) -> str: """Write about {topic}.""" raise NotHandled - # Simulate notebook/empty-module-docstring fallback case. - standalone.__system_prompt__ = "" - mock = MockCompletionHandler([make_text_response("ok")]) with handler(LiteLLMProvider()), handler(mock): standalone("fish") assert_single_system_message_first(mock.received_messages[0]) - assert mock.received_messages[0][0]["content"] == DEFAULT_SYSTEM_PROMPT + content = mock.received_messages[0][0]["content"] + # call_system is now the sole assembler: the content is a Markdown + # document introspected from the Template, not a stored attribute. + assert content != DEFAULT_SYSTEM_PROMPT + assert "### `standalone(topic: str) -> str`" in content + assert "Write about {topic}." in content class TestAgentDocstringFallback: - """Agent subclasses can fall back to inherited class docstrings.""" + """Agent subclasses' class docstrings flow into the assembled system message.""" + + def _system_content(self, template): + od = collections.OrderedDict() + with handler({_get_history: lambda: od}): + call_system(template) + return next(iter(od.values()))["content"] def test_missing_docstring_uses_inherited_doc(self): class MissingDocAgent(Agent): - pass + @Template.define + def act(self) -> str: + """Do something.""" + raise NotHandled assert MissingDocAgent.__doc__ is None - prompt = MissingDocAgent().__system_prompt__ - assert prompt + content = self._system_content(MissingDocAgent().act) + # No subclass docstring -> the Agent base-class docstring is used as the + # tier-3 "## Agent" section (inspect.getdoc walks the MRO). agent_doc = inspect.getdoc(Agent) assert agent_doc is not None - assert prompt == agent_doc + assert "## Agent `MissingDocAgent`" in content + assert agent_doc in content def test_non_empty_docstring_overrides_inherited_doc(self): class ValidDocAgent(Agent): @@ -487,11 +503,14 @@ class ValidDocAgent(Agent): Your goal is to satisfy the explicit Agent docstring requirement. """ + @Template.define + def act(self) -> str: + """Do something.""" + raise NotHandled + assert ValidDocAgent.__doc__ is not None - assert "You are a valid-docstring test agent." in ValidDocAgent.__doc__ - assert ( - "You are a valid-docstring test agent." in ValidDocAgent().__system_prompt__ - ) + content = self._system_content(ValidDocAgent().act) + assert "You are a valid-docstring test agent." in content class TestAgentCachedProperty: @@ -1644,7 +1663,6 @@ def test_tool_forward_ref(): import pydantic from effectful.handlers.llm.completions import ( - LexicalReaders, PythonRepl, _LexicalVariableTool, collect_tools, From e17af14b2885c88365da18cdc8843610b6062dd1 Mon Sep 17 00:00:00 2001 From: Eli Date: Tue, 30 Jun 2026 20:36:50 -0400 Subject: [PATCH 021/155] many simplifications --- effectful/handlers/llm/completions.py | 155 ++++++++++++++------------ effectful/handlers/llm/encoding.py | 22 +++- effectful/handlers/llm/evaluation.py | 7 ++ effectful/handlers/llm/template.py | 17 --- tests/conftest.py | 55 +++++++++ tests/test_handlers_llm.py | 13 ++- tests/test_handlers_llm_encoding.py | 47 +++++--- tests/test_handlers_llm_provider.py | 153 +++++++------------------ tests/test_handlers_llm_template.py | 92 +++++++-------- 9 files changed, 279 insertions(+), 282 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index e71d215b6..71f2566a0 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -25,6 +25,7 @@ ) from effectful.handlers.llm.encoding import ( + _TOOLS_KEY, DecodedToolCall, Encodable, _callable_type_from_signature, @@ -38,7 +39,6 @@ FinalTool, Template, Tool, - _is_recursive_signature, ) from effectful.internals.unification import nested_type from effectful.ops.semantics import fwd, handler @@ -175,38 +175,30 @@ def to_feedback_message(self, include_traceback: bool) -> Message: ) -@Operation.define -def collect_tools( +def _tools_in_scope( env: collections.abc.Mapping[str, typing.Any], -) -> collections.abc.Mapping[str, Tool]: +) -> collections.abc.Set[Tool]: """Return the tools available to a Template given its lexical context. Default rule: real `Tool` and `Template` values bound directly in `env`, plus `Tool` methods discovered through the MRO of any - `Agent` instance in `env`. Same-Tool-under-different-names is - deduped so each Tool appears exactly once. + `Agent` instance in `env`. - Handlers (see :class:`LexicalReaders`) may override this to add - synthetic readers, hide tools, etc. + Tools are identified by object, so the same `Tool` visible under + several bindings appears once. Names are derived from each tool's + `__name__` by :func:`call_assistant`, not from the binding name. """ - result: dict[str, Tool] = {} + result: set[Tool] = set() - for name, obj in env.items(): + for obj in env.values(): if isinstance(obj, Tool | Template): - result[name] = obj + result.add(obj) elif isinstance(obj, Agent): for cls in type(obj).__mro__: for attr_name in vars(cls): - if isinstance(getattr(obj, attr_name), Tool): - result[f"{name}__{attr_name}"] = getattr(obj, attr_name) - - # Same Tool can appear under multiple names when visible both in the - # enclosing scope and via an Agent instance's MRO. Keep only the - # last name for each unique tool object. - tool2name = {tool: name for name, tool in sorted(result.items())} - for name, tool in tuple(result.items()): - if tool2name[tool] != name: - del result[name] + attr = getattr(obj, attr_name) + if isinstance(attr, Tool): + result.add(attr) return result @@ -234,34 +226,35 @@ class _BoxedResponse[T](pydantic.BaseModel): def call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], - model: str, - **kwargs, + tools: collections.abc.Set[Tool] = frozenset(), ) -> AssistantResult[T]: """Low-level LLM request. Handlers may log/modify requests and delegate via fwd(). This effect is emitted for model request/response rounds so handlers can observe/log requests. + The available `tools` are passed explicitly as a set; handlers that expose + additional tools (synthetic readers, REPL access, synthesis) intercept this + operation and union them into `tools` before forwarding. Each tool's + model-visible name is derived from its `__name__` (see :func:`_name_tools`), + so collection and decoding agree on a single naming scheme. + Raises: ToolCallDecodingError: If a tool call cannot be decoded. The error includes the raw assistant message for retry handling. ResultDecodingError: If the result cannot be decoded. The error includes the raw assistant message for retry handling. """ - tools = collect_tools(env) - # Decode tool calls (and the code synthesized for them) against the lexical - # context, so they resolve names from the Template's scope. - env = collections.ChainMap( - typing.cast("collections.abc.MutableMapping[str, typing.Any]", env), - typing.cast("collections.abc.MutableMapping[str, typing.Any]", tools), - ) - tool_specs = { - k: typing.cast( + name2tool = {t.__name__: t for t in tools} + assert len(name2tool) == len(tools) + env = {_TOOLS_KEY: name2tool, **env} + tool_specs = [] + for name, t in sorted(name2tool.items()): + spec = typing.cast( pydantic.TypeAdapter[typing.Any], pydantic.TypeAdapter(Encodable[type(t)]), # type: ignore[misc] - ).dump_python(t, mode="json", context={k: t}) - for k, t in tools.items() - } + ).dump_python(t, mode="json", context={name: t}) + tool_specs.append(spec) # The OpenAI API requires a wrapper object for non-object structured output types, # so we create one on the fly here. Using a Pydantic model offloads JSON schema @@ -273,11 +266,9 @@ def call_assistant[T]( ) response: litellm.types.utils.ModelResponse = completion( - model, messages=list(_get_history().values()), response_format=None if response_type is str else response_format, - tools=list(tool_specs.values()), - **kwargs, + tools=tool_specs, ) choice = response.choices[0] assert isinstance(choice, litellm.types.utils.Choices) @@ -441,7 +432,7 @@ def _render_template_block(template: Template) -> str: parts.append(prompt) args = [ f"- `{name}` — `{_get_qualname(p.annotation)}`\n\n" - f" ```json\n {json.dumps(pydantic.TypeAdapter(Encodable[p.annotation]).json_schema())}\n ```" + f" ```json\n {json.dumps(pydantic.TypeAdapter(Encodable[p.annotation]).json_schema())}\n ```" # type: ignore[name-defined] for name, p in template.__signature__.parameters.items() ] if args: @@ -508,7 +499,7 @@ def _render_global_block(tool_types: collections.abc.Set[type[Tool]]) -> str: Encodable, *sorted(tool_types, key=_get_qualname), ]: - parts += [f"## `{obj.__name__}`\n\n{inspect.getdoc(obj)}"] + parts += [f"## `{obj.__name__}`\n\n{inspect.getdoc(obj)}"] # type: ignore[attr-defined] return "\n\n".join(p for p in parts if p) @@ -531,7 +522,7 @@ def call_system( class LexicalReaders(ObjectInterpretation): - """Override `collect_tools` to also expose plain values from the + """Intercept `call_assistant` to also expose plain values from the lexical context as zero-argument read-only Tools. Each non-Tool, non-Template, non-Agent value bound to a valid identifier is wrapped via `_LexicalVariableTool` if `Encodable[T]` accepts it; @@ -580,23 +571,29 @@ def tool_fn(): def _call_system(self, template, tool_types=frozenset()): return fwd(template, tool_types=tool_types | {self._LexicalVariableTool}) - @implements(collect_tools) - def _collect( - self, env: collections.abc.Mapping[str, typing.Any] - ) -> collections.abc.Mapping[str, Tool]: - result = dict(fwd()) + @implements(call_assistant) + def _call_assistant[T]( + self, + env: collections.abc.Mapping[str, typing.Any], + response_type: type[T], + tools: collections.abc.Set[Tool] = frozenset(), + ) -> AssistantResult[T]: + readers: set[Tool] = set(tools) + taken = {t.__name__ for t in tools} for name, obj in env.items(): - if name in result or not name.isidentifier() or isinstance(obj, Tool): + if ( + name in taken + or not name.isidentifier() + or isinstance(obj, Tool) + or (name.startswith("__") and name.endswith("__")) + ): continue try: - result[name] = self._LexicalVariableTool.define(obj, name=name) - except ( - pydantic.errors.PydanticSchemaGenerationError, - pydantic.errors.PydanticInvalidForJsonSchema, - pydantic.errors.PydanticUserError, - ): + readers.add(self._LexicalVariableTool.define(obj, name=name)) + taken.add(name) + except Exception: continue - return result + return fwd(env, response_type, readers) class SynthesizeAndCall(ObjectInterpretation): @@ -701,7 +698,11 @@ def _apply[**P, T]( bound_args = template.__signature__.bind(*args, **kwargs) bound_args.apply_defaults() tool = self._SynthesisFinalTool.define(template, bound_args) - with handler({collect_tools: lambda _: {**fwd(), tool.__name__: tool}}): # type: ignore + + def _add_synthesis_tool(env, response_type, tools=frozenset()): + return fwd(env, response_type, tools | {tool}) + + with handler({call_assistant: _add_synthesis_tool}): return fwd() @@ -714,8 +715,8 @@ class PythonRepl(ObjectInterpretation): Scoping mirrors how `__history__` is managed for Template calls: `PythonRepl` handles `Template.__apply__` to introduce a fresh `_repl_session` handler for - the duration of the call, and handles `collect_tools` to inject an `exec_code` - Tool routed to that session. The session is therefore introduced and + the duration of the call, and intercepts `call_assistant` to inject an + `exec_code` Tool routed to that session. The session is therefore introduced and eliminated by its own handler, bounded to the Template call by construction -- there is no global registry of sessions, and nested Template calls get their own isolated sessions. @@ -773,14 +774,18 @@ def _apply[**P, T]( ): return fwd() - @implements(collect_tools) - def _collect( - self, env: collections.abc.Mapping[str, typing.Any] - ) -> collections.abc.Mapping[str, Tool]: - tools = dict(fwd()) - tools[self.exec_code.__name__] = self.exec_code - tools[self.read_lexical_variable.__name__] = self.read_lexical_variable - return tools + @implements(call_assistant) + def _call_assistant[T]( + self, + env: collections.abc.Mapping[str, typing.Any], + response_type: type[T], + tools: collections.abc.Set[Tool] = frozenset(), + ) -> AssistantResult[T]: + return fwd( + env, + response_type, + tools | {self.exec_code, self.read_lexical_variable}, + ) class RetryLLMHandler(ObjectInterpretation): @@ -847,8 +852,7 @@ def _call_assistant[T]( self, env: collections.abc.Mapping[str, typing.Any], response_type: type[T], - model: str, - **kwargs, + tools: collections.abc.Set[Tool] = frozenset(), ) -> AssistantResult[T]: _message_sequence = _get_history().copy() @@ -892,6 +896,12 @@ def __init__(self, model="gpt-4o", **config): **inspect.signature(litellm.completion).bind_partial(**config).kwargs, } + @implements(completion) + def _completion(self, *args, **kwargs): + """Inject the provider's configuration (model and bound litellm kwargs) + into the low-level request before delegating.""" + return fwd(*args, **{**self.config, **kwargs}) + @implements(Template.__apply__) def _call[**P, T]( self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs @@ -901,9 +911,6 @@ def _call[**P, T]( bound_args.apply_defaults() env = template.__context__.new_child(bound_args.arguments) - if not _is_recursive_signature(template.__signature__): - env = env.new_child({k: None for k, v in env.items() if v is template}) - history: collections.OrderedDict[str, Message] = getattr( template, "__history__", collections.OrderedDict() ) # type: ignore @@ -923,7 +930,9 @@ def _call[**P, T]( is_final: bool = False while not is_final: message, tool_calls, result = call_assistant( - env, template.__signature__.return_annotation, **self.config + env, + template.__signature__.return_annotation, + _tools_in_scope(env) - {template}, ) if tool_calls: for tool_call in tool_calls: @@ -953,7 +962,7 @@ class LangfuseTracer(ObjectInterpretation): client: langfuse.Langfuse = dataclasses.field(default_factory=langfuse.get_client) @implements(completion) - def completion(self, model, *args, **kwargs): + def completion(self, *args, **kwargs): messages = kwargs.get("messages") if kwargs.get("tools") is not None: gen_input = {"tools": kwargs["tools"], "messages": messages} @@ -977,12 +986,12 @@ def completion(self, model, *args, **kwargs): with self.client.start_as_current_observation( as_type="generation", name="completion", - model=model, input=gen_input, model_parameters=model_parameters or None, metadata=metadata or None, ) as gen: response = fwd() + gen.update(model=response.model) usage = getattr(response, "usage", None) if usage is not None: gen.update( diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 72896191c..f403a928e 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -42,6 +42,11 @@ type ToolCallID = str +# Key under which the name->Tool mapping is stashed in the decoding context. +# Deliberately not a valid Python identifier, so it can never collide with a +# lexical variable name sharing the context (e.g. a reader named after its var). +_TOOLS_KEY = "$TOOLS" + CONTENT_BLOCK_TYPES: frozenset[str] = frozenset( literal for member in typing.get_args(OpenAIMessageContentListBlock) @@ -706,7 +711,11 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: evaluation.type_check(module, ctx, expected_params, expected_return) g: MutableMapping[str, Any] = {} - g.update(ctx) + # Only valid identifiers can be referenced as globals; skip sentinel keys + # such as `_TOOLS_KEY` that share the decoding context. + g.update( + {k: v for k, v in ctx.items() if isinstance(k, str) and k.isidentifier()} + ) bytecode: types.CodeType = evaluation.compile(module, filename) evaluation.exec(bytecode, g) @@ -824,7 +833,7 @@ def _validate_tool( assert isinstance(info.context, Mapping), "Tool decoding requires context" value = pydantic.TypeAdapter(ChatCompletionToolParam).validate_python(value) try: - return info.context[value["function"]["name"]] + return info.context[_TOOLS_KEY][value["function"]["name"]] except KeyError as e: raise NotImplementedError(f"Unknown tool: {value['function']['name']}") from e @@ -842,7 +851,7 @@ def _serialize_tool(value: Tool) -> ChatCompletionToolParam: response_format = litellm.utils.type_to_response_format_param(sig_model) assert response_format is not None ret_schema = pydantic.TypeAdapter( - Encodable[value.__signature__.return_annotation] + Encodable[value.__signature__.return_annotation] # type: ignore[name-defined] ).json_schema(mode="serialization") description = ( f"{getattr(value, '__qualname__', value.__name__)} : {value.__signature__}" @@ -882,7 +891,7 @@ def _validate_tool_call( value = OpenAIChatCompletionMessageToolCall.model_validate(value) ctx = info.context or {} assert value.function.name is not None - tool = ctx[value.function.name] + tool = ctx[_TOOLS_KEY][value.function.name] assert isinstance(tool, Tool) sig = inspect.signature(tool) decoded_args = {} @@ -918,7 +927,10 @@ def _serialize_tool_call( "type": "function", "id": value.id, "function": { - "name": value.tool.__name__, + # Use the name the tool was called by (possibly disambiguated by + # `call_assistant`), not the tool's `__name__`, so the call + # round-trips to the same identity the model and decoder share. + "name": value.name, "arguments": json.dumps(encoded_args), }, } diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index fc4e9b79c..79471010f 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -606,6 +606,13 @@ def mypy_type_check( ) func_name = last.name + # Drop sentinel keys that are not referenceable identifiers (e.g. the + # `$TOOLS` tool-mapping stashed in the decoding context); they cannot appear + # in synthesized code and would produce invalid stub syntax. + ctx = { + k: v for k, v in ctx.items() if all(seg.isidentifier() for seg in k.split(".")) + } + imports = collect_imports(ctx) # Ensure annotations in the postlude can be resolved (e.g. collections.abc.Callable, typing) baseline_imports: list[ast.stmt] = [ diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 44809ecb8..30d595ab2 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -307,23 +307,6 @@ def __prompt_template__(self) -> str: ) return f"{header}\n\n{self.__default__.__doc__}" - @property - def tools(self) -> Mapping[str, Tool]: - """Operations and Templates available as tools, plus synthetic - readers for other lexical symbols. Auto-captured from lexical context.""" - from effectful.handlers.llm.completions import collect_tools - - result = dict(collect_tools(self.__context__)) - - # We remove the template itself from the tool map unless it is explicitly - # marked as recursive (see test_template_method, test_template_method_nested_class). - if not _is_recursive_signature(self.__signature__): - for name, tool in tuple(result.items()): - if tool is self: - del result[name] - - return result - def __get__[S](self, instance: S | None, owner: type[S] | None = None): if hasattr(self, "_name_on_instance") and hasattr( instance, self._name_on_instance diff --git a/tests/conftest.py b/tests/conftest.py index fcdd4c3ba..52e3456b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,3 +37,58 @@ def pytest_runtest_call(item): pytest.xfail(str(e)) else: raise e + + +def offered_tools(env, *handlers): + """Name -> Tool mapping the model would be offered for lexical scope `env` + under the given handlers. + + Replaces the old ``collect_tools`` operation: tool collection now happens as + `call_assistant` seeds its `tools` set from :func:`_tools_in_scope` and the + augmenting handlers (``LexicalReaders``, ``PythonRepl``, ...) union more in. + This installs a capture handler that records the tools `call_assistant` + ultimately receives, named by :func:`_name_tools`. + """ + import contextlib + + from effectful.handlers.llm.completions import ( + _name_tools, + _tools_in_scope, + call_assistant, + ) + from effectful.ops.semantics import handler + from effectful.ops.syntax import ObjectInterpretation, implements + + captured: dict = {} + + class _Capture(ObjectInterpretation): + @implements(call_assistant) + def _ca(self, env_, response_type, tools=frozenset(), **kw): + captured.update(_name_tools(tools)) + return ({}, [], None) + + with contextlib.ExitStack() as stack: + stack.enter_context(handler(_Capture())) + for h in handlers: + stack.enter_context(handler(h)) + call_assistant(env, str, _tools_in_scope(env)) + return captured + + +def template_tools(template, *handlers): + """Name -> Tool mapping a `Template` would offer under the given handlers. + + Mirrors the behaviour of the removed ``Template.tools`` property: it applies + the same handler augmentation as :func:`offered_tools` and drops the template + itself unless its signature is recursive. + """ + import collections + + from effectful.handlers.llm.template import _is_recursive_signature + + result = offered_tools(template.__context__, *handlers) + if not _is_recursive_signature(template.__signature__): + result = collections.OrderedDict( + (n, t) for n, t in result.items() if t is not template + ) + return result diff --git a/tests/test_handlers_llm.py b/tests/test_handlers_llm.py index 9c9f2e7f1..9b5354a7b 100644 --- a/tests/test_handlers_llm.py +++ b/tests/test_handlers_llm.py @@ -5,6 +5,7 @@ from effectful.handlers.llm.template import IsRecursive from effectful.ops.semantics import NotHandled, handler from effectful.ops.syntax import ObjectInterpretation, implements +from tests.conftest import template_tools class SingleResponseLLMProvider[T](ObjectInterpretation): @@ -132,8 +133,8 @@ def write_story(topic: str, style: str) -> str: assert write_story.__context__["story_funny"] is story_funny # Templates in lexical context are exposed as callable tools - assert story_with_moral in write_story.tools.values() - assert story_funny in write_story.tools.values() + assert story_with_moral in template_tools(write_story).values() + assert story_funny in template_tools(write_story).values() def test_template_composition_with_chained_calls(): @@ -178,11 +179,11 @@ def test_mutually_recursive_templates(): assert "mutual_b" in mutual_b.__context__ # They should also be in each other's tools - assert mutual_a in mutual_b.tools.values() - assert mutual_b in mutual_a.tools.values() + assert mutual_a in template_tools(mutual_b).values() + assert mutual_b in template_tools(mutual_a).values() # And themselves (self-recursion) - assert mutual_a in mutual_a.tools.values() - assert mutual_b in mutual_b.tools.values() + assert mutual_a in template_tools(mutual_a).values() + assert mutual_b in template_tools(mutual_b).values() # Module-level variable for shadowing test diff --git a/tests/test_handlers_llm_encoding.py b/tests/test_handlers_llm_encoding.py index bd160dc0e..cc7423bc7 100644 --- a/tests/test_handlers_llm_encoding.py +++ b/tests/test_handlers_llm_encoding.py @@ -21,6 +21,7 @@ from PIL import Image from effectful.handlers.llm.encoding import ( + _TOOLS_KEY, CONTENT_BLOCK_TYPES, DecodedToolCall, Encodable, @@ -378,63 +379,71 @@ def _make_dtc(tool, kwargs, call_id): id="list-tuple-str-img", ), # --- Tool --- - pytest.param(type(_tool_add), _tool_add, {"_tool_add": _tool_add}, id="tool-add"), pytest.param( - type(_tool_greet), _tool_greet, {"_tool_greet": _tool_greet}, id="tool-greet" + type(_tool_add), + _tool_add, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, + id="tool-add", + ), + pytest.param( + type(_tool_greet), + _tool_greet, + {_TOOLS_KEY: {"_tool_greet": _tool_greet}}, + id="tool-greet", ), pytest.param( type(_tool_process), _tool_process, - {"_tool_process": _tool_process}, + {_TOOLS_KEY: {"_tool_process": _tool_process}}, id="tool-process", ), pytest.param( type(_tool_get_value), _tool_get_value, - {"_tool_get_value": _tool_get_value}, + {_TOOLS_KEY: {"_tool_get_value": _tool_get_value}}, id="tool-no-params", ), pytest.param( type(_tool_distance), _tool_distance, - {"_tool_distance": _tool_distance}, + {_TOOLS_KEY: {"_tool_distance": _tool_distance}}, id="tool-pydantic-param", ), pytest.param( type(_tool_style), _tool_style, - {"_tool_style": _tool_style}, + {_TOOLS_KEY: {"_tool_style": _tool_style}}, id="tool-literal-param", ), # --- DecodedToolCall --- pytest.param( DecodedToolCall, _make_dtc(_tool_add, {"a": 3, "b": 5}, "call_1"), - {"_tool_add": _tool_add}, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, id="dtc-add-3-5", ), pytest.param( DecodedToolCall, _make_dtc(_tool_add, {"a": 0, "b": -1}, "call_2"), - {"_tool_add": _tool_add}, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, id="dtc-add-0-neg", ), pytest.param( DecodedToolCall, _make_dtc(_tool_greet, {"name": "Alice"}, "call_3"), - {"_tool_greet": _tool_greet}, + {_TOOLS_KEY: {"_tool_greet": _tool_greet}}, id="dtc-greet-alice", ), pytest.param( DecodedToolCall, _make_dtc(_tool_process, {"items": [1, 2, 3], "label": "total"}, "call_4"), - {"_tool_process": _tool_process}, + {_TOOLS_KEY: {"_tool_process": _tool_process}}, id="dtc-process-items", ), pytest.param( DecodedToolCall, _make_dtc(_tool_distance, {"p": _PointModel(x=3, y=4)}, "call_5"), - {"_tool_distance": _tool_distance}, + {_TOOLS_KEY: {"_tool_distance": _tool_distance}}, id="dtc-pydantic-param", ), ] @@ -643,40 +652,44 @@ class Pair: TOOL_CALL_ERROR_CASES = [ pytest.param( - "nonexistent", "{}", {}, (KeyError, AssertionError), id="unknown-tool" + "nonexistent", + "{}", + {_TOOLS_KEY: {}}, + (KeyError, AssertionError), + id="unknown-tool", ), pytest.param( "_tool_add", '{"a": "not_an_int", "b": 2}', - {"_tool_add": _tool_add}, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, pydantic.ValidationError, id="wrong-arg-type", ), pytest.param( "_tool_add", '{"a": 1}', - {"_tool_add": _tool_add}, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, (pydantic.ValidationError, TypeError), id="missing-required-arg", ), pytest.param( "_tool_add", '{"a": 1, "b": 2, "c": 3}', - {"_tool_add": _tool_add}, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, pydantic.ValidationError, id="extra-arg", ), pytest.param( "_tool_add", "{not valid json}", - {"_tool_add": _tool_add}, + {_TOOLS_KEY: {"_tool_add": _tool_add}}, pydantic.ValidationError, id="invalid-json", ), pytest.param( "_tool_process", '{"items": ["a", "b"], "label": "total"}', - {"_tool_process": _tool_process}, + {_TOOLS_KEY: {"_tool_process": _tool_process}}, pydantic.ValidationError, id="wrong-list-element-type", ), diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index 13ee9abc3..56c437cde 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -40,11 +40,9 @@ ToolCallDecodingError, ToolCallExecutionError, _get_history, - _synthesis_final_tool, - _synthesis_template, + _tools_in_scope, call_assistant, call_tool, - collect_tools, completion, ) from effectful.handlers.llm.encoding import Encodable @@ -247,11 +245,11 @@ def test_with_config_params(self, request): def test_agent_tool_names_are_valid_integration(): agent = _ToolNameAgent() template = agent.ask - tools = template.tools - expected_helper_tool_name = f"self__{agent.helper.__name__}" + tools = _tools_in_scope(template.__context__) + names = {t.__name__ for t in tools} assert tools - assert expected_helper_tool_name in tools - assert all(re.fullmatch(r"[a-zA-Z0-9_-]+", name) for name in tools) + assert agent.helper.__name__ in names + assert all(re.fullmatch(r"[a-zA-Z0-9_-]+", name) for name in names) # End-to-end provider call. If tool names violate the schema, this raises BadRequest. with ( @@ -433,7 +431,7 @@ def __init__(self, responses: list[ModelResponse]): self.received_messages: list = [] @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): self.received_messages.append(list(messages) if messages else []) response = self.responses[min(self.call_count, len(self.responses) - 1)] self.call_count += 1 @@ -558,7 +556,6 @@ def test_retry_handler_succeeds_on_first_attempt(self): message, tool_calls, result = call_assistant( env={}, response_type=str, - model="test-model", ) assert mock_handler.call_count == 1 @@ -588,7 +585,7 @@ def test_retry_handler_retries_on_invalid_tool_call(self): message, tool_calls, result = call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) assert mock_handler.call_count == 2 @@ -620,7 +617,7 @@ def test_retry_handler_retries_on_unknown_tool(self): message, tool_calls, result = call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) assert mock_handler.call_count == 2 @@ -647,7 +644,7 @@ def test_retry_handler_exhausts_retries(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) # Should have attempted 3 times (1 initial + 2 retries) @@ -674,7 +671,7 @@ def test_retry_handler_with_zero_retries(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) def test_retry_handler_valid_tool_call_passes_through(self): @@ -697,7 +694,7 @@ def test_retry_handler_valid_tool_call_passes_through(self): message, tool_calls, result = call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) assert mock_handler.call_count == 1 @@ -771,7 +768,6 @@ def test_retry_handler_retries_on_invalid_result(self): message, tool_calls, result = call_assistant( env={}, response_type=int, - model="test-model", ) assert mock_handler.call_count == 2 @@ -803,7 +799,6 @@ def test_retry_handler_exhausts_retries_on_result_decoding(self): call_assistant( env={}, response_type=int, - model="test-model", ) # Should have attempted 3 times (1 initial + 2 retries) @@ -830,7 +825,7 @@ def test_retry_handler_raises_tool_call_decoding_error(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) error = exc_info.value @@ -860,7 +855,6 @@ def test_retry_handler_raises_result_decoding_error(self): call_assistant( env={}, response_type=int, - model="test-model", ) error = exc_info.value @@ -888,7 +882,7 @@ def test_retry_handler_error_feedback_contains_tool_name(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) # Check that the error feedback in the second call mentions the tool name @@ -918,7 +912,7 @@ def test_retry_handler_unknown_tool_error_contains_tool_name(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) # Check that the error feedback mentions the unknown tool @@ -948,7 +942,7 @@ def test_retry_handler_include_traceback_in_error_feedback(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) # Check that the error feedback includes traceback @@ -979,7 +973,7 @@ def test_retry_handler_no_traceback_when_disabled(self): call_assistant( env={"add_numbers": add_numbers}, response_type=str, - model="test-model", + tools={add_numbers}, ) # Check that the error feedback does not include traceback @@ -1078,8 +1072,8 @@ def test_tool_execution_error_not_pruned_from_messages(self): # We need a custom provider that actually calls call_tool class TestProvider(ObjectInterpretation): @implements(call_assistant) - def _call_assistant(self, env, response_type, model, **kwargs): - return fwd(env, response_type, model, **kwargs) + def _call_assistant(self, env, response_type, tools=frozenset(), **kwargs): + return fwd(env, response_type, tools, **kwargs) with ( handler(RetryLLMHandler()), @@ -1090,7 +1084,7 @@ def _call_assistant(self, env, response_type, model, **kwargs): message, tool_calls, result = call_assistant( env={"failing_tool": failing_tool}, response_type=str, - model="test-model", + tools={failing_tool}, ) # First call should succeed (tool call is valid) @@ -1364,7 +1358,9 @@ def _run(self, response: ModelResponse, env, response_type): handler({_get_history: lambda: message_sequence}), ): return call_assistant( - env=env, response_type=response_type, model="test-model" + env=env, + response_type=response_type, + tools=_tools_in_scope(env), ) def test_lone_matching_final_call_is_accepted(self): @@ -1563,8 +1559,10 @@ def variadic(*args: int) -> int: """Sum the arguments.""" raise NotHandled - with pytest.raises(TypeError, match="variadic"): - _synthesis_final_tool(variadic, {}) + with pytest.raises(NotImplementedError, match="variadic"): + SynthesizeAndCall._SynthesisFinalTool.define( + variadic, variadic.__signature__.bind() + ) class TestSynthesizeAndCallDoctests: @@ -1661,44 +1659,6 @@ def triple_it(x: int) -> int: assert result == 6 assert mock.call_count == 1 - def test_run_doctests_not_globally_hijacked_during_synthesis(self): - """The name/docstring/doctest behavior is local to the synthesis argument - (carried by _SynthesisSpec on its type), not a global run_doctests - override. So a *separate* Encodable[Callable] decode validates against its - OWN docstring even while a doctest-bearing Template is the synthesis - target -- under the old override it would have had the Template's - docstring spliced in and failed.""" - - @Template.define - def triple_it(x: int) -> int: - """Return triple {x}. - - >>> triple_it(2) - 6 - """ - raise NotHandled - - # A plain synthesized function whose OWN doctest passes but which doubles - # (so it would fail triple_it's spliced-in doctest under the old code). - module_code = ( - "def double(x: int) -> int:\n" - ' """Double x.\n' - "\n" - " >>> double(2)\n" - " 4\n" - ' """\n' - " return x * 2\n" - ) - with ( - handler(UnsafeEvalProvider()), - handler({_synthesis_template: lambda: triple_it}), - ): - f = pydantic.TypeAdapter(Encodable[Callable[[int], int]]).validate_python( - {"module_code": module_code}, context={} - ) - - assert f(3) == 6 - def test_agent_method_doctests_route_to_synthesized_function(self): """An Agent-method Template's doctests build their own instances (``agent = Doubler()``), so each ``agent.double(...)`` call dispatches a @@ -1809,7 +1769,7 @@ def test_call_assistant_no_duplicate_messages(self): class InnerAssistantHandler(ObjectInterpretation): @implements(completion) - def _completion(self_, model, messages, *args, **kwargs): + def _completion(self_, messages=None, *args, **kwargs): captured_messages.extend(list(messages)) response = { "id": "response_1", @@ -1828,7 +1788,6 @@ def _completion(self_, model, messages, *args, **kwargs): call_assistant( env={}, response_type=str, - model="test-model", ) # Forwarded messages should be [msg_a (prefix), msg_b (input)] — no duplicates @@ -1848,7 +1807,7 @@ class InnerAssistantHandler(ObjectInterpretation): call_count = 0 @implements(completion) - def _completion(self_, model, messages, *args, **kwargs): + def _completion(self_, messages=None, *args, **kwargs): call_log.append([m["id"] for m in messages]) self_.call_count += 1 response = { @@ -1870,13 +1829,11 @@ def _completion(self_, model, messages, *args, **kwargs): resp1, _, _ = call_assistant( env={}, response_type=str, - model="test-model", ) # Second call: input is the first response resp2, _, _ = call_assistant( env={}, response_type=str, - model="test-model", ) # First call: prefix=[] + input=[msg_user] @@ -1894,10 +1851,9 @@ def test_call_assistant_saves_only_on_successful_fwd(self): class FailingAssistantHandler(ObjectInterpretation): @implements(call_assistant) - def _call_assistant(self_, messages, *args, **kwargs): + def _call_assistant(self_, *args, **kwargs): raise RuntimeError("LLM call failed") - msg = {"id": "input_msg", "role": "user", "content": "hello"} frame_snapshot = dict(message_sequence) with pytest.raises(RuntimeError, match="LLM call failed"): @@ -1906,10 +1862,8 @@ def _call_assistant(self_, messages, *args, **kwargs): handler({_get_history: lambda: message_sequence}), ): call_assistant( - messages=[msg], env={}, response_type=str, - model="test-model", ) # Frame should be unchanged — no response message was saved @@ -2011,11 +1965,12 @@ def _drive_repl(body): `RetryLLMHandler`) around the call. Returns `body`'s result. """ box = [] + repl = PythonRepl() class _Loop(ObjectInterpretation): @implements(Template.__apply__) def _call(self, *_a, **_k): - box.append(body(collect_tools(collections.ChainMap({}))["exec_code"])) + box.append(body(repl.exec_code)) return None @Template.define @@ -2023,7 +1978,7 @@ def _t() -> None: """Drive one REPL-scoped call.""" raise NotImplementedError - with handler(_Loop()), handler(UnsafeEvalProvider()), handler(PythonRepl()): + with handler(_Loop()), handler(UnsafeEvalProvider()), handler(repl): _t() return box[0] @@ -2308,7 +2263,7 @@ def step_no_tool(self, topic: str) -> str: class TwoPhaseCompletionHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 if call_count == 1: @@ -2425,7 +2380,7 @@ def chat(self, msg: str) -> str: class MultiResponseHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 return make_text_response(f"reply {call_count}") @@ -2471,7 +2426,7 @@ def safe(self, task: str) -> str: class PhaseHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 if call_count == 1: @@ -2524,7 +2479,7 @@ def ask(self, question: str) -> str: class CountingHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 return make_text_response(f"answer {call_count}") @@ -2557,7 +2512,7 @@ def do(self, task: str) -> str: class MultiHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 return make_text_response(f"done {call_count}") @@ -2592,7 +2547,7 @@ def chat(self, msg: str) -> str: class MemoryHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 # Verify that previous messages are visible to later calls @@ -2635,7 +2590,7 @@ def step(self, n: int) -> str: class OrderHandler(ObjectInterpretation): @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): nonlocal call_count call_count += 1 return make_text_response(f"step {call_count}") @@ -2720,36 +2675,6 @@ def above(x: float) -> bool: assert fn(0.5) is False assert fn(threshold) is False - def test_template_exposes_lexical_classes(self): - """When `LexicalReaders` is installed, classes in the defining - scope are exposed as readers via the broad `Encodable[Callable]` - handler — the `Hand`/`Finger`/`generate_arm` motivating example - from #497. Without the handler the readers are gated off; this - test pins both contracts. - """ - - class Finger: - def wiggle(self) -> str: - return "wiggle" - - class Hand: - fingers: list[Finger] - - @Template.define - def describe_hand_action() -> str: - """Doc.""" - raise NotImplementedError - - # Off by default. - assert "Finger" not in describe_hand_action.tools - assert "Hand" not in describe_hand_action.tools - - # On under the handler. - with handler(LexicalReaders()): - tools = describe_hand_action.tools - assert "Finger" in tools - assert "Hand" in tools - class TestPythonReplIntegration: """The LLM can run code in a persistent session through `exec_code`.""" diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index fc41a4d68..b6e2ebd15 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -10,7 +10,6 @@ from effectful.handlers.llm import Agent, Template, Tool from effectful.handlers.llm.completions import ( - DEFAULT_SYSTEM_PROMPT, LexicalReaders, LiteLLMProvider, RetryLLMHandler, @@ -186,7 +185,7 @@ def __init__(self, responses: list[ModelResponse]): self.received_messages: list[list] = [] @implements(completion) - def _completion(self, model, messages=None, **kwargs): + def _completion(self, messages=None, **kwargs): self.received_messages.append(list(messages) if messages else []) response = self.responses[min(self.call_count, len(self.responses) - 1)] self.call_count += 1 @@ -391,7 +390,7 @@ def test_agent_second_call_has_one_system_message(self): def test_nested_agent_flow_has_one_system_message_per_round(self): mock = MockCompletionHandler( [ - make_tool_call_response("self__nested_tool", '{"payload": "demo"}'), + make_tool_call_response("nested_tool", '{"payload": "demo"}'), make_text_response("inner"), make_text_response("outer"), ] @@ -467,7 +466,6 @@ def standalone(topic: str) -> str: content = mock.received_messages[0][0]["content"] # call_system is now the sole assembler: the content is a Markdown # document introspected from the Template, not a stored attribute. - assert content != DEFAULT_SYSTEM_PROMPT assert "### `standalone(topic: str) -> str`" in content assert "Write about {topic}." in content @@ -651,7 +649,7 @@ def test_same_agent_nested_template_via_tool(self): """The scenario from issue #560 completes without error.""" mock = MockCompletionHandler( [ - make_tool_call_response("self__nested_tool", '{"payload": "demo"}'), + make_tool_call_response("nested_tool", '{"payload": "demo"}'), make_text_response("check passed"), make_text_response("all good"), ] @@ -667,7 +665,7 @@ def test_only_outermost_writes_to_history(self): """Inner template's messages are absent from agent.__history__.""" mock = MockCompletionHandler( [ - make_tool_call_response("self__nested_tool", '{"payload": "demo"}'), + make_tool_call_response("nested_tool", '{"payload": "demo"}'), make_text_response("inner"), make_text_response("outer"), ] @@ -691,7 +689,7 @@ def test_inner_template_gets_fresh_messages(self): not the outer template's in-flight messages.""" mock = MockCompletionHandler( [ - make_tool_call_response("self__nested_tool", '{"payload": "demo"}'), + make_tool_call_response("nested_tool", '{"payload": "demo"}'), make_text_response("inner"), make_text_response("outer"), ] @@ -715,7 +713,7 @@ def test_inner_template_sees_prior_completed_history(self): # First call: direct answer (no tool call) make_text_response("first"), # Second call: tool → nested → final - make_tool_call_response("self__nested_tool", '{"payload": "demo"}'), + make_tool_call_response("nested_tool", '{"payload": "demo"}'), make_text_response("inner"), make_text_response("second"), ] @@ -741,7 +739,7 @@ def test_sequential_call_after_nested_sees_history(self): mock = MockCompletionHandler( [ # First call: tool → nested → final - make_tool_call_response("self__nested_tool", '{"payload": "demo"}'), + make_tool_call_response("nested_tool", '{"payload": "demo"}'), make_text_response("inner"), make_text_response("first"), # Second call: direct answer @@ -794,10 +792,10 @@ def f(self) -> int: a = A(0) assert isinstance(a.f, Template) - assert a.random in a.f.tools.values() + assert a.random in template_tools(a.f).values() # f is the template itself — found via self but correctly removed (non-recursive) - assert a.f not in a.f.tools.values() - assert any(t() == 4 for t in a.f.tools.values() if t is a.random) + assert a.f not in template_tools(a.f).values() + assert any(t() == 4 for t in template_tools(a.f).values() if t is a.random) class B(A): """You are a derived template-method test agent. @@ -811,8 +809,8 @@ def reverse(self, s: str) -> str: b = B(1) assert isinstance(b.f, Template) - assert b.random in b.f.tools.values() - assert b.reverse in b.f.tools.values() + assert b.random in template_tools(b.f).values() + assert b.reverse in template_tools(b.f).values() def test_template_method_nested_class(): @@ -840,10 +838,10 @@ def f(self) -> int: a = A.B(True) assert isinstance(a.f, Template) # random is found via the enclosing function scope - assert "random" in a.f.tools + assert "random" in template_tools(a.f) # f is the template itself — found via self but correctly removed (non-recursive) - assert "f" not in a.f.tools - assert a.f.tools["random"]() == 4 + assert "f" not in template_tools(a.f) + assert template_tools(a.f)["random"]() == 4 def test_template_method_module(): @@ -915,7 +913,7 @@ def ask(self) -> str: raise NotHandled bar = Bar() - assert "helper" in bar.ask.tools + assert "helper" in template_tools(bar.ask) def test_dynamic_caller_not_leaked(self): """Variables from a dynamic caller (not lexical enclosure) should not @@ -948,9 +946,9 @@ def describe(self) -> str: raise NotHandled w = Widget() - assert w.measure in w.describe.tools.values() + assert w.measure in template_tools(w.describe).values() # The template itself is not in tools (non-recursive) - assert w.describe not in w.describe.tools.values() + assert w.describe not in template_tools(w.describe).values() def test_inherited_tools_visible(self): """Tools from a base Agent class are visible through the instance.""" @@ -976,7 +974,7 @@ def ask(self) -> str: raise NotHandled d = Derived() - assert d.base_tool in d.ask.tools.values() + assert d.base_tool in template_tools(d.ask).values() def test_tool_in_enclosing_function_visible_through_class(self): """function -> class -> Template.define: tool in the function is visible.""" @@ -992,7 +990,7 @@ def ask(self) -> str: """Ask something.""" raise NotHandled - assert "outer_tool" in Inner().ask.tools + assert "outer_tool" in template_tools(Inner().ask) def test_tool_in_enclosing_function_visible_through_nested_classes(self): """function -> class -> class -> Template.define: tool in the function @@ -1010,7 +1008,7 @@ def ask(self) -> str: """Ask something.""" raise NotHandled - assert "outer_tool" in Outer.Inner().ask.tools + assert "outer_tool" in template_tools(Outer.Inner().ask) def test_nested_function_then_class(self): """function -> function -> class -> Template.define: all enclosing @@ -1032,7 +1030,7 @@ def ask(self) -> str: outer_var = True # noqa: F841 cls = _make() - assert "inner_tool" in cls().ask.tools + assert "inner_tool" in template_tools(cls().ask) # The test method is a lexical encloser of _make, so its locals # are visible — matching Python's actual scoping rules. assert "outer_var" in cls().ask.__context__ @@ -1114,7 +1112,7 @@ def ask(x: int) -> int: """Compute {x}.""" raise NotHandled - assert "helper" in MyClass.ask.tools + assert "helper" in template_tools(MyClass.ask) def test_staticmethod_template_excludes_class_body(self): """A staticmethod Template does not capture class body locals.""" @@ -1664,11 +1662,10 @@ def test_tool_forward_ref(): from effectful.handlers.llm.completions import ( PythonRepl, - _LexicalVariableTool, - collect_tools, ) from effectful.handlers.llm.encoding import Encodable from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from tests.conftest import offered_tools, template_tools # Helpers for the test matrix @@ -1706,7 +1703,7 @@ def test_synthetic_reader_returns_captured_value(): time. In-place mutation of a mutable captured value is visible (same object reference); rebinding the source name is not.""" captured: list[int] = [1, 2, 3] - tool = _LexicalVariableTool.define(captured, name="x") + tool = LexicalReaders._LexicalVariableTool.define(captured, name="x") assert tool() == [1, 2, 3] captured.append(4) assert tool() == [1, 2, 3, 4] @@ -1717,7 +1714,7 @@ def test_synthetic_reader_snapshot_survives_rebind(): so rebinding the source name between construction and invocation has no effect on the captured value.""" env: dict = {"x": 42} - tool = _LexicalVariableTool.define(env["x"], name="x") + tool = LexicalReaders._LexicalVariableTool.define(env["x"], name="x") env["x"] = 99 assert tool() == 42 @@ -1726,7 +1723,7 @@ def test_synthetic_reader_snapshot_survives_deletion(): """The closure holds the value directly, so deleting the source name does not invalidate the reader.""" env: dict = {"x": 42} - tool = _LexicalVariableTool.define(env["x"], name="x") + tool = LexicalReaders._LexicalVariableTool.define(env["x"], name="x") del env["x"] assert tool() == 42 @@ -1750,7 +1747,7 @@ def test_synthetic_reader_snapshot_survives_deletion(): def test_lexical_variable_tool_returns_value(name, value): """`_LexicalVariableTool` builds a Tool when `Encodable[T]` schema generates; calling it returns the captured value.""" - tool = _LexicalVariableTool.define(value, name=name) + tool = LexicalReaders._LexicalVariableTool.define(value, name=name) assert tool() is value @@ -1800,8 +1797,7 @@ def test_collect_tools_exposes_callable_shaped_values(name, make_value): handler and become synthesis-shaped tools.""" value = make_value() env = {name: value} - with handler(LexicalReaders()): - assert name in collect_tools(env) + assert name in offered_tools(env, LexicalReaders()) def test_lexical_reader_exposes_data_values(): @@ -1815,16 +1811,15 @@ def test_lexical_reader_exposes_data_values(): "d": {"k": 1}, "model": _SimpleModel(x=1, y="hi"), } - with handler(LexicalReaders()): - result = collect_tools(env) + result = offered_tools(env, LexicalReaders()) assert {"x", "s", "lst", "d", "model"} <= set(result) for k, v in env.items(): assert result[k]() is v def test_template_tools_includes_synthetic_readers_for_locals(): - """The Template.tools property includes synthetic readers for - plain values in lexical scope when `LexicalReaders` is installed.""" + """A template offers synthetic readers for plain values in lexical scope + when `LexicalReaders` is installed.""" _test_data = [10, 20, 30] @Template.define @@ -1832,18 +1827,16 @@ def t() -> int: """Doc.""" raise NotHandled - with handler(LexicalReaders()): - tools = t.tools - assert "_test_data" in tools - assert tools["_test_data"]() == [10, 20, 30] + tools = template_tools(t, LexicalReaders()) + assert "_test_data" in tools + assert tools["_test_data"]() == [10, 20, 30] def test_lexical_readers_handler_enables_collection(): """Installing `LexicalReaders` flips the gate; the same values are exposed as zero-arg reader tools.""" env = {"x": 42, "s": "hello"} - with handler(LexicalReaders()): - result = collect_tools(env) + result = offered_tools(env, LexicalReaders()) assert result["x"]() == 42 assert result["s"]() == "hello" @@ -1855,20 +1848,18 @@ def test_lexical_readers_handler_enables_collection(): def test_python_repl_off_by_default(): """Without `PythonRepl`, `exec_code` is not collected.""" - assert "exec_code" not in collect_tools({"x": 1}) + assert "exec_code" not in offered_tools({"x": 1}) def test_python_repl_exposes_exec_code(): """With `PythonRepl` installed, `exec_code` is collected alongside the base tools.""" - with handler(PythonRepl()): - assert "exec_code" in collect_tools({"x": 1}) + assert "exec_code" in offered_tools({"x": 1}, PythonRepl()) def test_python_repl_composes_with_lexical_readers(): """Readers and the REPL tool coexist when both handlers are installed.""" - with handler(LexicalReaders()), handler(PythonRepl()): - result = collect_tools({"data": [1, 2, 3]}) + result = offered_tools({"data": [1, 2, 3]}, LexicalReaders(), PythonRepl()) assert "exec_code" in result assert "data" in result @@ -1883,11 +1874,12 @@ def _drive_repl(body): `__history__`. Returns `body`'s result. """ box = [] + repl = PythonRepl() class _Loop(ObjectInterpretation): @implements(Template.__apply__) def _call(self, *_a, **_k): - exec_code = collect_tools(collections.ChainMap({}))["exec_code"] + exec_code = repl.exec_code # Bodies pass source strings; decode them to code objects as the LLM # tool boundary would (`Encodable[CodeType]` compiles the source). decode = pydantic.TypeAdapter(Encodable[CodeType]).validate_python @@ -1899,7 +1891,7 @@ def _t() -> None: """Drive one REPL-scoped call.""" raise NotImplementedError - with handler(_Loop()), handler(UnsafeEvalProvider()), handler(PythonRepl()): + with handler(_Loop()), handler(UnsafeEvalProvider()), handler(repl): _t() return box[0] From eecd8ec20832a0dc58323325f7f6390009568e89 Mon Sep 17 00:00:00 2001 From: Eli Date: Tue, 30 Jun 2026 23:08:30 -0400 Subject: [PATCH 022/155] Add terminal renderer --- docs/source/codeadapt.py | 7 + docs/source/codeadapt_agent.py | 7 + effectful/handlers/llm/completions.py | 437 ++++++++++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 452 insertions(+) diff --git a/docs/source/codeadapt.py b/docs/source/codeadapt.py index 70046acb4..32c59e4c7 100644 --- a/docs/source/codeadapt.py +++ b/docs/source/codeadapt.py @@ -26,6 +26,7 @@ PythonRepl, RetryLLMHandler, SynthesizeAndCall, + TerminalRenderer, ) from effectful.handlers.llm.evaluation import UnsafeEvalProvider from effectful.ops.semantics import handler @@ -269,9 +270,15 @@ def main( action="store_true", help="Whether to log LLM calls and metadata to Langfuse", ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) args = parser.parse_args() with ( handler(LiteLLMProvider(model=args.model, tool_choice="required")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), handler(UnsafeEvalProvider()), handler(PythonRepl()), handler(SynthesizeAndCall()), diff --git a/docs/source/codeadapt_agent.py b/docs/source/codeadapt_agent.py index cdac82623..5e0d2419d 100644 --- a/docs/source/codeadapt_agent.py +++ b/docs/source/codeadapt_agent.py @@ -26,6 +26,7 @@ PythonRepl, RetryLLMHandler, SynthesizeAndCall, + TerminalRenderer, ) from effectful.handlers.llm.evaluation import UnsafeEvalProvider from effectful.ops.semantics import handler @@ -115,9 +116,15 @@ def main(args: argparse.Namespace) -> None: action="store_true", help="Whether to log LLM calls and metadata to Langfuse", ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) args = parser.parse_args() with ( handler(LiteLLMProvider(model=args.model, tool_choice="required")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), handler(UnsafeEvalProvider()), handler(PythonRepl()), handler(SynthesizeAndCall()), diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 71f2566a0..d35a379d5 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -1,4 +1,5 @@ import abc +import ast import builtins import collections import collections.abc @@ -6,6 +7,7 @@ import functools import inspect import json +import time import traceback import types import typing @@ -14,6 +16,14 @@ import langfuse import litellm import pydantic +import rich.console +import rich.live +import rich.markdown +import rich.panel +import rich.spinner +import rich.styled +import rich.syntax +import rich.text import tenacity from litellm import ( ChatCompletionFunctionMessage, @@ -885,6 +895,433 @@ def _call_tool[T](self, tool_call: DecodedToolCall[T]) -> ToolResult[T]: raise +def _message_text(content: None | str | collections.abc.Iterable[typing.Any]) -> str: + """Flatten a message ``content`` to display text. + + ``content`` may be a plain string or a list of content blocks (dicts with a + ``type`` discriminator, e.g. ``{"type": "text", "text": ...}``, as produced + by :func:`~effectful.handlers.llm.encoding.to_content_blocks`). Text blocks + contribute their text; other block types show a ``[type]`` placeholder. + """ + if content is None: + return "" + if isinstance(content, str): + return content + parts: list[str] = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + parts.append(block.get("text") or "") + else: + parts.append(f"[{block.get('type', 'content')}]") + else: + parts.append(str(block)) + return "".join(parts) + + +class _PartialToolCall(typing.TypedDict): + """A tool call being assembled from streamed deltas (name + raw JSON args).""" + + name: str + args: str + + +class _PartialAssistant(typing.TypedDict): + """The in-progress assistant turn accumulated from streaming deltas. + + ``tool_calls`` is keyed by each tool call's streaming ``index`` so fragments + for the same call (which arrive across many chunks) coalesce. + """ + + content: str + reasoning_content: str + tool_calls: dict[int, _PartialToolCall] + + +def _accumulate( + partial: _PartialAssistant, delta: litellm.types.utils.Delta | None +) -> None: + """Fold one streaming ``delta`` into the in-progress assistant ``partial``. + + Concatenates ``content`` and ``reasoning_content``, and accumulates each + streamed tool-call fragment (``function.name`` / ``function.arguments``) into + ``partial["tool_calls"]`` keyed by the tool call's ``index``. + """ + if delta is None: + return + partial["content"] += delta.content or "" + # `reasoning_content` is absent (not just None) on deltas that carry none. + partial["reasoning_content"] += getattr(delta, "reasoning_content", None) or "" + tc: litellm.types.utils.ChatCompletionDeltaToolCall + for tc in delta.tool_calls or []: + slot = partial["tool_calls"].setdefault(tc.index, {"name": "", "args": ""}) + if tc.function is not None: + slot["name"] = tc.function.name or slot["name"] + slot["args"] += tc.function.arguments or "" + + +# Panel border colors keyed by message role. +_ROLE_STYLES = { + "system": "grey50", + "user": "cyan", + "assistant": "green", + "tool": "yellow", +} + + +# Longest field body rendered in a panel before it is truncated, keeping +# large-but-static messages (notably the system prompt) from dominating the +# frame. Each renderable truncates with its own native mechanism: `Syntax` by +# whole lines (`line_range`); `Markdown` has none, so its source is clipped by +# whole lines. +_MAX_LINES = 40 + + +def _syntax(code: str, lexer: str) -> rich.console.RenderableType: + """Syntax-highlight `code` using the terminal palette, truncated to + `_MAX_LINES` via `Syntax.line_range` (no parsing, safe on partial input).""" + syntax = rich.syntax.Syntax( + code, + lexer, + theme="ansi_dark", + word_wrap=True, + background_color="default", + line_range=(1, _MAX_LINES), + ) + total = code.count("\n") + 1 + if total <= _MAX_LINES: + return syntax + note = rich.text.Text(f"… (+{total - _MAX_LINES} more lines)", style="dim") + return rich.console.Group(syntax, note) + + +def _render_markdown(text: str, *, clip: bool = True) -> rich.console.RenderableType: + """Render prose (system/user/assistant content) as Markdown. + + `Markdown` -- unlike `Syntax` -- has no native length limit, so when ``clip`` + the source is truncated to `_MAX_LINES` whole lines first. The live streaming + panel passes ``clip=False`` so the growing tail stays fully visible. + """ + lines = text.splitlines() + if clip and len(lines) > _MAX_LINES: + text = ( + "\n".join(lines[:_MAX_LINES]) + + f"\n\n*… (+{len(lines) - _MAX_LINES} more lines)*" + ) + return rich.markdown.Markdown(text, code_theme="ansi_dark") + + +def _render_reasoning(text: str, *, clip: bool = True) -> rich.console.RenderableType: + """Render reasoning as dimmed Markdown. + + `Markdown` takes no ``style=``, so `rich.styled.Styled` applies a ``dim`` + base -- the Markdown-compatible analog of the old dim-italic plain text. A + base ``italic`` interferes with Markdown's own paragraph styling (dropping + the dim too), so only ``dim`` is used. ``clip`` is forwarded to + `_render_markdown`. + """ + return rich.styled.Styled(_render_markdown(text, clip=clip), "dim") + + +def _render_data(value: typing.Any) -> rich.console.RenderableType: + """Render an already-parsed JSON value (tool result / structured-output + answer / tool-call arguments) as pretty, highlighted, line-truncated JSON.""" + return _syntax(json.dumps(value, indent=2), "json") + + +# Structured-output answers are wrapped by `_BoxedResponse` as `{"value": ...}` +# (call_assistant); the wrapper is display noise. Sourced from the model. +_BOX_FIELD = next(iter(_BoxedResponse.model_fields)) + + +def _render_content(text: str, *, unwrap: bool = False) -> rich.console.RenderableType: + """Render message content, choosing by shape rather than role: JSON + objects/arrays (tool results, direct structured-output answers) as pretty + JSON, everything else (prose, the Markdown system/user prompts) as Markdown. + + When ``unwrap`` (for a direct structured-output answer), a lone + ``_BoxedResponse`` ``{"value": ...}`` wrapper is stripped to its payload. + """ + if text.lstrip()[:1] in ("{", "["): + try: + value = json.loads(text) + except ValueError: + pass + else: + if unwrap and isinstance(value, dict) and set(value) == {_BOX_FIELD}: + value = value[_BOX_FIELD] + return _render_data(value) + return _render_markdown(text) + + +def _is_python(text: str) -> bool: + """Whether `text` looks like a Python source snippet worth highlighting. + + Detects code by *content* rather than schema/field name, so it covers every + `Encodable` type that serializes Python as a string -- the synthesis + `SynthesizedFunction.module_code` field, `exec_code`'s `types.CodeType` + argument, and any future code-carrying tool -- uniformly. Requires a + multi-line string that parses as a module with at least one real statement + (not a lone expression), which excludes prose and JSON-as-string. + """ + if "\n" not in text: + return False + try: + tree = ast.parse(text) + except SyntaxError: + return False + return any(not isinstance(node, ast.Expr) for node in tree.body) + + +def _extract_code(args: typing.Any) -> str | None: + """Return an embedded Python source string from parsed tool-call arguments. + + Walks nested dicts (a synthesized callable is ``{"implementation": + {"module_code": ...}}``; `exec_code` is a flat ``{"code": ...}``) and returns + the first string value that :func:`_is_python` recognizes. + """ + if isinstance(args, str): + return args if _is_python(args) else None + if isinstance(args, dict): + for value in args.values(): + found = _extract_code(value) + if found is not None: + return found + return None + + +def _render_tool_call( + name: str, args: str, *, streaming: bool +) -> rich.console.RenderableType: + """Render one tool call: a ``→ name`` header over its arguments. + + Synthesized code is shown as Python; ordinary arguments as pretty JSON. While + ``streaming`` the argument JSON is still partial (unparseable), so the raw + fragment is highlighted as JSON instead. + """ + header = rich.text.Text(f"→ {name}", style="bold magenta") + parsed: typing.Any = None + if not streaming: + try: + parsed = json.loads(args) + except ValueError: + parsed = None + code = _extract_code(parsed) + if code is not None: + body: rich.console.RenderableType = _syntax(code, "python") + elif parsed is not None: + body = _render_data(parsed) + else: + body = _syntax(args, "json") if args else rich.text.Text("…", style="dim") + return rich.console.Group(header, body) + + +def _message_panel(message: Message) -> rich.panel.Panel: + """Render a single completed history message as a titled panel. + + Every role (including ``system``) is shown; long field bodies are truncated + to `_MAX_LINES` lines so the frame stays readable. + """ + # A loose view for reads of keys not declared across the whole `Message` + # union (`reasoning_content`, `tool_calls`), which typecheckers infer as + # `object`; these messages are dynamically built dicts (see `_make_message`). + msg = typing.cast("collections.abc.Mapping[str, typing.Any]", message) + role = msg.get("role", "?") + renderables: list[rich.console.RenderableType] = [] + reasoning = _message_text(msg.get("reasoning_content")) + if reasoning: + renderables.append(_render_reasoning(reasoning)) + content = _message_text(msg.get("content")) + if content: + renderables.append(_render_content(content, unwrap=role == "assistant")) + for tc in msg.get("tool_calls") or []: + fn = tc.get("function", {}) if isinstance(tc, dict) else {} + renderables.append( + _render_tool_call( + fn.get("name") or "?", fn.get("arguments") or "", streaming=False + ) + ) + body = rich.console.Group(*renderables) if renderables else rich.text.Text("") + return rich.panel.Panel( + body, + title=role, + title_align="left", + border_style=_ROLE_STYLES.get(role, "white"), + ) + + +def _partial_panel( + partial: _PartialAssistant, ttft: float | None = None, *, streaming: bool = True +) -> rich.panel.Panel: + """Render the in-progress (or just-finished) assistant turn as a live panel. + + When ``ttft`` (time-to-first-token, seconds) is known it is shown as a + subtitle -- how long the model spent prefilling before the first delta. + + ``streaming`` is forwarded to `_render_tool_call`: the caller passes + ``streaming=False`` for the final frame, once the stream is exhausted and the + tool-call arguments form complete JSON, so they render as pretty JSON / + synthesized code rather than the raw partial payload. This is the only chance + the *terminating* tool call gets to settle -- there is no later `completion` + to re-render it as a history message. + """ + # Content and reasoning render as Markdown even mid-stream (Markdown never + # raises on incomplete text) and are shown in full (clip=False) so the + # growing tail stays visible. + renderables: list[rich.console.RenderableType] = [] + if partial["reasoning_content"]: + renderables.append(_render_reasoning(partial["reasoning_content"], clip=False)) + if partial["content"]: + renderables.append(_render_markdown(partial["content"], clip=False)) + for _, slot in sorted(partial["tool_calls"].items()): + renderables.append( + _render_tool_call(slot["name"], slot["args"], streaming=streaming) + ) + body = ( + rich.console.Group(*renderables) + if renderables + else rich.text.Text("…", style="dim") + ) + subtitle = ( + rich.text.Text(f"TTFT {ttft:.1f}s", style="dim") if ttft is not None else None + ) + return rich.panel.Panel( + body, + title="assistant", + title_align="left", + subtitle=subtitle, + subtitle_align="right", + border_style="green", + ) + + +class _PrefillStatus: + """Live "prefilling…" line shown until the first streamed chunk arrives. + + litellm/provider APIs report no prompt-processing progress, so there is no + true prefill percentage. Instead this shows the (locally counted) prompt + size and a ticking elapsed timer, which is what a large prompt's + time-to-first-token latency actually reflects. + + It is re-rendered by :class:`rich.live.Live`'s background refresh thread + while the main thread blocks on the first chunk, so the spinner animates and + the timer ticks on their own -- :meth:`__rich__` recomputes elapsed each call. + """ + + def __init__(self, prompt_tokens: int | None, start: float): + self._spinner = rich.spinner.Spinner("dots", style="cyan") + self._prompt_tokens = prompt_tokens + self._start = start + + def __rich__(self) -> rich.spinner.Spinner: + elapsed = time.monotonic() - self._start + size = ( + f"{self._prompt_tokens:,} tokens" + if self._prompt_tokens is not None + else "prompt" + ) + self._spinner.update( + text=rich.text.Text(f" prefilling {size}… {elapsed:.1f}s", style="cyan") + ) + return self._spinner + + +def _render_frame( + history: collections.abc.Sequence[Message], + partial: _PartialAssistant, + *, + status: _PrefillStatus | None = None, + ttft: float | None = None, + streaming: bool = True, +) -> rich.console.Group: + """Build the full frame: one panel per history message, then either the live + ``status`` line (while prefilling, before any token) or the partial turn.""" + tail = ( + status + if status is not None + else _partial_panel(partial, ttft=ttft, streaming=streaming) + ) + return rich.console.Group(*[_message_panel(m) for m in history], tail) + + +@dataclasses.dataclass(frozen=True) +class TerminalRenderer(ObjectInterpretation): + """Stream `completion` and live-render the whole message sequence. + + Opt-in debugging handler: forces streaming, redraws the entire (partial) + message history from scratch on every chunk via :class:`rich.live.Live` + (reasoning, generation, and tool-call arguments appear as they are produced), + then reassembles a normal ``ModelResponse`` via + :func:`litellm.stream_chunk_builder` so the rest of the pipeline is unchanged. + """ + + console: rich.console.Console = dataclasses.field( + default_factory=rich.console.Console + ) + + @implements(completion) + def _completion(self, *args, **kwargs) -> typing.Any: + kwargs = { + **kwargs, + "stream": True, + "stream_options": {"include_usage": True}, + } + stream: litellm.CustomStreamWrapper = fwd(*args, **kwargs) + + # The request already carries the full message history as `messages`. + history: list[Message] = list(kwargs.get("messages") or []) + + chunks: list[litellm.types.utils.ModelResponseStream] = [] + partial: _PartialAssistant = { + "content": "", + "reasoning_content": "", + "tool_calls": {}, + } + + # Count prompt tokens locally to size the prefill wait. `model` is injected + # downstream by LiteLLMProvider, so it may be absent here -- token_counter + # falls back to a default tokenizer, giving an approximate count. + try: + prompt_tokens: int | None = litellm.token_counter( + model=kwargs.get("model", ""), + messages=history, + tools=kwargs.get("tools"), + ) + except Exception: + prompt_tokens = None + + start = time.monotonic() + status: _PrefillStatus | None = _PrefillStatus(prompt_tokens, start) + ttft: float | None = None + + with rich.live.Live( + _render_frame(history, partial, status=status), + console=self.console, + vertical_overflow="visible", + ) as live: + for chunk in stream: + chunks.append(chunk) + if chunk.choices: + _accumulate(partial, chunk.choices[0].delta) + # The first chunk carrying any content ends prefill; record TTFT + # and drop the status line in favor of the streaming panel. + if status is not None and ( + partial["content"] + or partial["reasoning_content"] + or partial["tool_calls"] + ): + ttft = time.monotonic() - start + status = None + live.update(_render_frame(history, partial, status=status, ttft=ttft)) + + # The args are now complete JSON; re-render settled so the final + # (loop-terminating) tool call shows as pretty JSON / synthesized + # code rather than the raw streaming payload. + live.update(_render_frame(history, partial, ttft=ttft, streaming=False)) + + return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages")) + + class LiteLLMProvider(ObjectInterpretation): """Implements templates using the LiteLLM API.""" diff --git a/pyproject.toml b/pyproject.toml index 7a0a2cfe6..3ed3c8596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ numpyro = [ llm = [ "langfuse", "litellm", + "rich", "tenacity", "mypy", "autoflake", From 991be6241af5cb31c4826214be3d36dedb917be8 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 1 Jul 2026 03:00:42 -0400 Subject: [PATCH 023/155] rename system prompt block helpers to avoid renderer confusion --- effectful/handlers/llm/__init__.py | 4 +- effectful/handlers/llm/completions.py | 33 +++++----- effectful/handlers/llm/template.py | 89 +++++++++++++++++++++++---- 3 files changed, 100 insertions(+), 26 deletions(-) diff --git a/effectful/handlers/llm/__init__.py b/effectful/handlers/llm/__init__.py index b446eb310..d07a1f135 100644 --- a/effectful/handlers/llm/__init__.py +++ b/effectful/handlers/llm/__init__.py @@ -15,7 +15,9 @@ - **`Tool`** — a normal Python callable exposed to the model. Its signature and docstring become the schema the model sees; the model calls it by name with JSON arguments and receives the encoded result. Tools in a template's lexical - scope are offered to the model automatically. Define one with `Tool.define`. + scope are offered to the model automatically; because scope is ordinary Python + scope, an `Agent` (or an enclosing function) naturally partitions tools and + templates into disjoint sets. Define one with `Tool.define`. - **`Agent`** — a class mixin giving each instance a persistent message history, so its `Template` methods accumulate conversation context across calls. diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index d35a379d5..dc0ce6f14 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -397,7 +397,7 @@ def _get_qualname(cls) -> str: return name if module in (None, "builtins") else f"{module}.{name}" -def _render_vars_block(env: collections.abc.Mapping[str, typing.Any]) -> str: +def _system_vars_block(env: collections.abc.Mapping[str, typing.Any]) -> str: """Markdown table of the non-module bindings in scope (name -> type). Excludes dunder names (``__main__`` etc.) and names already bound to their @@ -416,7 +416,7 @@ def _render_vars_block(env: collections.abc.Mapping[str, typing.Any]) -> str: return f"## Lexical scope\n\n| name | type |\n| --- | --- |\n{body}" -def _render_imports_block(env: collections.abc.Mapping[str, typing.Any]) -> str: +def _system_imports_block(env: collections.abc.Mapping[str, typing.Any]) -> str: """Markdown table of the imported modules in scope (name -> module name). Excludes dunder names and names already bound to their standard builtin. @@ -434,7 +434,7 @@ def _render_imports_block(env: collections.abc.Mapping[str, typing.Any]) -> str: return f"## Imported modules\n\n| name | module |\n| --- | --- |\n{body}" -def _render_template_block(template: Template) -> str: +def _system_template_block(template: Template) -> str: """Markdown spec for a single `Template`: header, prompt, arg schemas.""" parts = [f"### `{template.__name__}{template.__signature__}`"] prompt = inspect.getdoc(template.__default__) or "" @@ -450,7 +450,7 @@ def _render_template_block(template: Template) -> str: return "\n\n".join(parts) -def _render_agent_block(template: Template) -> str: +def _system_agent_block(template: Template) -> str: """One lexical inventory plus the spec of every Template sharing the current history (an Agent's methods, or just ``template``).""" inst = ( @@ -475,7 +475,7 @@ def _render_agent_block(template: Template) -> str: # Order by name so the prompt is stable across method reordering in source. specs = "\n\n".join( - _render_template_block(t) for t in sorted(templates, key=lambda t: t.__name__) + _system_template_block(t) for t in sorted(templates, key=lambda t: t.__name__) ) sections = [ f"## Agent `{_get_qualname(type(inst))}`\n\n{agent_doc}" if agent_doc else "", @@ -484,7 +484,7 @@ def _render_agent_block(template: Template) -> str: return "\n\n".join(s for s in sections if s) -def _render_module_block(mod: types.ModuleType | None) -> str: +def _system_module_block(mod: types.ModuleType | None) -> str: """Markdown section with the source (or docstring fallback) of a module.""" if mod is None: return "" @@ -496,7 +496,7 @@ def _render_module_block(mod: types.ModuleType | None) -> str: return f"## Module `{mod.__name__}`\n\n{doc}" if doc else "" -def _render_global_block(tool_types: collections.abc.Set[type[Tool]]) -> str: +def _system_global_block(tool_types: collections.abc.Set[type[Tool]]) -> str: """Constant framework-concept prefix, sourced from real docstrings.""" import effectful.handlers.llm as _llm @@ -519,11 +519,11 @@ def call_system( ) -> Message: """Assemble and install the system message (a Markdown document).""" sections = [ - _render_global_block(tool_types), - _render_module_block(inspect.getmodule(template)), - _render_agent_block(template), - _render_imports_block(template.__context__), - _render_vars_block(template.__context__), + _system_global_block(tool_types), + _system_module_block(inspect.getmodule(template)), + _system_agent_block(template), + _system_imports_block(template.__context__), + _system_vars_block(template.__context__), ] content = "\n\n".join(s for s in sections if s) message = _make_message(dict(role="system", content=content)) @@ -656,8 +656,13 @@ class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): (see its spec below). The harness applies that function to the original inputs and its return value becomes the answer, so write the function body as a drop-in implementation of the Template. The function may reference - names from the lexical scope (see the *Lexical scope* table). Calling this - tool terminates the completion. + names from the lexical scope (see the *Lexical scope* table). + + Give the function a docstring containing `>>>` doctests that demonstrate + its intended behavior on examples. On submission the harness runs those + doctests: a solution whose doctests fail (or that errors when applied) is + rejected and fed back to you to revise, so the answer only stands once the + function's own doctests pass. Calling this tool terminates the completion. """ __toolname__: typing.ClassVar[typing.Literal["submit_solution"]] = ( diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 30d595ab2..d3567f9f2 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -155,15 +155,24 @@ class Template[**P, T](Tool[P, T]): ## Constructing Templates - Templates are constructed by calling `Template.define`. - `Template.define` should be used as a decorator on a function or method. - The function must be fully type-annotated and have a docstring. - The body of the function must contain only `raise NotHandled`. - See `effectful.ops.types.Operation.define` for more information on the use of `Template.define`. + Apply `Template.define` as a decorator to a fully type-annotated function or + method whose body is `raise NotHandled`. The docstring is a + [format string](https://docs.python.org/3/library/string.html#format-string-syntax) + prompt: its `{...}` fields are filled at call time (see *Prompt assembly* + below) and the LLM's response is decoded to the return type. - The template docstring is a [format string](https://docs.python.org/3/library/string.html#format-string-syntax), - which may refer to the template arguments. - When the template is called, the arguments and docstring are formatted into a prompt for the LLM and the LLM's response is returned. + `Template.define` validates the definition and raises if: + + - the function has no docstring (every `Tool` needs one); + - a `{...}` field names something that is neither a parameter nor a name in + lexical scope — every field must resolve at call time; + - a doctest example (`>>>`) in the docstring contains an active `{...}` field: + doctests must be constant, since the whole docstring is formatted into the + prompt at call time; escape any literal braces as `{{` and `}}`; + - the `IsRecursive` annotation is applied to a parameter rather than the + return type. + + See `effectful.ops.types.Operation.define` for more on `Template.define`. The following template writes limericks on a given theme: @@ -204,9 +213,36 @@ class Template[**P, T](Tool[P, T]): ## Using tools - Instances of `Tool` that are in the lexical scope of a `Template` may be called by the LLM during template completion. - Templates are themselves tools which enables the construction of complex agent workflows. - When a method is defined as a template, other methods on the class that are decorated with `Tool.define` or `Template.define` are provided to the template as tools. + Instances of `Tool` in a `Template`'s lexical scope may be called by the LLM + during completion, and are offered automatically. Scope follows ordinary + Python rules: enclosing-function locals, module globals, and — for a method + template — sibling `Tool`/`Template` methods on the same class. A template + cannot call a tool it cannot lexically see, so it should use only tools that + are in scope and relevant to the task. Templates are themselves tools, + enabling composition into agent workflows. + + ## Prompt assembly + + A call produces two messages. The **system message** is assembled once per + conversation, ordered most-constant-first so it caches well. Its sections, in + order: + + | # | Section heading | Content | Constant over | + | - | --------------- | ------- | ------------- | + | 1 | `## Template` / `Tool` / `Agent` / `Encodable` (+ any handler blocks) | Framework concepts — sourced from these class docstrings | the process | + | 2 | `## Module ` | Source of the template's module (docstring if source is unavailable) | the module | + | 3 | `## Agent ` + `## Templates` | Agent docstring, then a `### ` spec — prompt with `{...}` holes intact and argument JSON schemas — for every template sharing the instance's history (an `Agent`'s methods, or just this template) | the instance | + | 4 | `## Imported modules` | Table of in-scope imports (name → module) | the scope | + | 5 | `## Lexical scope` | Table of other in-scope bindings (name → type) | the scope | + + The **user message** is the per-call part — only its changing values are + re-sent each turn; everything constant lives in the system message above. It + has two parts: + + | # | Part | Content | + | - | ---- | ------- | + | 1 | Header | `` — identifies which template this turn calls | + | 2 | Body | The docstring with each `{...}` hole replaced by the encoded value of that argument or in-scope name (non-text values, such as images, as separate content blocks) | """ @@ -428,6 +464,37 @@ def send(self, user_input: str) -> str: chatbot.send("Remind me again, where am I?") # sees prior context ``` + ## Encapsulation via lexical scope + + Since scope is ordinary Python scope, defining agents inside a function + partitions their `Template`s and `Tool`s into disjoint sets: + + ```python + class Chatbot(Agent): + @Template.define + def respond(self, user_query: str) -> str: ... + + class TravelAdvisor(Agent): + @Template.define + def recommend(self, user_query: str) -> str: ... + @Tool.define + def search_weather(self, city: str) -> str: ... + + def main(): + chatbot, advisor = Chatbot(), TravelAdvisor() + + @Template.define + def simulate(chatbot, advisor) -> str: + \"""Use {chatbot} and {advisor} to simulate a conversation.\""" + ... + ``` + + `chatbot.respond` sees only its own methods (plus module-level definitions), + not `advisor`'s; `simulate` sees `chatbot` and `advisor`, but they cannot see + `simulate`. Inlining these definitions into module scope instead would let + every template see every other. Agents that need overlapping toolsets should + share tools through a common base class or mixin rather than redefining them. + """ @functools.cached_property From cc0b742646cf1bdd68ef66dc650796d67861f9e6 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 1 Jul 2026 12:18:09 -0400 Subject: [PATCH 024/155] Tweak system prompt --- docs/source/codeadapt_agent.py | 12 ++ effectful/handlers/llm/__init__.py | 3 +- effectful/handlers/llm/completions.py | 154 +++++++++++++++++++++----- effectful/handlers/llm/template.py | 14 +-- tests/test_handlers_llm_template.py | 4 +- 5 files changed, 152 insertions(+), 35 deletions(-) diff --git a/docs/source/codeadapt_agent.py b/docs/source/codeadapt_agent.py index 5e0d2419d..c39a67777 100644 --- a/docs/source/codeadapt_agent.py +++ b/docs/source/codeadapt_agent.py @@ -15,6 +15,7 @@ import argparse import contextlib import os +import pathlib import tenacity @@ -26,6 +27,7 @@ PythonRepl, RetryLLMHandler, SynthesizeAndCall, + SystemPromptDumper, TerminalRenderer, ) from effectful.handlers.llm.evaluation import UnsafeEvalProvider @@ -121,10 +123,20 @@ def main(args: argparse.Namespace) -> None: action="store_true", help="Live-render the streaming message history in the terminal", ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) args = parser.parse_args() with ( handler(LiteLLMProvider(model=args.model, tool_choice="required")), handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), handler(UnsafeEvalProvider()), handler(PythonRepl()), handler(SynthesizeAndCall()), diff --git a/effectful/handlers/llm/__init__.py b/effectful/handlers/llm/__init__.py index d07a1f135..c8f2e5316 100644 --- a/effectful/handlers/llm/__init__.py +++ b/effectful/handlers/llm/__init__.py @@ -47,6 +47,7 @@ observed, logged, or overridden by installing additional handlers. """ +from .encoding import Encodable from .template import Agent, Template, Tool -__all__ = ["Agent", "Template", "Tool"] +__all__ = ["Agent", "Template", "Tool", "Encodable"] diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index dc0ce6f14..b7b28b519 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -7,6 +7,8 @@ import functools import inspect import json +import pathlib +import re import time import traceback import types @@ -397,6 +399,75 @@ def _get_qualname(cls) -> str: return name if module in (None, "builtins") else f"{module}.{name}" +# Matches an ATX heading's leading ``#``s (1-6, followed by whitespace) at the +# start of a line, e.g. ``## Foo``. The lookahead avoids matching ``#!`` or a +# ``#tag`` that is not a heading. +_ATX_HEADING = re.compile(r"^(#{1,6})(?=\s)") + + +def _shift_headings(md: str, by: int) -> str: + """Shift every ATX heading in `md` by `by` levels (clamped to 1..6). + + Fenced code blocks (``` ``` ``` / ``` ~~~ ```) are skipped so ``#`` inside code -- + Python comments, shell shebangs -- is left untouched. + """ + if by == 0 or not md: + return md + out: list[str] = [] + fence: str | None = None + for line in md.splitlines(): + stripped = line.lstrip() + if fence is None and (stripped.startswith("```") or stripped.startswith("~~~")): + fence = stripped[:3] + elif fence is not None and stripped.startswith(fence): + fence = None + elif fence is None: + m = _ATX_HEADING.match(line) + if m: + level = max(1, min(6, len(m.group(1)) + by)) + line = "#" * level + line[m.end(1) :] + out.append(line) + return "\n".join(out) + + +def _rebase_headings(md: str, top: int) -> str: + """Renumber the headings in `md` so its shallowest one sits at level `top`, + preserving relative nesting; text with no headings is returned unchanged. + + Used to nest a docstring that was authored with its own ``##``-rooted + heading hierarchy beneath a deeper section heading when the system prompt is + assembled, so the composed document has a single coherent outline. + """ + if not md: + return md + fence: str | None = None + levels: list[int] = [] + for line in md.splitlines(): + stripped = line.lstrip() + if fence is None and (stripped.startswith("```") or stripped.startswith("~~~")): + fence = stripped[:3] + elif fence is not None and stripped.startswith(fence): + fence = None + elif fence is None: + m = _ATX_HEADING.match(line) + if m: + levels.append(len(m.group(1))) + if not levels: + return md + return _shift_headings(md, top - min(levels)) + + +def _section(title: str, body: str) -> str: + """Wrap `body` as a top-level ``# title`` section, or ``""`` if body is empty. + + Callers pass a `body` whose own headings already start at ``##`` (rebasing + incorporated docstrings with `_rebase_headings` as needed), so every section + is a self-contained subtree rooted at its ``#`` heading. + """ + body = body.strip() + return f"# {title}\n\n{body}" if body else "" + + def _system_vars_block(env: collections.abc.Mapping[str, typing.Any]) -> str: """Markdown table of the non-module bindings in scope (name -> type). @@ -413,7 +484,7 @@ def _system_vars_block(env: collections.abc.Mapping[str, typing.Any]) -> str: if not rows: return "" body = "\n".join(f"| `{n}` | `{t}` |" for n, t in sorted(rows.items())) - return f"## Lexical scope\n\n| name | type |\n| --- | --- |\n{body}" + return _section("Lexical scope", f"| name | type |\n| --- | --- |\n{body}") def _system_imports_block(env: collections.abc.Mapping[str, typing.Any]) -> str: @@ -431,12 +502,16 @@ def _system_imports_block(env: collections.abc.Mapping[str, typing.Any]) -> str: if not rows: return "" body = "\n".join(f"| `{n}` | `{m}` |" for n, m in sorted(rows.items())) - return f"## Imported modules\n\n| name | module |\n| --- | --- |\n{body}" + return _section("Imported modules", f"| name | module |\n| --- | --- |\n{body}") def _system_template_block(template: Template) -> str: - """Markdown spec for a single `Template`: header, prompt, arg schemas.""" - parts = [f"### `{template.__name__}{template.__signature__}`"] + """Markdown spec for a single `Template`: header, prompt, arg schemas. + + Emitted at ``##`` so each template reads as a subsection of the enclosing + agent/template ``#`` section (see `_system_agent_block`). + """ + parts = [f"## `{template.__name__}{template.__signature__}`"] prompt = inspect.getdoc(template.__default__) or "" if prompt: parts.append(prompt) @@ -451,8 +526,9 @@ def _system_template_block(template: Template) -> str: def _system_agent_block(template: Template) -> str: - """One lexical inventory plus the spec of every Template - sharing the current history (an Agent's methods, or just ``template``).""" + """The ``#`` section for the task: the Agent's docstring (if any) followed by + a ``##`` spec for every Template sharing the current history (an Agent's + methods, or just ``template`` for a free-function template).""" inst = ( template.__default__.__self__ if isinstance(template.__default__, types.MethodType) @@ -460,6 +536,7 @@ def _system_agent_block(template: Template) -> str: ) if isinstance(inst, Agent): agent_doc = inspect.getdoc(type(inst)) or "" + title = f"Agent `{_get_qualname(type(inst))}`" templates = set() for cls in type(inst).__mro__: for attr in vars(cls): @@ -471,46 +548,55 @@ def _system_agent_block(template: Template) -> str: templates.add(value) else: agent_doc = "" + title = "Template" templates = {template} # Order by name so the prompt is stable across method reordering in source. specs = "\n\n".join( _system_template_block(t) for t in sorted(templates, key=lambda t: t.__name__) ) - sections = [ - f"## Agent `{_get_qualname(type(inst))}`\n\n{agent_doc}" if agent_doc else "", - f"## Templates\n\n{specs}", - ] - return "\n\n".join(s for s in sections if s) + # The agent docstring is intro prose for the section; rebase its own headings + # to sit at ``##`` alongside the per-template specs. + body = "\n\n".join(p for p in [_rebase_headings(agent_doc, 2), specs] if p) + return _section(title, body) def _system_module_block(mod: types.ModuleType | None) -> str: - """Markdown section with the source (or docstring fallback) of a module.""" + """The ``#`` section carrying the source (or docstring fallback) of a module.""" if mod is None: return "" try: src = inspect.getsource(mod) - return f"## Module `{mod.__name__}`\n\n```python\n{src}\n```" + body = f"```python\n{src}\n```" except (OSError, TypeError): doc = inspect.getdoc(mod) - return f"## Module `{mod.__name__}`\n\n{doc}" if doc else "" + if not doc: + return "" + body = _rebase_headings(doc, 2) + return _section(f"Module `{mod.__name__}`", body) def _system_global_block(tool_types: collections.abc.Set[type[Tool]]) -> str: - """Constant framework-concept prefix, sourced from real docstrings.""" + """The constant ``#`` framework-concept section, sourced from real docstrings. + + The module overview and each concept nest as ``##`` subsections. Core + concept classes carry a synthesized ``## `Name``` heading and their own + docstring subsections are demoted to ``###``; the synthetic tool docstrings + already open with a descriptive ``##`` heading, so they are used verbatim + (rebased if needed) rather than labelled with their private class names. + """ import effectful.handlers.llm as _llm assert all(issubclass(t, Tool) and t not in {Tool, Template} for t in tool_types) - parts = [inspect.getdoc(_llm) or ""] - for obj in [ - Template, - Tool, - Agent, - Encodable, - *sorted(tool_types, key=_get_qualname), - ]: - parts += [f"## `{obj.__name__}`\n\n{inspect.getdoc(obj)}"] # type: ignore[attr-defined] - return "\n\n".join(p for p in parts if p) + parts = [_rebase_headings(inspect.getdoc(_llm) or "", 2)] + for typ in sorted(map(lambda name: getattr(_llm, name), _llm.__all__), key=_get_qualname): + parts += [ + f"## `{_get_qualname(typ)}`\n\n{_rebase_headings(inspect.getdoc(typ) or '', 3)}" + ] + for t in sorted(tool_types, key=_get_qualname): + parts += [_rebase_headings(inspect.getdoc(t) or "", 2)] + body = "\n\n".join(p for p in parts if p.strip()) + return _section("The effectful LLM framework", body) @Operation.define @@ -1327,6 +1413,24 @@ def _completion(self, *args, **kwargs) -> typing.Any: return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages")) +@dataclasses.dataclass(frozen=True) +class SystemPromptDumper(ObjectInterpretation): + """Dump the system prompt produced by `call_system` to a Markdown file. + + Opt-in debugging handler: intercepts `call_system`, forwards to let the + prompt be assembled and installed as usual, then writes the resulting + system message content to `path`, overwriting the whole file each time. + """ + + path: pathlib.Path + + @implements(call_system) + def _call_system(self, template, tool_types=frozenset()): + message = fwd() + self.path.write_text(_message_text(message.get("content"))) + return message + + class LiteLLMProvider(ObjectInterpretation): """Implements templates using the LiteLLM API.""" diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index d3567f9f2..591553e4c 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -224,16 +224,16 @@ class Template[**P, T](Tool[P, T]): ## Prompt assembly A call produces two messages. The **system message** is assembled once per - conversation, ordered most-constant-first so it caches well. Its sections, in - order: + conversation, ordered most-constant-first so it caches well. Each section is a + top-level `#` heading whose contents nest beneath it; in order: | # | Section heading | Content | Constant over | | - | --------------- | ------- | ------------- | - | 1 | `## Template` / `Tool` / `Agent` / `Encodable` (+ any handler blocks) | Framework concepts — sourced from these class docstrings | the process | - | 2 | `## Module ` | Source of the template's module (docstring if source is unavailable) | the module | - | 3 | `## Agent ` + `## Templates` | Agent docstring, then a `### ` spec — prompt with `{...}` holes intact and argument JSON schemas — for every template sharing the instance's history (an `Agent`'s methods, or just this template) | the instance | - | 4 | `## Imported modules` | Table of in-scope imports (name → module) | the scope | - | 5 | `## Lexical scope` | Table of other in-scope bindings (name → type) | the scope | + | 1 | `# The effectful LLM framework` | Framework concepts — the package overview plus a `##` subsection per concept (`Template`, `Tool`, `Agent`, `Encodable`, and any handler tool blocks), sourced from these docstrings | the process | + | 2 | `# Module ` | Source of the template's module (docstring if source is unavailable) | the module | + | 3 | `# Agent ` (or `# Template`) | Agent docstring, then a `## ` spec — prompt with `{...}` holes intact and argument JSON schemas — for every template sharing the instance's history (an `Agent`'s methods, or just this template) | the instance | + | 4 | `# Imported modules` | Table of in-scope imports (name → module) | the scope | + | 5 | `# Lexical scope` | Table of other in-scope bindings (name → type) | the scope | The **user message** is the per-call part — only its changing values are re-sent each turn; everything constant lives in the system message above. It diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index b6e2ebd15..1b8d5bfdf 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -489,10 +489,10 @@ def act(self) -> str: assert MissingDocAgent.__doc__ is None content = self._system_content(MissingDocAgent().act) # No subclass docstring -> the Agent base-class docstring is used as the - # tier-3 "## Agent" section (inspect.getdoc walks the MRO). + # tier-3 "# Agent" section (inspect.getdoc walks the MRO). agent_doc = inspect.getdoc(Agent) assert agent_doc is not None - assert "## Agent `MissingDocAgent`" in content + assert "# Agent `MissingDocAgent`" in content assert agent_doc in content def test_non_empty_docstring_overrides_inherited_doc(self): From d515a416f73b218284ed0812aa23fa647391b2a1 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 1 Jul 2026 13:47:14 -0400 Subject: [PATCH 025/155] Remove IsRecursive --- effectful/handlers/llm/completions.py | 25 ++- effectful/handlers/llm/template.py | 90 ++--------- tests/conftest.py | 32 ++-- tests/test_handlers_llm.py | 220 -------------------------- tests/test_handlers_llm_template.py | 201 +++++++++++++++++------ 5 files changed, 203 insertions(+), 365 deletions(-) delete mode 100644 tests/test_handlers_llm.py diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index b7b28b519..4f9d9e8fd 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -248,8 +248,8 @@ def call_assistant[T]( The available `tools` are passed explicitly as a set; handlers that expose additional tools (synthetic readers, REPL access, synthesis) intercept this operation and union them into `tools` before forwarding. Each tool's - model-visible name is derived from its `__name__` (see :func:`_name_tools`), - so collection and decoding agree on a single naming scheme. + model-visible name is derived from its `__name__`, so collection and + decoding agree on a single naming scheme. Raises: ToolCallDecodingError: If a tool call cannot be decoded. The error @@ -376,13 +376,22 @@ def call_tool[T](tool_call: DecodedToolCall[T]) -> ToolResult[T]: @Operation.define def call_user( - template: str, + template: Template, env: collections.abc.Mapping[str, typing.Any], ) -> Message: """ - Format a template applied to arguments into a user message. + Format a `Template`'s prompt applied to arguments into a user message. + + The prompt is the template's header (``name(signature)``, with braces + escaped so it is not itself formatted) followed by its docstring; its + ``{...}`` fields are filled from `env`. """ - parts = format_as_content_blocks(template, env) + assert template.__default__.__doc__ is not None + header = f"{template.__name__}{template.__signature__}".replace("{", "{{").replace( + "}", "}}" + ) + prompt = f"{header}\n\n{template.__default__.__doc__}" + parts = format_as_content_blocks(prompt, env) message = _make_message(dict(role="user", content=parts)) append_message(message) return message @@ -589,7 +598,9 @@ def _system_global_block(tool_types: collections.abc.Set[type[Tool]]) -> str: assert all(issubclass(t, Tool) and t not in {Tool, Template} for t in tool_types) parts = [_rebase_headings(inspect.getdoc(_llm) or "", 2)] - for typ in sorted(map(lambda name: getattr(_llm, name), _llm.__all__), key=_get_qualname): + for typ in sorted( + map(lambda name: getattr(_llm, name), _llm.__all__), key=_get_qualname + ): parts += [ f"## `{_get_qualname(typ)}`\n\n{_rebase_headings(inspect.getdoc(typ) or '', 3)}" ] @@ -1469,7 +1480,7 @@ def _call[**P, T]( ): message: Message = call_system(template) - message = call_user(template.__prompt_template__, env) + message = call_user(template, env) # loop based on: https://cookbook.openai.com/examples/reasoning_function_calls result: T | None = None diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 591553e4c..e4870ffda 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -1,4 +1,5 @@ import abc +import collections import doctest import functools import inspect @@ -6,61 +7,9 @@ import string import types import typing -from collections import ChainMap, OrderedDict from collections.abc import Callable, Mapping, MutableMapping -from typing import Annotated, Any -from effectful.ops.types import Annotation, Operation - - -class _IsRecursiveAnnotation(Annotation): - """ - A special type annotation for return types in the signature of a - :class:`Template` that indicates it may make recursive calls. - - .. warning:: - - :class:`IsRecursive` annotations are only defined to ascribe - return annotations, and if used in a parameter will raise a - :class:`TypeError` at tool construction time. - - - - **Example usage**: - - We illustrate the use of :class:`IsRecursive` below: - - >>> from typing import Annotated - >>> from effectful.handlers.llm import Template - >>> from effectful.handlers.llm.template import IsRecursive - - >>> - @Template.define - def factorial(n: int) -> Annotated[int, IsRecursive]: - \"""Compute the n factorial for n={n}. Can call itself (`factorial`) recursively, but must be on smaller arguments.\""" - raise NotHandled - """ - - @classmethod - def infer_annotations(cls, sig: inspect.Signature) -> inspect.Signature: - for name, ty in sig.parameters.items(): - if not ty or not typing.get_origin(ty) is Annotated: - continue - if any(isinstance(arg, cls) for arg in typing.get_args(ty)): - raise TypeError( - f"Illegal annotation {ty} for parameter {name}, IsRecursive must only be used to annotate return types." - ) - return sig - - -IsRecursive = _IsRecursiveAnnotation() - - -def _is_recursive_signature(sig: inspect.Signature): - if typing.get_origin(sig.return_annotation) is not Annotated: - return False - annotations = typing.get_args(sig.return_annotation) - return any(annotation is IsRecursive for annotation in annotations) +from effectful.ops.types import Operation class Tool[**P, T](Operation[P, T]): @@ -104,10 +53,6 @@ def __init__(self, default: Callable[P, T], name: str | None = None): raise ValueError("Tools must have docstrings.") super().__init__(default, name=name) - @property - def __signature__(self): - return IsRecursive.infer_annotations(super().__signature__) - @classmethod def define(cls, *args, **kwargs) -> "Tool[P, T]": """Define a tool. @@ -168,9 +113,7 @@ class Template[**P, T](Tool[P, T]): lexical scope — every field must resolve at call time; - a doctest example (`>>>`) in the docstring contains an active `{...}` field: doctests must be constant, since the whole docstring is formatted into the - prompt at call time; escape any literal braces as `{{` and `}}`; - - the `IsRecursive` annotation is applied to a parameter rather than the - return type. + prompt at call time; escape any literal braces as `{{` and `}}`. See `effectful.ops.types.Operation.define` for more on `Template.define`. @@ -246,7 +189,7 @@ class Template[**P, T](Tool[P, T]): """ - __context__: ChainMap[str, Any] + __context__: collections.ChainMap[str, typing.Any] @classmethod def _validate_doctests_constant(cls, template: "Template", doc: str) -> None: @@ -298,7 +241,7 @@ def _validate_doctests_constant(cls, template: "Template", doc: str) -> None: def _validate_prompt( cls, template: "Template", - context: ChainMap[str, Any], + context: collections.ChainMap[str, typing.Any], ) -> None: """Validate that all format string variables in the docstring refer to names resolvable at call time. @@ -311,7 +254,8 @@ def _validate_prompt( :raises TypeError: If any format string variable cannot be resolved, or a format field is spliced into a doctest example. """ - doc = template.__prompt_template__ + assert template.__doc__ is not None + doc = template.__doc__ cls._validate_doctests_constant(template, doc) formatter = string.Formatter() param_names = set(template.__signature__.parameters.keys()) @@ -335,14 +279,6 @@ def _validate_prompt( f"{{{template.__signature__}}} or lexical scope." ) - @property - def __prompt_template__(self) -> str: - assert self.__default__.__doc__ is not None - header = f"{self.__name__}{self.__signature__}".replace("{", "{{").replace( - "}", "}}" - ) - return f"{header}\n\n{self.__default__.__doc__}" - def __get__[S](self, instance: S | None, owner: type[S] | None = None): if hasattr(self, "_name_on_instance") and hasattr( instance, self._name_on_instance @@ -399,10 +335,10 @@ def define[**Q, V]( ] enclosing_fns.reverse() # innermost first for frame walking - globals_proxy: types.MappingProxyType[str, Any] = types.MappingProxyType( + globals_proxy: types.MappingProxyType[str, typing.Any] = types.MappingProxyType( frame.f_globals ) - contexts: list[types.MappingProxyType[str, Any]] = [] + contexts: list[types.MappingProxyType[str, typing.Any]] = [] for fn_name in enclosing_fns: while frame is not None and frame.f_locals is not frame.f_globals: if frame.f_code.co_name == fn_name: @@ -411,8 +347,8 @@ def define[**Q, V]( break frame = frame.f_back contexts.append(globals_proxy) - context: ChainMap[str, Any] = ChainMap( - *typing.cast(list[MutableMapping[str, Any]], contexts) + context: collections.ChainMap[str, typing.Any] = collections.ChainMap( + *typing.cast(list[MutableMapping[str, typing.Any]], contexts) ) op = super().define(default, *args, **kwargs) op.__context__ = context # type: ignore[attr-defined] @@ -498,5 +434,5 @@ def simulate(chatbot, advisor) -> str: """ @functools.cached_property - def __history__(self) -> OrderedDict[str, Mapping[str, Any]]: - return OrderedDict() + def __history__(self) -> collections.OrderedDict[str, Mapping[str, typing.Any]]: + return collections.OrderedDict() diff --git a/tests/conftest.py b/tests/conftest.py index 52e3456b8..f59a069b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,31 +40,35 @@ def pytest_runtest_call(item): def offered_tools(env, *handlers): - """Name -> Tool mapping the model would be offered for lexical scope `env` + """Set of Tools the model would be offered for lexical scope `env` under the given handlers. Replaces the old ``collect_tools`` operation: tool collection now happens as `call_assistant` seeds its `tools` set from :func:`_tools_in_scope` and the augmenting handlers (``LexicalReaders``, ``PythonRepl``, ...) union more in. This installs a capture handler that records the tools `call_assistant` - ultimately receives, named by :func:`_name_tools`. + ultimately receives. + + Tools are kept by object identity, not by name: two distinct tools that + share a ``__name__`` (e.g. the same method bound to different instances) + are both preserved. Callers checking name presence should compare against + ``{t.__name__ for t in offered_tools(...)}``. """ import contextlib from effectful.handlers.llm.completions import ( - _name_tools, _tools_in_scope, call_assistant, ) from effectful.ops.semantics import handler from effectful.ops.syntax import ObjectInterpretation, implements - captured: dict = {} + captured: set = set() class _Capture(ObjectInterpretation): @implements(call_assistant) def _ca(self, env_, response_type, tools=frozenset(), **kw): - captured.update(_name_tools(tools)) + captured.update(tools) return ({}, [], None) with contextlib.ExitStack() as stack: @@ -76,19 +80,13 @@ def _ca(self, env_, response_type, tools=frozenset(), **kw): def template_tools(template, *handlers): - """Name -> Tool mapping a `Template` would offer under the given handlers. + """Set of Tools a `Template` would offer under the given handlers. Mirrors the behaviour of the removed ``Template.tools`` property: it applies the same handler augmentation as :func:`offered_tools` and drops the template - itself unless its signature is recursive. + itself, matching `LiteLLMProvider`'s ``_tools_in_scope(env) - {template}``. + Like :func:`offered_tools`, tools are kept by object identity. """ - import collections - - from effectful.handlers.llm.template import _is_recursive_signature - - result = offered_tools(template.__context__, *handlers) - if not _is_recursive_signature(template.__signature__): - result = collections.OrderedDict( - (n, t) for n, t in result.items() if t is not template - ) - return result + return { + t for t in offered_tools(template.__context__, *handlers) if t is not template + } diff --git a/tests/test_handlers_llm.py b/tests/test_handlers_llm.py deleted file mode 100644 index 9b5354a7b..000000000 --- a/tests/test_handlers_llm.py +++ /dev/null @@ -1,220 +0,0 @@ -from collections.abc import Callable -from typing import Annotated - -from effectful.handlers.llm import Template -from effectful.handlers.llm.template import IsRecursive -from effectful.ops.semantics import NotHandled, handler -from effectful.ops.syntax import ObjectInterpretation, implements -from tests.conftest import template_tools - - -class SingleResponseLLMProvider[T](ObjectInterpretation): - """Simplified mock provider that returns a single response for any prompt.""" - - def __init__(self, response: T): - """Initialize with a single response string. - - Args: - response: The response to return for any template call - """ - self.response = response - - @implements(Template.__apply__) - def _call[**P]( - self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs - ) -> T: - return self.response - - -# Test templates from the notebook examples -@Template.define -def limerick(theme: str) -> str: - """Write a limerick on the theme of {theme}.""" - raise NotHandled - - -@Template.define -def haiku(theme: str) -> str: - """Write a haiku on the theme of {theme}.""" - raise NotHandled - - -@Template.define -def primes(first_digit: int) -> int: - """Give exactly one prime number with {first_digit} as the first digit. Respond with only the number.""" - raise NotHandled - - -@Template.define -def count_char(char: str) -> Callable[[str], int]: - """Write a function which takes a string and counts the occurrances of '{char}'.""" - raise NotHandled - - -# Mutually recursive templates (module-level for live globals) -@Template.define -def mutual_a() -> Annotated[str, IsRecursive]: - """Use mutual_a and mutual_b as tools to do task A.""" - raise NotHandled - - -@Template.define -def mutual_b() -> Annotated[str, IsRecursive]: - """Use mutual_a and mutual_b as tools to do task B.""" - raise NotHandled - - -def test_primes_decode_int(): - """Test the primes template correctly decodes integer response.""" - mock_provider = SingleResponseLLMProvider(61) - - with handler(mock_provider): - result = primes(6) - assert result == 61 - assert isinstance(result, int) - - -class FailingThenSucceedingProvider[T](ObjectInterpretation): - """Mock provider that fails a specified number of times before succeeding.""" - - def __init__( - self, - fail_count: int, - success_response: T, - exception_factory: Callable[[], Exception], - ): - """Initialize the provider. - - Args: - fail_count: Number of times to fail before succeeding - success_response: Response to return after failures - exception_factory: Factory function that creates exceptions to raise - """ - self.fail_count = fail_count - self.success_response = success_response - self.exception_factory = exception_factory - self.call_count = 0 - - @implements(Template.__apply__) - def _call[**P]( - self, template: Template[P, T], *args: P.args, **kwargs: P.kwargs - ) -> T: - self.call_count += 1 - if self.call_count <= self.fail_count: - raise self.exception_factory() - return self.success_response - - -def test_template_captures_other_templates_in_lexical_context(): - """Test that Templates defined in lexical scope are captured (orchestrator pattern).""" - - # Define sub-templates first - @Template.define - def story_with_moral(topic: str) -> str: - """Write a story about {topic} with a moral lesson.""" - raise NotHandled - - @Template.define - def story_funny(topic: str) -> str: - """Write a funny story about {topic}.""" - raise NotHandled - - # Main orchestrator template has access to sub-templates - @Template.define - def write_story(topic: str, style: str) -> str: - """Write a story about {topic} in style {style}.""" - raise NotHandled - - # __context__ is a ChainMap(locals, globals) - locals shadow globals - # Sub-templates should be visible in lexical context - assert "story_with_moral" in write_story.__context__ - assert "story_funny" in write_story.__context__ - assert write_story.__context__["story_with_moral"] is story_with_moral - assert write_story.__context__["story_funny"] is story_funny - - # Templates in lexical context are exposed as callable tools - assert story_with_moral in template_tools(write_story).values() - assert story_funny in template_tools(write_story).values() - - -def test_template_composition_with_chained_calls(): - """Test calling one template and passing result to another.""" - - @Template.define - def generate_topic() -> str: - """Generate an interesting topic for a story.""" - raise NotHandled - - @Template.define - def write_story(topic: str) -> str: - """Write a short story about {topic}.""" - raise NotHandled - - # Verify generate_topic is in write_story's lexical context - assert "generate_topic" in write_story.__context__ - - # Test chained template calls - mock_provider = SingleResponseLLMProvider("A magical forest") - - with handler(mock_provider): - topic = generate_topic() - assert topic == "A magical forest" - - # Now use that topic in the next template - mock_provider2 = SingleResponseLLMProvider( - "Once upon a time in a magical forest..." - ) - - with handler(mock_provider2): - story = write_story(topic) - assert story == "Once upon a time in a magical forest..." - - -def test_mutually_recursive_templates(): - """Test that module-level templates can see each other (mutual recursion).""" - # Both mutual_a and mutual_b should see each other via ChainMap (globals visible) - assert "mutual_a" in mutual_a.__context__ - assert "mutual_b" in mutual_a.__context__ - assert "mutual_a" in mutual_b.__context__ - assert "mutual_b" in mutual_b.__context__ - - # They should also be in each other's tools - assert mutual_a in template_tools(mutual_b).values() - assert mutual_b in template_tools(mutual_a).values() - # And themselves (self-recursion) - assert mutual_a in template_tools(mutual_a).values() - assert mutual_b in template_tools(mutual_b).values() - - -# Module-level variable for shadowing test -shadow_test_value = "global" - - -def test_lexical_context_shadowing(): - """Test that local variables shadow global variables in lexical context.""" - # Local shadows global - shadow_test_value = "local" # noqa: F841 - intentional shadowing - - @Template.define - def template_with_shadowed_var() -> str: - """Test template.""" - raise NotHandled - - # The lexical context should see the LOCAL value, not global - assert "shadow_test_value" in template_with_shadowed_var.__context__ - assert ( - template_with_shadowed_var.__context__["shadow_test_value"] == shadow_test_value - ) - - -def test_lexical_context_sees_globals_when_no_local(): - """Test that globals are visible when there's no local shadow.""" - - @Template.define - def template_sees_global() -> str: - """Test template.""" - raise NotHandled - - # Should see the global value (no local shadow in this scope) - assert "shadow_test_value" in template_sees_global.__context__ - assert template_sees_global.__context__["shadow_test_value"] == "global" diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 1b8d5bfdf..754f3e419 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -36,7 +36,7 @@ def _[**P, T]( bound_args = inspect.signature(template).bind(*args, **kwargs) bound_args.apply_defaults() env = template.__context__.new_child(bound_args.arguments) - model_input = call_user(template.__prompt_template__, env) + model_input = call_user(template, env) template_result = model_input["content"] assert len(template_result) == 1 return template_result[0]["text"] @@ -792,10 +792,10 @@ def f(self) -> int: a = A(0) assert isinstance(a.f, Template) - assert a.random in template_tools(a.f).values() + assert a.random in template_tools(a.f) # f is the template itself — found via self but correctly removed (non-recursive) - assert a.f not in template_tools(a.f).values() - assert any(t() == 4 for t in template_tools(a.f).values() if t is a.random) + assert a.f not in template_tools(a.f) + assert any(t() == 4 for t in template_tools(a.f) if t is a.random) class B(A): """You are a derived template-method test agent. @@ -809,8 +809,8 @@ def reverse(self, s: str) -> str: b = B(1) assert isinstance(b.f, Template) - assert b.random in template_tools(b.f).values() - assert b.reverse in template_tools(b.f).values() + assert b.random in template_tools(b.f) + assert b.reverse in template_tools(b.f) def test_template_method_nested_class(): @@ -837,11 +837,12 @@ def f(self) -> int: a = A.B(True) assert isinstance(a.f, Template) + tools = template_tools(a.f) # random is found via the enclosing function scope - assert "random" in template_tools(a.f) + assert random in tools # f is the template itself — found via self but correctly removed (non-recursive) - assert "f" not in template_tools(a.f) - assert template_tools(a.f)["random"]() == 4 + assert a.f not in tools + assert random() == 4 def test_template_method_module(): @@ -913,7 +914,7 @@ def ask(self) -> str: raise NotHandled bar = Bar() - assert "helper" in template_tools(bar.ask) + assert helper in template_tools(bar.ask) def test_dynamic_caller_not_leaked(self): """Variables from a dynamic caller (not lexical enclosure) should not @@ -946,9 +947,9 @@ def describe(self) -> str: raise NotHandled w = Widget() - assert w.measure in template_tools(w.describe).values() + assert w.measure in template_tools(w.describe) # The template itself is not in tools (non-recursive) - assert w.describe not in template_tools(w.describe).values() + assert w.describe not in template_tools(w.describe) def test_inherited_tools_visible(self): """Tools from a base Agent class are visible through the instance.""" @@ -974,7 +975,7 @@ def ask(self) -> str: raise NotHandled d = Derived() - assert d.base_tool in template_tools(d.ask).values() + assert d.base_tool in template_tools(d.ask) def test_tool_in_enclosing_function_visible_through_class(self): """function -> class -> Template.define: tool in the function is visible.""" @@ -990,7 +991,7 @@ def ask(self) -> str: """Ask something.""" raise NotHandled - assert "outer_tool" in template_tools(Inner().ask) + assert outer_tool in template_tools(Inner().ask) def test_tool_in_enclosing_function_visible_through_nested_classes(self): """function -> class -> class -> Template.define: tool in the function @@ -1008,7 +1009,7 @@ def ask(self) -> str: """Ask something.""" raise NotHandled - assert "outer_tool" in template_tools(Outer.Inner().ask) + assert outer_tool in template_tools(Outer.Inner().ask) def test_nested_function_then_class(self): """function -> function -> class -> Template.define: all enclosing @@ -1026,11 +1027,11 @@ def ask(self) -> str: """Ask.""" raise NotHandled - return MyClass + return MyClass, inner_tool outer_var = True # noqa: F841 - cls = _make() - assert "inner_tool" in template_tools(cls().ask) + cls, inner_tool = _make() + assert inner_tool in template_tools(cls().ask) # The test method is a lexical encloser of _make, so its locals # are visible — matching Python's actual scoping rules. assert "outer_var" in cls().ask.__context__ @@ -1112,7 +1113,7 @@ def ask(x: int) -> int: """Compute {x}.""" raise NotHandled - assert "helper" in template_tools(MyClass.ask) + assert helper in template_tools(MyClass.ask) def test_staticmethod_template_excludes_class_body(self): """A staticmethod Template does not capture class body locals.""" @@ -1249,7 +1250,7 @@ def poem(topic: str, style: str) -> str: """Write a {style} poem about {topic}.""" raise NotHandled - assert poem.__prompt_template__.endswith("Write a {style} poem about {topic}.") + assert poem.__default__.__doc__.endswith("Write a {style} poem about {topic}.") def test_validate_no_vars(): @@ -1260,7 +1261,7 @@ def simple() -> str: """Just a plain prompt with no variables.""" raise NotHandled - assert simple.__prompt_template__.endswith("Just a plain prompt with no variables.") + assert simple.__default__.__doc__.endswith("Just a plain prompt with no variables.") def test_validate_undefined_var(): @@ -1297,7 +1298,7 @@ def greet(self, day: str) -> str: """Agent '{self.name}' says hello on {day}.""" raise NotHandled - assert Agent.greet.__prompt_template__.endswith( + assert Agent.greet.__default__.__doc__.endswith( "Agent '{self.name}' says hello on {day}." ) @@ -1312,7 +1313,7 @@ def ok(a: str, b: str) -> str: raise NotHandled # The underlying Template should exist - assert ok.__func__.__prompt_template__.endswith("Combine {a} and {b}.") + assert ok.__func__.__default__.__doc__.endswith("Combine {a} and {b}.") def test_validate_staticmethod_undefined(): @@ -1366,7 +1367,7 @@ def convert(feet: int) -> float: """How many miles is {feet} feet? There are {feet_per_mile} feet per mile.""" raise NotHandled - assert "feet_per_mile" in convert.__prompt_template__ + assert "feet_per_mile" in convert.__default__.__doc__ def test_validate_both_params_and_lexical(): @@ -1378,7 +1379,7 @@ def write_poem(topic: str) -> str: """Write a poem about {topic} by {author}.""" raise NotHandled - assert write_poem.__prompt_template__.endswith( + assert write_poem.__default__.__doc__.endswith( "Write a poem about {topic} by {author}." ) @@ -1553,7 +1554,7 @@ def dbl(x: int) -> int: """ raise NotHandled - assert "dbl(2)" in dbl.__prompt_template__ + assert "dbl(2)" in dbl.__default__.__doc__ def test_validate_param_spliced_into_doctest_source_rejected(): @@ -1616,7 +1617,7 @@ def make_dict(x: int) -> dict: """ raise NotHandled - assert "make_dict(2)" in make_dict.__prompt_template__ + assert "make_dict(2)" in make_dict.__default__.__doc__ def test_validate_field_in_prose_with_constant_doctest_ok(): @@ -1631,7 +1632,7 @@ def about(theme: str) -> int: """ raise NotHandled - assert "{theme}" in about.__prompt_template__ + assert "{theme}" in about.__default__.__doc__ # Forward ref through Tool subclass of Operation. @@ -1797,7 +1798,8 @@ def test_collect_tools_exposes_callable_shaped_values(name, make_value): handler and become synthesis-shaped tools.""" value = make_value() env = {name: value} - assert name in offered_tools(env, LexicalReaders()) + # The reader for `value` returns it verbatim; identify it by that value. + assert any(t() is value for t in offered_tools(env, LexicalReaders())) def test_lexical_reader_exposes_data_values(): @@ -1811,10 +1813,10 @@ def test_lexical_reader_exposes_data_values(): "d": {"k": 1}, "model": _SimpleModel(x=1, y="hi"), } - result = offered_tools(env, LexicalReaders()) - assert {"x", "s", "lst", "d", "model"} <= set(result) - for k, v in env.items(): - assert result[k]() is v + tools = offered_tools(env, LexicalReaders()) + # Each value is exposed as a reader that returns the very same object. + for v in env.values(): + assert any(t() is v for t in tools) def test_template_tools_includes_synthetic_readers_for_locals(): @@ -1827,18 +1829,21 @@ def t() -> int: """Doc.""" raise NotHandled - tools = template_tools(t, LexicalReaders()) - assert "_test_data" in tools - assert tools["_test_data"]() == [10, 20, 30] + # Restrict to reader tools (safe to call) rather than other in-scope tools. + readers = [ + tool + for tool in template_tools(t, LexicalReaders()) + if isinstance(tool, LexicalReaders._LexicalVariableTool) + ] + assert any(reader() == [10, 20, 30] for reader in readers) def test_lexical_readers_handler_enables_collection(): """Installing `LexicalReaders` flips the gate; the same values are exposed as zero-arg reader tools.""" env = {"x": 42, "s": "hello"} - result = offered_tools(env, LexicalReaders()) - assert result["x"]() == 42 - assert result["s"]() == "hello" + tools = offered_tools(env, LexicalReaders()) + assert {t() for t in tools} == {42, "hello"} # --------------------------------------------------------------------------- @@ -1848,20 +1853,23 @@ def test_lexical_readers_handler_enables_collection(): def test_python_repl_off_by_default(): """Without `PythonRepl`, `exec_code` is not collected.""" - assert "exec_code" not in offered_tools({"x": 1}) + assert PythonRepl().exec_code not in offered_tools({"x": 1}) def test_python_repl_exposes_exec_code(): """With `PythonRepl` installed, `exec_code` is collected alongside the base tools.""" - assert "exec_code" in offered_tools({"x": 1}, PythonRepl()) + repl = PythonRepl() + assert repl.exec_code in offered_tools({"x": 1}, repl) def test_python_repl_composes_with_lexical_readers(): """Readers and the REPL tool coexist when both handlers are installed.""" - result = offered_tools({"data": [1, 2, 3]}, LexicalReaders(), PythonRepl()) - assert "exec_code" in result - assert "data" in result + repl = PythonRepl() + tools = offered_tools({"data": [1, 2, 3]}, LexicalReaders(), repl) + assert repl.exec_code in tools # the REPL tool + readers = [t for t in tools if isinstance(t, LexicalReaders._LexicalVariableTool)] + assert any(reader() == [1, 2, 3] for reader in readers) # the data reader def _drive_repl(body): @@ -1933,3 +1941,108 @@ def outer(exec_code): inner_sees_outer, outer_after = _drive_repl(outer) assert inner_sees_outer == "False\n" # the nested session is isolated assert outer_after == "1 True\n" # the outer session survived the nested call + + +# --------------------------------------------------------------------------- +# Lexical-context capture and decoding (consolidated from test_handlers_llm.py) +# --------------------------------------------------------------------------- + + +@Template.define +def primes(first_digit: int) -> int: + """Give exactly one prime number with {first_digit} as the first digit. Respond with only the number.""" + raise NotHandled + + +# Mutually recursive templates (module-level so globals are live for each other). +@Template.define +def mutual_a() -> str: + """Use mutual_a and mutual_b as tools to do task A.""" + raise NotHandled + + +@Template.define +def mutual_b() -> str: + """Use mutual_a and mutual_b as tools to do task B.""" + raise NotHandled + + +# Module-level variable for the shadowing tests below. +shadow_test_value = "global" + + +def test_primes_decode_int(): + """A non-string return type is decoded from the model's structured output.""" + mock = MockCompletionHandler([make_text_response('{"value": 61}')]) + + with handler(LiteLLMProvider()), handler(mock): + result = primes(6) + + assert result == 61 + assert isinstance(result, int) + + +def test_template_captures_other_templates_in_lexical_context(): + """Templates defined in lexical scope are captured and offered as tools.""" + + @Template.define + def story_with_moral(topic: str) -> str: + """Write a story about {topic} with a moral lesson.""" + raise NotHandled + + @Template.define + def story_funny(topic: str) -> str: + """Write a funny story about {topic}.""" + raise NotHandled + + @Template.define + def write_story(topic: str, style: str) -> str: + """Write a story about {topic} in style {style}.""" + raise NotHandled + + # __context__ is a ChainMap(locals, globals) - sub-templates are visible. + assert write_story.__context__["story_with_moral"] is story_with_moral + assert write_story.__context__["story_funny"] is story_funny + + # Templates in lexical context are exposed as callable tools. + assert story_with_moral in template_tools(write_story) + assert story_funny in template_tools(write_story) + + +def test_mutually_recursive_templates(): + """Module-level templates see each other (mutual recursion) via globals.""" + assert "mutual_a" in mutual_a.__context__ + assert "mutual_b" in mutual_a.__context__ + assert "mutual_a" in mutual_b.__context__ + assert "mutual_b" in mutual_b.__context__ + + # Each sees the other as a callable tool. + assert mutual_a in template_tools(mutual_b) + assert mutual_b in template_tools(mutual_a) + # A template is always dropped from its own toolset. + assert mutual_a not in template_tools(mutual_a) + assert mutual_b not in template_tools(mutual_b) + + +def test_lexical_context_shadowing(): + """Local variables shadow global variables in lexical context.""" + shadow_test_value = "local" # noqa: F841 - intentional shadowing + + @Template.define + def template_with_shadowed_var() -> str: + """Test template.""" + raise NotHandled + + # The lexical context should see the LOCAL value, not global. + assert template_with_shadowed_var.__context__["shadow_test_value"] == "local" + + +def test_lexical_context_sees_globals_when_no_local(): + """Globals are visible when there's no local shadow.""" + + @Template.define + def template_sees_global() -> str: + """Test template.""" + raise NotHandled + + assert template_sees_global.__context__["shadow_test_value"] == "global" From 0ba7274ba143487495a26721ea44738b6ef6c69e Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 1 Jul 2026 14:29:00 -0400 Subject: [PATCH 026/155] include docstring --- docs/source/codeadapt.py | 12 ++++++++++++ effectful/handlers/llm/encoding.py | 1 + 2 files changed, 13 insertions(+) diff --git a/docs/source/codeadapt.py b/docs/source/codeadapt.py index 32c59e4c7..695a0d7d4 100644 --- a/docs/source/codeadapt.py +++ b/docs/source/codeadapt.py @@ -14,6 +14,7 @@ import argparse import contextlib import os +import pathlib from typing import Literal, NamedTuple import tenacity @@ -26,6 +27,7 @@ PythonRepl, RetryLLMHandler, SynthesizeAndCall, + SystemPromptDumper, TerminalRenderer, ) from effectful.handlers.llm.evaluation import UnsafeEvalProvider @@ -275,10 +277,20 @@ def main( action="store_true", help="Live-render the streaming message history in the terminal", ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) args = parser.parse_args() with ( handler(LiteLLMProvider(model=args.model, tool_choice="required")), handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), handler(UnsafeEvalProvider()), handler(PythonRepl()), handler(SynthesizeAndCall()), diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index f403a928e..ba298a854 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -370,6 +370,7 @@ def _pydantic_type_tuple(ty): nt_model = pydantic.create_model( ty.__name__, __config__={"extra": "forbid"}, + __doc__=ty.__doc__, **{f: (t, ...) for f, t in zip(nt_fields, nt_types)}, ) From 1a1f42feaca9ee8f36ee7c27af79e2ccaff46c81 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 1 Jul 2026 20:29:01 -0400 Subject: [PATCH 027/155] move around --- effectful/handlers/llm/encoding.py | 135 +++++++++++++++-------------- 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index ba298a854..121eae42d 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -45,7 +45,7 @@ # Key under which the name->Tool mapping is stashed in the decoding context. # Deliberately not a valid Python identifier, so it can never collide with a # lexical variable name sharing the context (e.g. a reader named after its var). -_TOOLS_KEY = "$TOOLS" +_TOOLS_KEY: typing.Literal["$TOOLS"] = "$TOOLS" CONTENT_BLOCK_TYPES: frozenset[str] = frozenset( literal @@ -540,16 +540,76 @@ def _method_instance(self, other: Template) -> Any | None: class SynthesizedFunction(pydantic.BaseModel): - """Structured output for function synthesis. - - Pydantic model representing synthesized code with function name and module code. + """ + Structured output for function synthesis. """ module_code: str = pydantic.Field( ..., - description="Complete Python module code (no imports needed)", + description=textwrap.dedent(""" + A string containing the complete Python source code for the function. + The code MUST satisfy the following constraints, or it will fail validation: + + + 1. The code MUST be one complete syntactically valid Python module. + 2. The code MUST NOT use star imports or ``__future__`` imports. + 3. The function definition MUST be the LAST statement - do not add any code after it. + 4. The function MUST have type annotations for all parameters and the return type. + 5. You may include doctest examples (lines starting with >>>) inside the function's + docstring to demonstrate and verify its behavior; these examples are run as tests. + + """), ) + @pydantic.field_validator("module_code") + @classmethod + def _validate_module_code(cls, value: str) -> str: + module: ast.AST = ast.parse(value) + + if not isinstance(module, ast.Module) or not module.body: + raise ValueError( + "decode() requires module code with at least one statement." + ) + + last_stmt = module.body[-1] + if not isinstance(last_stmt, ast.FunctionDef): + raise ValueError( + f"decode() requires the last statement to be a function definition, " + f"got {type(last_stmt).__name__}" + ) + + # Check that the function has type annotations for all parameters + for arg in last_stmt.args.args: + if arg.annotation is None: + raise ValueError( + f"decode() requires all parameters to have type annotations, " + f"parameter '{arg.arg}' is missing an annotation" + ) + + # Check that the function has a return type annotation + if last_stmt.returns is None: + raise ValueError( + "decode() requires the function to have a return type annotation" + ) + + # no __future__ imports are allowed + for stmt in module.body: + if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__": + raise ValueError( + "decode() does not allow __future__ imports in the module code" + ) + + # no star imports are allowed + for stmt in module.body: + if isinstance(stmt, ast.ImportFrom) and stmt.names: + for alias in stmt.names: + if alias.name == "*": + raise ValueError( + "decode() does not allow star imports in the module code" + ) + + return value + def _create_typed_synthesized_function( callable_type: type[Callable], @@ -578,43 +638,11 @@ def _create_typed_synthesized_function( else: type_signature = str(callable_type) - description = f"""Given the specification above, generate a Python function satisfying the following specification and type signature. - -{type_signature} - - -1. Produce one block of Python code. -2. The function MUST have type annotations for all parameters and the return type. -3. The function definition must be the LAST statement - do not add any code after it. -4. You may include doctest examples (lines starting with >>>) inside the function's - docstring to demonstrate and verify its behavior; these examples are run as tests. -5. Do not add any executable code after the function definition (the doctest examples - in the docstring are the only usage examples allowed). - -""" - - # Use pydantic.create_model to create a proper model with the description - # The __doc__ becomes the model's description in the JSON schema - model = pydantic.create_model( + return pydantic.create_model( "TypedSynthesizedFunction", __base__=SynthesizedFunction, - __doc__=description, + __doc__=f"""Python function with signature {type_signature}""", ) - return model - - -def _validate_signature_ast( - func_ast: ast.FunctionDef | ast.AsyncFunctionDef, - expected_params: list[type] | None, -) -> None: - """Validate the function signature from AST before execution.""" - if expected_params is not None: - ast_params = func_ast.args.args + func_ast.args.posonlyargs - if len(ast_params) != len(expected_params): - raise ValueError( - f"decode() expected function with {len(expected_params)} parameters, " - f"got {len(ast_params)}" - ) def _validate_signature_callable( @@ -694,21 +722,7 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: ctx = info.context or {} filename = f"" - module: ast.AST = evaluation.parse(encoded.module_code, filename) - - if not isinstance(module, ast.Module) or not module.body: - raise ValueError( - "decode() requires module code with at least one statement." - ) - - last_stmt = module.body[-1] - if not isinstance(last_stmt, ast.FunctionDef): - raise ValueError( - f"decode() requires the last statement to be a function definition, " - f"got {type(last_stmt).__name__}" - ) - - _validate_signature_ast(last_stmt, expected_params) + module: ast.Module = evaluation.parse(encoded.module_code, filename) evaluation.type_check(module, ctx, expected_params, expected_return) g: MutableMapping[str, Any] = {} @@ -720,18 +734,7 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: bytecode: types.CodeType = evaluation.compile(module, filename) evaluation.exec(bytecode, g) - func_name = last_stmt.name - if func_name not in g: - raise ValueError( - f"decode() expected function '{func_name}' to be defined in globals" - ) - - result = g[func_name] - if not callable(result): - raise ValueError( - f"decode() expected '{func_name}' to be callable, got {type(result)}" - ) - + result = g[module.body[-1].name] # type: ignore _validate_signature_callable(result, expected_params, expected_return) if metadata is not None: From c26081fb386f9705ae9bb56ce3f054760173e666 Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 13 Jul 2026 18:35:28 +0900 Subject: [PATCH 028/155] formatting nits --- effectful/handlers/llm/evaluation.py | 47 ++++++++++++++-------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index 04a54c2d1..ed60cf2a1 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -16,9 +16,6 @@ import tempfile import types import typing -from collections.abc import MutableMapping -from types import CodeType -from typing import Any from mypy import api as mypy_api from RestrictedPython import ( @@ -70,7 +67,7 @@ def type_check(source: str, lo: int | None = None, hi: int | None = None) -> Non @defop def run_doctests( obj: collections.abc.Callable | type | types.ModuleType, - globs: typing.Mapping[str, Any], + globs: collections.abc.Mapping[str, typing.Any], ) -> None: """Run the doctests found in a synthesized object's docstring. @@ -88,7 +85,7 @@ def run_doctests( @defop -def compile(module: ast.Module, filename: str) -> CodeType: +def compile(module: ast.Module, filename: str) -> types.CodeType: """ Compile an AST into a Python code object. @@ -104,8 +101,8 @@ def compile(module: ast.Module, filename: str) -> CodeType: @defop def exec( - bytecode: CodeType, - env: dict[str, Any], + bytecode: types.CodeType, + env: dict[str, typing.Any], ) -> None: """ Execute a compiled code object. @@ -177,7 +174,9 @@ def _find_def_at_lineno( return None -def _region_errors(stdout: str, lo: int | None, hi: int | None) -> list[dict[str, Any]]: +def _region_errors( + stdout: str, lo: int | None, hi: int | None +) -> list[dict[str, typing.Any]]: """mypy ``--output=json`` diagnostics of severity ``error`` whose reported line falls within ``[lo, hi]`` -- the spliced region. An open bound (``None``) is unbounded on that side, so ``lo=hi=None`` reports every error. @@ -188,7 +187,7 @@ def _region_errors(stdout: str, lo: int | None, hi: int | None) -> list[dict[str for exit status < 2; a fatal status emits text, not JSON, and is handled by the caller before this runs. """ - errors: list[dict[str, Any]] = [] + errors: list[dict[str, typing.Any]] = [] for line in stdout.splitlines(): if not line.strip(): continue @@ -201,7 +200,7 @@ def _region_errors(stdout: str, lo: int | None, hi: int | None) -> list[dict[str def splice_into_source( - generated: ast.Module, anchor: Any + generated: ast.Module, anchor: typing.Any ) -> tuple[str, int, int] | None: """Splice `generated` into the anchor Template's own function body, in its real module source. @@ -351,14 +350,14 @@ def parse(self, source: str, filename: str) -> ast.Module: return ast.parse(source, filename=filename, mode="exec") @implements(compile) - def compile(self, module: ast.AST, filename: str) -> CodeType: + def compile(self, module: ast.AST, filename: str) -> types.CodeType: return builtins.compile(typing.cast(typing.Any, module), filename, "exec") @implements(exec) def exec( self, - bytecode: CodeType, - env: dict[str, Any], + bytecode: types.CodeType, + env: dict[str, typing.Any], ) -> None: # Ensure builtins exist in the execution environment. env.setdefault("__builtins__", __builtins__) @@ -370,7 +369,7 @@ def exec( def run_doctests( self, obj: collections.abc.Callable | type | types.ModuleType, - globs: typing.Mapping[str, Any], + globs: collections.abc.Mapping[str, typing.Any], ) -> None: assert hasattr(obj, "__name__") name = obj.__name__ @@ -410,7 +409,7 @@ class RestrictedEvalProvider(ObjectInterpretation): RestrictedPython is not a complete sandbox, but it enforces a restricted language subset and expects you to provide a constrained exec environment. - policy : dict[str, Any], optional + policy : dict[str, typing.Any], optional RestrictedPython compile_restricted policy for compilation """ @@ -441,7 +440,7 @@ def parse(self, source: str, filename: str) -> ast.Module: return ast.parse(source, filename=filename, mode="exec") @implements(compile) - def compile(self, module: ast.Module, filename: str) -> CodeType: + def compile(self, module: ast.Module, filename: str) -> types.CodeType: # RestrictedPython can compile from an AST directly. return compile_restricted( module, @@ -453,11 +452,11 @@ def compile(self, module: ast.Module, filename: str) -> CodeType: @implements(exec) def exec( self, - bytecode: CodeType, - env: dict[str, Any], + bytecode: types.CodeType, + env: dict[str, typing.Any], ) -> None: # Build restricted globals from RestrictedPython's defaults - rglobals: dict[str, Any] = safe_globals.copy() + rglobals: dict[str, typing.Any] = safe_globals.copy() # Enable class definitions (required for Python 3) rglobals["__metaclass__"] = type @@ -504,7 +503,7 @@ class _OpCommandCompiler(codeop.CommandCompiler): def __call__( self, source: str, filename: str = "", symbol: str = "single" - ) -> CodeType: + ) -> types.CodeType: # `runsource` passes symbol="single"; we ignore it and compile in the # exec mode the ops produce, so a complete multi-statement block runs in # one shot. Incomplete/invalid input raises SyntaxError, which @@ -537,14 +536,14 @@ class ReplSession(code.InteractiveInterpreter): stdout: io.StringIO stderr: io.StringIO - def __init__(self, env: MutableMapping[str, Any]): + def __init__(self, env: collections.abc.MutableMapping[str, typing.Any]): # Run in a fresh writable dict seeded with a flat view of `env`. This is # forced by `exec`: its globals must be one real dict (a ChainMap is # rejected), and a REPL needs a single persistent namespace so a function # defined in one snippet sees a name a later snippet binds. Seeding a flat # copy also leaves the lexical seed untouched, so REPL assignments never # leak into the surrounding scope. - scope: dict[str, Any] = dict(env) + scope: dict[str, typing.Any] = dict(env) # When `env` is the per-call `ChainMap` (its outer layers are read-only # frame proxies), splice this dict in as an extra shadowing first layer so # the bindings are *also* visible to the rest of the Template call @@ -561,7 +560,7 @@ def __init__(self, env: MutableMapping[str, Any]): self.stdout = io.StringIO() self.stderr = io.StringIO() - def runcode(self, code: CodeType) -> None: + def runcode(self, code: types.CodeType) -> None: # Mirrors `InteractiveInterpreter.runcode` exactly; the only difference # is that `exec` here is the effect operation, so execution routes # through the installed eval provider. `showtraceback` reports failures @@ -573,7 +572,7 @@ def runcode(self, code: CodeType) -> None: except: self.showtraceback() - def exec_code(self, code: CodeType) -> str: + def exec_code(self, code: types.CodeType) -> str: """Run Python in a persistent, stateful session and return its output. This is a long-lived REPL, not a one-shot sandbox: every call runs in the From 600f0581c9ca24cacb57610bd4664661e0512455 Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 13 Jul 2026 18:47:29 +0900 Subject: [PATCH 029/155] formatting nits --- effectful/handlers/llm/encoding.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index e51e51e2c..a6956f247 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -698,7 +698,13 @@ def _pydantic_callable( f"Callable type signature incomplete: {callable_type}. " "Expected Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType]." ) - param_types, expected_return = type_args[0], type_args[-1] + if type_args[1] is None: + raise pydantic.errors.PydanticSchemaGenerationError( + "Cannot decode/synthesize callable without a concrete type signature. " + "Use Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType] " + "with a concrete return type (not Any)." + ) + param_types, expected_return = type_args[0], type_args[1] typed_enc = _create_typed_synthesized_function(callable_type) if param_types is not ... and isinstance(param_types, list | tuple): expected_params = list(param_types) @@ -720,13 +726,6 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: f"got {type(value)}" ) - if expected_return is None: - raise TypeError( - "Cannot decode/synthesize callable without a concrete type signature. " - "Use Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType] " - "with a concrete return type (not Any)." - ) - ctx = info.context or {} filename = f"" module: ast.AST = evaluation.parse(encoded.module_code, filename) From 8c116156d64ebe412969f0880619c173ca43b9b4 Mon Sep 17 00:00:00 2001 From: Eli Date: Sat, 18 Jul 2026 12:22:17 -0400 Subject: [PATCH 030/155] fix failed merge --- docs/source/codeadapt.py | 8 ++++---- effectful/handlers/llm/completions.py | 10 +++++++--- effectful/internals/unification.py | 4 +++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/source/codeadapt.py b/docs/source/codeadapt.py index 695a0d7d4..dc48f6bef 100644 --- a/docs/source/codeadapt.py +++ b/docs/source/codeadapt.py @@ -15,7 +15,7 @@ import contextlib import os import pathlib -from typing import Literal, NamedTuple +import typing import tenacity @@ -56,7 +56,7 @@ def least_beautiful_base(threshold: int) -> int: raise NotHandled -class LineupClue(NamedTuple): +class LineupClue(typing.NamedTuple): """ A clue about the relative ordering of n people, numbered 0 to n - 1, in a line. Used to describe puzzles like the classic "zebra puzzle" represented in `solve_lineup`. @@ -70,7 +70,7 @@ class LineupClue(NamedTuple): - ``("adj", a, b)`` -- persons ``a`` and ``b`` are in adjacent positions """ - kind: Literal["at", "left", "imm_left", "adj"] + kind: typing.Literal["at", "left", "imm_left", "adj"] a: int b: int @@ -186,7 +186,7 @@ def musr_object_placement( def main( - task: Literal["beautiful", "lineup", "countdown", "paragraph", "typos", "musr"], + task: typing.Literal["beautiful", "lineup", "countdown", "paragraph", "typos", "musr"], ) -> None: if task == "beautiful": threshold = 10 diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index e1b7e0aa4..26037a9bd 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -690,6 +690,7 @@ def _call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), + anchor: types.FunctionType | None = None, ) -> AssistantResult[T]: readers: set[Tool] = set(tools) taken = {t.__name__ for t in tools} @@ -706,7 +707,7 @@ def _call_assistant[T]( taken.add(name) except Exception: continue - return fwd(env, response_type, readers) + return fwd(env, response_type, readers, anchor=anchor) class SynthesizeAndCall(ObjectInterpretation): @@ -817,8 +818,8 @@ def _apply[**P, T]( bound_args.apply_defaults() tool = self._SynthesisFinalTool.define(template, bound_args) - def _add_synthesis_tool(env, response_type, tools=frozenset()): - return fwd(env, response_type, tools | {tool}) + def _add_synthesis_tool(env, response_type, tools=frozenset(), anchor=None): + return fwd(env, response_type, tools | {tool}, anchor=anchor) with handler({call_assistant: _add_synthesis_tool}): return fwd() @@ -898,11 +899,13 @@ def _call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), + anchor: types.FunctionType | None = None, ) -> AssistantResult[T]: return fwd( env, response_type, tools | {self.exec_code, self.read_lexical_variable}, + anchor=anchor, ) @@ -971,6 +974,7 @@ def _call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), + anchor: types.FunctionType | None = None, ) -> AssistantResult[T]: _message_sequence = _get_history().copy() diff --git a/effectful/internals/unification.py b/effectful/internals/unification.py index 0ebd0295c..6ed9ef6cb 100644 --- a/effectful/internals/unification.py +++ b/effectful/internals/unification.py @@ -1101,7 +1101,9 @@ def _(value: collections.abc.Collection): @nested_type.register def _(value: tuple): - if type(value) != tuple or len(value) == 0: + if hasattr(value, "_fields"): + return Box(type(value)) + elif type(value) != tuple or len(value) == 0: return nested_type.dispatch(collections.abc.Sequence)(value) else: return Box(tuple[tuple(nested_type(item).value for item in value)]) # type: ignore From 6b665aab6429441b36ad286d190a156d489ba9e2 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 12:34:00 -0400 Subject: [PATCH 031/155] split into more files --- docs/source/codeadapt.py | 301 ------------------ docs/source/codeadapt_agent.py | 147 --------- .../llm_examples/constrained_paragraph.py | 101 ++++++ docs/source/llm_examples/countdown.py | 119 +++++++ docs/source/llm_examples/fix_typos.py | 103 ++++++ .../llm_examples/least_beautiful_base.py | 101 ++++++ docs/source/llm_examples/lineup.py | 133 ++++++++ docs/source/llm_examples/musr.py | 131 ++++++++ pyproject.toml | 1 + 9 files changed, 689 insertions(+), 448 deletions(-) delete mode 100644 docs/source/codeadapt.py delete mode 100644 docs/source/codeadapt_agent.py create mode 100644 docs/source/llm_examples/constrained_paragraph.py create mode 100644 docs/source/llm_examples/countdown.py create mode 100644 docs/source/llm_examples/fix_typos.py create mode 100644 docs/source/llm_examples/least_beautiful_base.py create mode 100644 docs/source/llm_examples/lineup.py create mode 100644 docs/source/llm_examples/musr.py diff --git a/docs/source/codeadapt.py b/docs/source/codeadapt.py deleted file mode 100644 index dc48f6bef..000000000 --- a/docs/source/codeadapt.py +++ /dev/null @@ -1,301 +0,0 @@ -"""CodeAdapt: solving hard problems by writing and running Python. - -You are a careful problem solver and an expert Python programmer. You answer by -writing code, not by reasoning in prose alone: problems that are error-prone to -work out by hand are often easy to brute-force or verify with a short program. - -You MUST use the ``submit_solution`` tool to give your final answer, -the harness will not accept a final answer in direct text. - -You can use whatever other tools are available to develop your solution, -and refine incorrect attempts given feedback from failures of ``submit_solution``. -""" - -import argparse -import contextlib -import os -import pathlib -import typing - -import tenacity - -from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled - - -@Template.define -def least_beautiful_base(threshold: int) -> int: - r"""Find the least integer base b >= 2 for which there are more than - {threshold} ``b``-eautiful integers. - - A positive integer n is ``b``-eautiful if it has exactly two digits when - written in base b and those two digits sum to ``sqrt(n)``. For example, 81 - is 13-eautiful because 81 = 6_3 in base 13 and 6 + 3 = sqrt(81). - - >>> least_beautiful_base(0) - 3 - >>> least_beautiful_base(1) - 7 - >>> least_beautiful_base(5) - 31 - >>> least_beautiful_base(7) - 211 - """ - raise NotHandled - - -class LineupClue(typing.NamedTuple): - """ - A clue about the relative ordering of n people, numbered 0 to n - 1, in a line. - Used to describe puzzles like the classic "zebra puzzle" represented in `solve_lineup`. - Each `LineupClue` corresponds to a single ordering constraint, ``(kind, a, b)``. - - The meaning of ``a`` and ``b`` depends on ``kind``: - - - ``("at", a, k)`` -- person ``a`` is at position ``k`` - - ``("left", a, b)`` -- person ``a`` is somewhere left of person ``b`` - - ``("imm_left", a, b)`` -- person ``a`` is immediately left of person ``b`` - - ``("adj", a, b)`` -- persons ``a`` and ``b`` are in adjacent positions - """ - - kind: typing.Literal["at", "left", "imm_left", "adj"] - a: int - b: int - - -@Template.define -def solve_lineup(n: int, clues: list[LineupClue]) -> list[int]: - """Solve a 'zebra'-style ordering puzzle: place n={n} people, numbered 0 to - n - 1, in a line in positions 1 to n (each position used once) so that every - `LineupClue` in the following list holds: - - {clues} - - Every puzzle has exactly one consistent arrangement. Return the list of - positions ``[position of 0, position of 1, ..., position of n - 1]``, - as shown in the following worked examples: - - >>> solve_lineup(3, [LineupClue("at", 0, 1), LineupClue("left", 1, 2)]) - [1, 2, 3] - >>> solve_lineup(4, [LineupClue("left", 0, 1), LineupClue("left", 1, 2), LineupClue("left", 2, 3)]) - [1, 2, 3, 4] - >>> solve_lineup(4, [LineupClue("imm_left", 0, 1), LineupClue("at", 2, 4), LineupClue("left", 3, 0)]) - [2, 3, 4, 1] - >>> solve_lineup(5, [LineupClue("at", 0, 3), LineupClue("imm_left", 1, 2), LineupClue("left", 3, 4), LineupClue("at", 4, 5)]) - [3, 1, 2, 4, 5] - """ - raise NotHandled - - -@Template.define -def countdown_reachable(numbers: list[int], target: int) -> bool: - """In the Countdown numbers game, decide whether {target} can be made from - {numbers}, using each number exactly once and combining them with + - * / - (every intermediate division must come out exact). - - >>> countdown_reachable([2, 3, 5], 11) - True - >>> countdown_reachable([1, 1], 5) - False - >>> countdown_reachable([4, 7, 8, 9], 100) - True - >>> countdown_reachable([5, 5, 5], 3) - False - """ - raise NotHandled - - -@Template.define -def constrained_paragraph(endings: list[str]) -> str: - r"""Write a short paragraph whose sentences end, in order, with the words in - {endings}: one sentence per word, each ending with that exact word. - - The examples below split the returned paragraph into sentences and compare the - last word of each (lowercased, punctuation stripped) against the requested - endings -- so a synthesized function must build text with the right shape: - - >>> import re - >>> def endings_of(paragraph): - ... sents = [s for s in re.split(r"(?<=[.!?])\s+", paragraph.strip()) if s] - ... return [re.findall(r"[A-Za-z']+", s)[-1].lower() for s in sents] - >>> endings_of(constrained_paragraph(["walk", "tumbling", "another", "lunatic"])) - ['walk', 'tumbling', 'another', 'lunatic'] - >>> endings_of(constrained_paragraph(["dawn", "river"])) - ['dawn', 'river'] - """ - raise NotHandled - - -@Template.define -def fix_typos(text: str) -> str: - """Output the following text exactly, with no changes at all except for fixing - the misspellings. Leave every other stylistic decision -- commas, US vs British - spellings, capitalization, line breaks -- exactly as in the original: - - {text} - - Only misspelled words may change; every correctly spelled word and all - punctuation and whitespace must be preserved verbatim. Identify the typos, then - apply the corrections with code so that nothing else can drift. - - >>> fix_typos("We inctroduce a probablistic method in the presense of noise.") - 'We introduce a probabilistic method in the presence of noise.' - >>> fix_typos("Teh quick borwn fox jumpps over the lazy dog.") - 'The quick brown fox jumps over the lazy dog.' - """ - raise NotHandled - - -@Template.define -def musr_object_placement( - story: str, person: str, item: str, locations: list[str] -) -> str: - """A MuSR object-placement question: a theory-of-mind puzzle. Read the story - and decide, from {locations}, where {person} would look for the {item}. - - The answer is the last place {person} *saw* the {item}: the last move they - watched, or any later moment they directly saw it somewhere; or its original - location if they never saw it after that. A person's belief does not change - while they are not watching, so where the {item} actually ends up and where - {person} believes it is can differ. - - {story} - - >>> musr_object_placement( - ... "Danny set the earphones in the recording booth, then stepped out for a " - ... "call. While he was gone, Emma quietly moved them to the producer's desk.", - ... "Danny", - ... "earphones", - ... ["recording booth", "producer's desk"], - ... ) - 'recording booth' - """ - raise NotHandled - - -def main( - task: typing.Literal["beautiful", "lineup", "countdown", "paragraph", "typos", "musr"], -) -> None: - if task == "beautiful": - threshold = 10 - print(f"Least b with > {threshold} b-eautiful integers") - print(f"Answer: {least_beautiful_base(threshold)}") - elif task == "lineup": - puzzle = [ - LineupClue("imm_left", 0, 1), - LineupClue("imm_left", 1, 2), - LineupClue("at", 3, 5), - LineupClue("left", 4, 0), - ] - print(f"Zebra-style ordering puzzle: n=5, clues={puzzle}") - print(f"Answer: {solve_lineup(5, puzzle)}") - elif task == "countdown": - numbers, target = [3, 6, 25, 50], 147 - print(f"Countdown: reach {target} from {numbers}") - print(f"Answer: {countdown_reachable(numbers, target)}") - elif task == "paragraph": - endings = ["mountain", "whisper", "thunder"] - print(f"Paragraph with sentences ending in {endings}") - print(f"Answer: {constrained_paragraph(endings)}") - elif task == "typos": - text = ( - "We inctroduce a probablistic algorithm that estimates the " - "timne-varying location in the presense of measurment noise." - ) - print(f"Fix only the typos in:\n{text}") - print(f"Answer: {fix_typos(text)}") - elif task == "musr": - STUDIO_STORY = """\ -In the heart of the bustling studio, Ricky, Emma, and Danny readied themselves \ -for a day of creating magic. Ricky, the gifted singer-songwriter, had his \ -precious notebook of lyrics on the producer's desk. Emma, their producer, was \ -cognizant of the notebook's place at her desk. Across the room, Danny, the studio \ -assistant, kept the earphones in the recording booth. They were all aware of the \ -arrangement -- the notebook on the producer's desk, the earphones in the \ -recording booth. - -Ricky gently places his notebook onto the piano, then becomes engrossed in \ -perfecting his song. Emma, engrossed in her thoughts, deftly moves the earphones \ -to the producer's desk. At that moment Danny was in a stirring conversation with a \ -visiting sound engineer; the visitor stood blocking Danny's general overview of \ -the studio space. - -Later, delicately lifting Ricky's notebook, Danny orchestrates its move to the \ -producer's desk. At the desk, he glimpses a pair of earphones indirectly drawing \ -his attention amidst his routine of tidying up. Meanwhile Emma, from inside a \ -sound-proofed booth, was lost in reviewing already-recorded tracks, out of \ -Danny's view.""" - person = "Danny" - item = "earphones" - locations = ["piano", "producer's desk", "recording booth"] - answer = musr_object_placement(STUDIO_STORY, person, item, locations) - print(f"MuSR: where would {person} look for the {item}?") - print(f"Answer: {answer}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="CodeAdapt: solve hard problems by writing and running code" - ) - parser.add_argument( - "--task", - choices=("beautiful", "lineup", "countdown", "paragraph", "typos", "musr"), - default="beautiful", - help="Which problem to solve", - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", - ) - parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", - ) - args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args.task) diff --git a/docs/source/codeadapt_agent.py b/docs/source/codeadapt_agent.py deleted file mode 100644 index c39a67777..000000000 --- a/docs/source/codeadapt_agent.py +++ /dev/null @@ -1,147 +0,0 @@ -"""CodeAdapt as an Agent: in-context learning across a conversation. - -This is the Agent-method variant of ``codeadapt.py``. Instead of a free -function, the task is a :class:`~effectful.handlers.llm.Template` method on an -:class:`~effectful.handlers.llm.Agent` subclass, so each call accumulates -message history on the instance and the model can take advantage of in-context -learning across calls. - -The synthesized function is a drop-in syntactic replacement for the method body --- it keeps ``self`` in its signature -- and the worked examples in the method's -docstring are run as doctests against that synthesized function (calls on -freshly constructed agents are rerouted to it rather than re-invoking the model). -""" - -import argparse -import contextlib -import os -import pathlib - -import tenacity - -from effectful.handlers.llm import Agent, Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled - - -class CodeAdaptAgent(Agent): - """ - You are a careful problem solver and an expert Python programmer. You answer by - writing code, not by reasoning in prose alone: problems that are error-prone to - work out by hand are often easy to brute-force or verify with a short program. - - You MUST use the ``submit_solution`` tool to give your final answer, - the harness will not accept a final answer in direct text. - - You can use whatever other tools are available to develop your solution, - and refine incorrect attempts given feedback from failures of ``submit_solution``. - """ - - @Template.define - def countdown_reachable(self, numbers: list[int], target: int) -> bool: - """In the Countdown numbers game, decide whether {target} can be made from - {numbers}, using each number exactly once and combining them with + - * / - (every intermediate division must come out exact). - - >>> agent = CodeAdaptAgent() - >>> agent.countdown_reachable([2, 3, 5], 11) - True - >>> agent.countdown_reachable([1, 1], 5) - False - >>> agent.countdown_reachable([4, 7, 8, 9], 100) - True - >>> agent.countdown_reachable([5, 5, 5], 3) - False - """ - raise NotHandled - - -def main(args: argparse.Namespace) -> None: - if args.task == "countdown": - agent = CodeAdaptAgent() - # Fresh examples (none appear in the docstring doctests), each paired with its - # known-correct answer so we can validate the agent's output. - test_examples: list[tuple[list[int], int, bool]] = [ - ([3, 6, 25, 50], 147, True), # (50 - 25) * 6 - 3 - ([1, 2, 3, 4], 24, True), # 1 * 2 * 3 * 4 - ([2, 4, 8], 9, False), # all-even operands can never reach an odd target - ] - for numbers, target, expected in test_examples: - print(f"Testing countdown_reachable({numbers}, {target})...") - answer = agent.countdown_reachable(numbers, target) - status = "OK" if answer == expected else "WRONG" - print( - f"[{status}] reach {target} from {numbers}: {answer} (expected {expected})" - ) - assert answer == expected, ( - f"countdown_reachable({numbers}, {target}) = {answer}, expected {expected}" - ) - else: - raise ValueError(f"Unknown task {args.task}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="CodeAdapt agent: solve reasoning tasks by writing code" - ) - parser.add_argument( - "--task", - choices=("countdown",), - default="countdown", - help="Which problem to solve", - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", - ) - parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", - ) - args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) diff --git a/docs/source/llm_examples/constrained_paragraph.py b/docs/source/llm_examples/constrained_paragraph.py new file mode 100644 index 000000000..a4596a128 --- /dev/null +++ b/docs/source/llm_examples/constrained_paragraph.py @@ -0,0 +1,101 @@ +"""Solving hard problems by writing and running Python. + +You are a careful problem solver and an expert Python programmer. You answer by +writing code, not by reasoning in prose alone: problems that are error-prone to +work out by hand are often easy to brute-force or verify with a short program. +""" + +import argparse +import contextlib +import os +import pathlib + +import tenacity + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler + + +@Template.define +def constrained_paragraph(endings: list[str]) -> str: + r"""Write a short paragraph whose sentences end, in order, with the words in + {endings}: one sentence per word, each ending with that exact word. + + The examples below split the returned paragraph into sentences and compare the + last word of each (lowercased, punctuation stripped) against the requested + endings -- so a synthesized function must build text with the right shape: + + >>> import re + >>> def endings_of(paragraph): + ... sents = [s for s in re.split(r"(?<=[.!?])\s+", paragraph.strip()) if s] + ... return [re.findall(r"[A-Za-z']+", s)[-1].lower() for s in sents] + >>> endings_of(constrained_paragraph(["walk", "tumbling", "another", "lunatic"])) + ['walk', 'tumbling', 'another', 'lunatic'] + >>> endings_of(constrained_paragraph(["dawn", "river"])) + ['dawn', 'river'] + """ + + +def main(args: argparse.Namespace) -> None: + endings = ["mountain", "whisper", "thunder"] + print(f"Paragraph with sentences ending in {endings}") + print(f"Answer: {constrained_paragraph(endings)}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/docs/source/llm_examples/countdown.py b/docs/source/llm_examples/countdown.py new file mode 100644 index 000000000..32d95a619 --- /dev/null +++ b/docs/source/llm_examples/countdown.py @@ -0,0 +1,119 @@ +""" +In-context learning to solve problems with code across a conversation. +""" + +import argparse +import collections.abc +import contextlib +import os +import pathlib + +import tenacity + +from effectful.handlers.llm import Agent, Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler + + +class CountdownSolver(Agent): + """ + You are a careful problem solver and an expert Python programmer. You answer by + writing code, not by reasoning in prose alone: problems that are error-prone to + work out by hand are often easy to brute-force or verify with a short program. + """ + + @Template.define + def solve(self, numbers: collections.abc.Sequence[int], target: int) -> bool: + """In the Countdown numbers game, decide whether {target} can be made from + {numbers}, using each number exactly once and combining them with + - * / + (every intermediate division must come out exact). + + >>> agent = CountdownSolver() + >>> agent.solve([2, 3, 5], 11) + True + >>> agent.solve([1, 1], 5) + False + >>> agent.solve([4, 7, 8, 9], 100) + True + >>> agent.solve([5, 5, 5], 3) + False + """ + + +def main(args: argparse.Namespace) -> None: + agent = CountdownSolver() + # Fresh examples (none appear in the docstring doctests), each paired with its + # known-correct answer so we can validate the agent's output. + test_examples: list[tuple[list[int], int, bool]] = [ + ([3, 6, 25, 50], 147, True), # (50 - 25) * 6 - 3 + ([1, 2, 3, 4], 24, True), # 1 * 2 * 3 * 4 + ([2, 4, 8], 9, False), # all-even operands can never reach an odd target + ] + for numbers, target, expected in test_examples: + print(f"Testing solve({numbers}, {target})...") + answer = agent.solve(numbers, target) + status = "OK" if answer == expected else "WRONG" + print( + f"[{status}] solve({numbers}, {target}): {answer} (expected {expected})" + ) + assert answer == expected, ( + f"solve({numbers}, {target}) = {answer}, expected {expected}" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/docs/source/llm_examples/fix_typos.py b/docs/source/llm_examples/fix_typos.py new file mode 100644 index 000000000..ce5baa8ad --- /dev/null +++ b/docs/source/llm_examples/fix_typos.py @@ -0,0 +1,103 @@ +"""Solving hard problems by writing and running Python. + +You are a careful problem solver and an expert Python programmer. You answer by +writing code, not by reasoning in prose alone: problems that are error-prone to +work out by hand are often easy to brute-force or verify with a short program. +""" + +import argparse +import contextlib +import os +import pathlib + +import tenacity + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler + + +@Template.define +def fix_typos(text: str) -> str: + """Output the following text exactly, with no changes at all except for fixing + the misspellings. Leave every other stylistic decision -- commas, US vs British + spellings, capitalization, line breaks -- exactly as in the original: + + {text} + + Only misspelled words may change; every correctly spelled word and all + punctuation and whitespace must be preserved verbatim. Identify the typos, then + apply the corrections with code so that nothing else can drift. + + >>> fix_typos("We inctroduce a probablistic method in the presense of noise.") + 'We introduce a probabilistic method in the presence of noise.' + >>> fix_typos("Teh quick borwn fox jumpps over the lazy dog.") + 'The quick brown fox jumps over the lazy dog.' + """ + + +def main(args: argparse.Namespace) -> None: + text = ( + "We inctroduce a probablistic algorithm that estimates the " + "timne-varying location in the presense of measurment noise." + ) + print(f"Fix only the typos in:\n{text}") + print(f"Answer: {fix_typos(text)}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/docs/source/llm_examples/least_beautiful_base.py b/docs/source/llm_examples/least_beautiful_base.py new file mode 100644 index 000000000..93ee06263 --- /dev/null +++ b/docs/source/llm_examples/least_beautiful_base.py @@ -0,0 +1,101 @@ +"""Solving hard problems by writing and running Python. + +You are a careful problem solver and an expert Python programmer. You answer by +writing code, not by reasoning in prose alone: problems that are error-prone to +work out by hand are often easy to brute-force or verify with a short program. +""" + +import argparse +import contextlib +import os +import pathlib + +import tenacity + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler + + +@Template.define +def least_beautiful_base(threshold: int) -> int: + r"""Find the least integer base b >= 2 for which there are more than + {threshold} ``b``-eautiful integers. + + A positive integer n is ``b``-eautiful if it has exactly two digits when + written in base b and those two digits sum to ``sqrt(n)``. For example, 81 + is 13-eautiful because 81 = 6_3 in base 13 and 6 + 3 = sqrt(81). + + >>> least_beautiful_base(0) + 3 + >>> least_beautiful_base(1) + 7 + >>> least_beautiful_base(5) + 31 + >>> least_beautiful_base(7) + 211 + """ + + +def main(args: argparse.Namespace) -> None: + threshold = 10 + print(f"Least b with > {threshold} b-eautiful integers") + print(f"Answer: {least_beautiful_base(threshold)}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/docs/source/llm_examples/lineup.py b/docs/source/llm_examples/lineup.py new file mode 100644 index 000000000..42f526250 --- /dev/null +++ b/docs/source/llm_examples/lineup.py @@ -0,0 +1,133 @@ +"""Solving hard problems by writing and running Python. + +You are a careful problem solver and an expert Python programmer. You answer by +writing code, not by reasoning in prose alone: problems that are error-prone to +work out by hand are often easy to brute-force or verify with a short program. +""" + +import argparse +import collections.abc +import contextlib +import dataclasses +import os +import pathlib +import typing + +import tenacity + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler +from effectful.ops.types import NotHandled + + +@dataclasses.dataclass(frozen=True) +class LineupClue: + """ + A clue about the relative ordering of n people, numbered 0 to n - 1, in a line. + Used to describe puzzles like the classic "zebra puzzle" represented in `solve_lineup`. + Each `LineupClue` corresponds to a single ordering constraint, ``(kind, a, b)``. + + The meaning of ``a`` and ``b`` depends on ``kind``: + + - ``("at", a, k)`` -- person ``a`` is at position ``k`` + - ``("left", a, b)`` -- person ``a`` is somewhere left of person ``b`` + - ``("imm_left", a, b)`` -- person ``a`` is immediately left of person ``b`` + - ``("adj", a, b)`` -- persons ``a`` and ``b`` are in adjacent positions + """ + + kind: typing.Literal["at", "left", "imm_left", "adj"] + a: int + b: int + + +@Template.define +def solve_lineup(n: int, clues: collections.abc.Sequence[LineupClue]) -> list[int]: + """Solve a 'zebra'-style ordering puzzle: place n={n} people, numbered 0 to + n - 1, in a line in positions 1 to n (each position used once) so that every + `LineupClue` in the following list holds: + + {clues} + + Every puzzle has exactly one consistent arrangement. Return the list of + positions ``[position of 0, position of 1, ..., position of n - 1]``, + as shown in the following worked examples: + + >>> solve_lineup(3, [LineupClue("at", 0, 1), LineupClue("left", 1, 2)]) + [1, 2, 3] + >>> solve_lineup(4, [LineupClue("left", 0, 1), LineupClue("left", 1, 2), LineupClue("left", 2, 3)]) + [1, 2, 3, 4] + >>> solve_lineup(4, [LineupClue("imm_left", 0, 1), LineupClue("at", 2, 4), LineupClue("left", 3, 0)]) + [2, 3, 4, 1] + >>> solve_lineup(5, [LineupClue("at", 0, 3), LineupClue("imm_left", 1, 2), LineupClue("left", 3, 4), LineupClue("at", 4, 5)]) + [3, 1, 2, 4, 5] + """ + + +def main(args: argparse.Namespace) -> None: + puzzle = [ + LineupClue("imm_left", 0, 1), + LineupClue("imm_left", 1, 2), + LineupClue("at", 3, 5), + LineupClue("left", 4, 0), + ] + print(f"Zebra-style ordering puzzle: n=5, clues={puzzle}") + print(f"Answer: {solve_lineup(5, puzzle)}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/docs/source/llm_examples/musr.py b/docs/source/llm_examples/musr.py new file mode 100644 index 000000000..63a21d9d2 --- /dev/null +++ b/docs/source/llm_examples/musr.py @@ -0,0 +1,131 @@ +"""Solving hard problems by writing and running Python. + +You are a careful problem solver and an expert Python programmer. You answer by +writing code, not by reasoning in prose alone: problems that are error-prone to +work out by hand are often easy to brute-force or verify with a short program. +""" + +import argparse +import collections.abc +import contextlib +import os +import pathlib + +import tenacity + +from effectful.handlers.llm import Template +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler + + +@Template.define +def musr_object_placement( + story: str, person: str, item: str, locations: collections.abc.Sequence[str] +) -> str: + """A MuSR object-placement question: a theory-of-mind puzzle. Read the story + and decide, from {locations}, where {person} would look for the {item}. + + The answer is the last place {person} *saw* the {item}: the last move they + watched, or any later moment they directly saw it somewhere; or its original + location if they never saw it after that. A person's belief does not change + while they are not watching, so where the {item} actually ends up and where + {person} believes it is can differ. + + {story} + + >>> musr_object_placement( + ... "Danny set the earphones in the recording booth, then stepped out for a " + ... "call. While he was gone, Emma quietly moved them to the producer's desk.", + ... "Danny", + ... "earphones", + ... ["recording booth", "producer's desk"], + ... ) + 'recording booth' + """ + + +def main(args: argparse.Namespace) -> None: + STUDIO_STORY = """\ +In the heart of the bustling studio, Ricky, Emma, and Danny readied themselves \ +for a day of creating magic. Ricky, the gifted singer-songwriter, had his \ +precious notebook of lyrics on the producer's desk. Emma, their producer, was \ +cognizant of the notebook's place at her desk. Across the room, Danny, the studio \ +assistant, kept the earphones in the recording booth. They were all aware of the \ +arrangement -- the notebook on the producer's desk, the earphones in the \ +recording booth. + +Ricky gently places his notebook onto the piano, then becomes engrossed in \ +perfecting his song. Emma, engrossed in her thoughts, deftly moves the earphones \ +to the producer's desk. At that moment Danny was in a stirring conversation with a \ +visiting sound engineer; the visitor stood blocking Danny's general overview of \ +the studio space. + +Later, delicately lifting Ricky's notebook, Danny orchestrates its move to the \ +producer's desk. At the desk, he glimpses a pair of earphones indirectly drawing \ +his attention amidst his routine of tidying up. Meanwhile Emma, from inside a \ +sound-proofed booth, was lost in reviewing already-recorded tracks, out of \ +Danny's view.""" + person = "Danny" + item = "earphones" + locations = ["piano", "producer's desk", "recording booth"] + answer = musr_object_placement(STUDIO_STORY, person, item, locations) + print(f"MuSR: where would {person} look for the {item}?") + print(f"Answer: {answer}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + args = parser.parse_args() + with ( + handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), + handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), + handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) + if args.dump_system_prompt + else contextlib.nullcontext(), + handler(UnsafeEvalProvider()), + handler(PythonRepl()), + handler(SynthesizeAndCall()), + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), + handler(LexicalReaders()), + handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), + ): + main(args) diff --git a/pyproject.toml b/pyproject.toml index c4d64436a..a1f96bb98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,4 +97,5 @@ testpaths = ["./effectful", "./tests"] [tool.mypy] ignore_missing_imports = true warn_unused_ignores = true +allow_empty_bodies = true exclude = "build|test_internals_unification_typeddict" From 395c51f9d3bf33b0d79b3891c5b2f4a6f07feb19 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 13:58:32 -0400 Subject: [PATCH 032/155] Add a library-level harness.py --- .../llm_examples/constrained_paragraph.py | 77 ++----- docs/source/llm_examples/countdown.py | 96 +++------ docs/source/llm_examples/fix_typos.py | 80 ++------ .../llm_examples/least_beautiful_base.py | 74 +------ docs/source/llm_examples/lineup.py | 107 ++++------ docs/source/llm_examples/musr.py | 86 +++----- effectful/handlers/llm/harness.py | 193 ++++++++++++++++++ 7 files changed, 324 insertions(+), 389 deletions(-) create mode 100644 effectful/handlers/llm/harness.py diff --git a/docs/source/llm_examples/constrained_paragraph.py b/docs/source/llm_examples/constrained_paragraph.py index a4596a128..f75102a9d 100644 --- a/docs/source/llm_examples/constrained_paragraph.py +++ b/docs/source/llm_examples/constrained_paragraph.py @@ -6,25 +6,8 @@ """ import argparse -import contextlib -import os -import pathlib - -import tenacity from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler @Template.define @@ -47,55 +30,19 @@ def constrained_paragraph(endings: list[str]) -> str: """ -def main(args: argparse.Namespace) -> None: - endings = ["mountain", "whisper", "thunder"] - print(f"Paragraph with sentences ending in {endings}") - print(f"Answer: {constrained_paragraph(endings)}") - - -if __name__ == "__main__": +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", - ) - parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", + "--endings", + nargs="+", + default=["mountain", "whisper", "thunder"], + metavar="WORD", + help="Words each sentence must end with, in order", ) args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) + print(f"Paragraph with sentences ending in {args.endings}") + print(f"Answer: {constrained_paragraph(args.endings)}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/countdown.py b/docs/source/llm_examples/countdown.py index 32d95a619..ed7807e07 100644 --- a/docs/source/llm_examples/countdown.py +++ b/docs/source/llm_examples/countdown.py @@ -4,25 +4,8 @@ import argparse import collections.abc -import contextlib -import os -import pathlib - -import tenacity from effectful.handlers.llm import Agent, Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler class CountdownSolver(Agent): @@ -50,8 +33,35 @@ def solve(self, numbers: collections.abc.Sequence[int], target: int) -> bool: """ -def main(args: argparse.Namespace) -> None: +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--numbers", + nargs="+", + type=int, + default=None, + metavar="N", + help="Numbers to combine (used with --target for a single problem)", + ) + parser.add_argument( + "--target", + type=int, + default=None, + help="Target value to make from --numbers", + ) + args = parser.parse_args() + if (args.numbers is None) != (args.target is None): + parser.error("--numbers and --target must be given together") + agent = CountdownSolver() + + # A custom problem has no known answer to validate against, so just solve it. + if args.numbers is not None: + print(f"Testing solve({args.numbers}, {args.target})...") + answer = agent.solve(args.numbers, args.target) + print(f"solve({args.numbers}, {args.target}): {answer}") + return + # Fresh examples (none appear in the docstring doctests), each paired with its # known-correct answer so we can validate the agent's output. test_examples: list[tuple[list[int], int, bool]] = [ @@ -63,57 +73,11 @@ def main(args: argparse.Namespace) -> None: print(f"Testing solve({numbers}, {target})...") answer = agent.solve(numbers, target) status = "OK" if answer == expected else "WRONG" - print( - f"[{status}] solve({numbers}, {target}): {answer} (expected {expected})" - ) + print(f"[{status}] solve({numbers}, {target}): {answer} (expected {expected})") assert answer == expected, ( f"solve({numbers}, {target}) = {answer}, expected {expected}" ) if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", - ) - parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", - ) - args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) + main() diff --git a/docs/source/llm_examples/fix_typos.py b/docs/source/llm_examples/fix_typos.py index ce5baa8ad..16a966b21 100644 --- a/docs/source/llm_examples/fix_typos.py +++ b/docs/source/llm_examples/fix_typos.py @@ -6,25 +6,8 @@ """ import argparse -import contextlib -import os -import pathlib - -import tenacity from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler @Template.define @@ -46,58 +29,21 @@ def fix_typos(text: str) -> str: """ -def main(args: argparse.Namespace) -> None: - text = ( - "We inctroduce a probablistic algorithm that estimates the " - "timne-varying location in the presense of measurment noise." - ) - print(f"Fix only the typos in:\n{text}") - print(f"Answer: {fix_typos(text)}") - - -if __name__ == "__main__": +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", + "--text", type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", - ) - parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", + default=( + "We inctroduce a probablistic algorithm that estimates the " + "timne-varying location in the presense of measurment noise." + ), + help="Text whose typos should be fixed", ) args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) + print(f"Fix only the typos in:\n{args.text}") + print(f"Answer: {fix_typos(args.text)}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/least_beautiful_base.py b/docs/source/llm_examples/least_beautiful_base.py index 93ee06263..62f2c9234 100644 --- a/docs/source/llm_examples/least_beautiful_base.py +++ b/docs/source/llm_examples/least_beautiful_base.py @@ -6,25 +6,8 @@ """ import argparse -import contextlib -import os -import pathlib - -import tenacity from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler @Template.define @@ -47,55 +30,18 @@ def least_beautiful_base(threshold: int) -> int: """ -def main(args: argparse.Namespace) -> None: - threshold = 10 - print(f"Least b with > {threshold} b-eautiful integers") - print(f"Answer: {least_beautiful_base(threshold)}") - - -if __name__ == "__main__": +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", + "--threshold", type=int, - default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", - ) - parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", + default=10, + help="Find the least base with more than this many b-eautiful integers", ) args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) + print(f"Least b with > {args.threshold} b-eautiful integers") + print(f"Answer: {least_beautiful_base(args.threshold)}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/lineup.py b/docs/source/llm_examples/lineup.py index 42f526250..86b850234 100644 --- a/docs/source/llm_examples/lineup.py +++ b/docs/source/llm_examples/lineup.py @@ -7,28 +7,10 @@ import argparse import collections.abc -import contextlib import dataclasses -import os -import pathlib import typing -import tenacity - from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled @dataclasses.dataclass(frozen=True) @@ -74,60 +56,53 @@ def solve_lineup(n: int, clues: collections.abc.Sequence[LineupClue]) -> list[in """ -def main(args: argparse.Namespace) -> None: - puzzle = [ - LineupClue("imm_left", 0, 1), - LineupClue("imm_left", 1, 2), - LineupClue("at", 3, 5), - LineupClue("left", 4, 0), - ] - print(f"Zebra-style ordering puzzle: n=5, clues={puzzle}") - print(f"Answer: {solve_lineup(5, puzzle)}") - - -if __name__ == "__main__": +def main() -> None: + kinds = typing.get_args(LineupClue.__annotations__["kind"]) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", + "--n", type=int, default=5, - help="Number of retries for malformed/failing LLM output", - ) - parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", - ) - parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", + help="Number of people in the line (used with --clue)", ) parser.add_argument( - "--dump-system-prompt", - type=str, + "--clue", + dest="clues", + action="append", + nargs=3, + metavar=("KIND", "A", "B"), default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", + help=( + f"An ordering constraint 'KIND A B' where KIND is one of " + f"{'/'.join(kinds)} (e.g. --clue imm_left 0 1); repeatable" + ), ) args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) + + if args.clues is not None: + n = args.n + clues = [] + for kind, a, b in args.clues: + if kind not in kinds: + parser.error( + f"invalid clue kind {kind!r}; choose from {'/'.join(kinds)}" + ) + try: + clues.append(LineupClue(kind, int(a), int(b))) + except ValueError: + parser.error(f"clue positions must be integers, got {a!r} {b!r}") + else: + n = 5 + clues = [ + LineupClue("imm_left", 0, 1), + LineupClue("imm_left", 1, 2), + LineupClue("at", 3, 5), + LineupClue("left", 4, 0), + ] + + print(f"Zebra-style ordering puzzle: n={n}, clues={clues}") + print(f"Answer: {solve_lineup(n, clues)}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/musr.py b/docs/source/llm_examples/musr.py index 63a21d9d2..edde9db9f 100644 --- a/docs/source/llm_examples/musr.py +++ b/docs/source/llm_examples/musr.py @@ -7,25 +7,8 @@ import argparse import collections.abc -import contextlib -import os -import pathlib - -import tenacity from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import ( - LangfuseTracer, - LexicalReaders, - LiteLLMProvider, - PythonRepl, - RetryLLMHandler, - SynthesizeAndCall, - SystemPromptDumper, - TerminalRenderer, -) -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler @Template.define @@ -54,7 +37,7 @@ def musr_object_placement( """ -def main(args: argparse.Namespace) -> None: +def main() -> None: STUDIO_STORY = """\ In the heart of the bustling studio, Ricky, Emma, and Danny readied themselves \ for a day of creating magic. Ricky, the gifted singer-songwriter, had his \ @@ -75,57 +58,38 @@ def main(args: argparse.Namespace) -> None: his attention amidst his routine of tidying up. Meanwhile Emma, from inside a \ sound-proofed booth, was lost in reviewing already-recorded tracks, out of \ Danny's view.""" - person = "Danny" - item = "earphones" - locations = ["piano", "producer's desk", "recording booth"] - answer = musr_object_placement(STUDIO_STORY, person, item, locations) - print(f"MuSR: where would {person} look for the {item}?") - print(f"Answer: {answer}") - - -if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", + "--story", type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed/failing LLM output", + default=STUDIO_STORY, + help="The narrative describing where the item is moved and who saw it", ) parser.add_argument( - "--langfuse", - action="store_true", - help="Whether to log LLM calls and metadata to Langfuse", + "--person", + type=str, + default="Danny", + help="The person whose belief about the item's location is queried", ) parser.add_argument( - "--render", - action="store_true", - help="Live-render the streaming message history in the terminal", + "--item", + type=str, + default="earphones", + help="The object being tracked", ) parser.add_argument( - "--dump-system-prompt", - type=str, - default=None, - metavar="PATH", - help="Dump the assembled system prompt to this Markdown file", + "--locations", + nargs="+", + default=["piano", "producer's desk", "recording booth"], + metavar="LOCATION", + help="Candidate locations to choose the answer from", ) args = parser.parse_args() - with ( - handler(LiteLLMProvider(model=args.model, tool_choice="required", api_base="http://localhost:8030/v1", api_key="")), - handler(TerminalRenderer()) if args.render else contextlib.nullcontext(), - handler(SystemPromptDumper(path=pathlib.Path(args.dump_system_prompt))) - if args.dump_system_prompt - else contextlib.nullcontext(), - handler(UnsafeEvalProvider()), - handler(PythonRepl()), - handler(SynthesizeAndCall()), - handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(args.num_retries))), - handler(LexicalReaders()), - handler(LangfuseTracer()) if args.langfuse else contextlib.nullcontext(), - ): - main(args) + + answer = musr_object_placement(args.story, args.person, args.item, args.locations) + print(f"MuSR: where would {args.person} look for the {args.item}?") + print(f"Answer: {answer}") + + +if __name__ == "__main__": + main() diff --git a/effectful/handlers/llm/harness.py b/effectful/handlers/llm/harness.py new file mode 100644 index 000000000..ec80bf453 --- /dev/null +++ b/effectful/handlers/llm/harness.py @@ -0,0 +1,193 @@ +"""A reusable harness for running `effectful.handlers.llm` example scripts. + +The example scripts under ``docs/source/llm_examples`` share a fixed stack of +handlers -- a LiteLLM provider, a Python REPL, retry/decoding logic, and so on -- +that turns a bare `Template`/`Agent` into something runnable. This module +factors that stack into a single object, `harness`, so the scripts themselves +carry none of the boilerplate. + +`harness` is a `contextlib.ContextDecorator`, so it can be used programmatically +either as a context manager or as a decorator:: + + with harness(model="gpt-4o", render=True): + main() + + @harness(model="gpt-4o") + def main() -> None: + ... + +Run as a module it becomes a command-line launcher that wraps an arbitrary +script in the same context:: + + python -m effectful.handlers.llm.harness + +Harness flags (``--model``, ``--num-retries``, ``--langfuse``, ``--render``, +``--dump-system-prompt``) are consumed here; every other flag is passed through +to the script unchanged. +""" + +import argparse +import contextlib +import os +import pathlib +import runpy +import sys + +import tenacity + +from effectful.handlers.llm.completions import ( + LangfuseTracer, + LexicalReaders, + LiteLLMProvider, + PythonRepl, + RetryLLMHandler, + SynthesizeAndCall, + SystemPromptDumper, + TerminalRenderer, +) +from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.ops.semantics import handler + + +class harness(contextlib.ContextDecorator): + """Install the standard `effectful.handlers.llm` handler stack. + + Constructing a `harness` records the configuration; entering it (as a + context manager, decorator, or via the module CLI) installs the handlers and + exiting removes them. The handlers, in installation order, are: + + 1. `LiteLLMProvider` -- the model backend. + 2. `TerminalRenderer` -- live-render the streaming history (if ``render``). + 3. `SystemPromptDumper` -- dump the system prompt (if ``dump_system_prompt``). + 4. `UnsafeEvalProvider` and `PythonRepl` -- run model-authored Python. + 5. `SynthesizeAndCall` -- synthesize a function and call it. + 6. `RetryLLMHandler` -- retry malformed/failing model output. + 7. `LexicalReaders` -- expose lexically-scoped tools to the model. + 8. `LangfuseTracer` -- log calls to Langfuse (if ``langfuse``). + + Args: + model: LLM model to use. + num_retries: Attempts for malformed/failing model output. + langfuse: Log LLM calls and metadata to Langfuse. + render: Live-render the streaming message history in the terminal. + dump_system_prompt: If set, dump the assembled system prompt to this + Markdown file. + tool_choice: ``tool_choice`` forwarded to the provider. + api_base: API base URL forwarded to the provider. + api_key: API key forwarded to the provider. + """ + + def __init__( + self, + *, + model: str = "", + num_retries: int = 5, + langfuse: bool = False, + render: bool = False, + dump_system_prompt: str | os.PathLike[str] | None = None, + tool_choice: str = "required", + api_base: str = "http://localhost:8030/v1", + api_key: str = "", + ) -> None: + self.model = model + self.num_retries = num_retries + self.langfuse = langfuse + self.render = render + self.dump_system_prompt = dump_system_prompt + self.tool_choice = tool_choice + self.api_base = api_base + self.api_key = api_key + + def __enter__(self) -> "harness": + stack = contextlib.ExitStack() + stack.enter_context( + handler( + LiteLLMProvider( + model=self.model, + tool_choice=self.tool_choice, + api_base=self.api_base, + api_key=self.api_key, + ) + ) + ) + if self.render: + stack.enter_context(handler(TerminalRenderer())) + if self.dump_system_prompt: + stack.enter_context( + handler(SystemPromptDumper(path=pathlib.Path(self.dump_system_prompt))) + ) + stack.enter_context(handler(UnsafeEvalProvider())) + stack.enter_context(handler(PythonRepl())) + stack.enter_context(handler(SynthesizeAndCall())) + stack.enter_context( + handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(self.num_retries))) + ) + stack.enter_context(handler(LexicalReaders())) + if self.langfuse: + stack.enter_context(handler(LangfuseTracer())) + self._stack = stack + return self + + def __exit__(self, *exc_info) -> bool | None: + return self._stack.__exit__(*exc_info) + + +def _parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: + """Split ``argv`` into harness options and pass-through script flags.""" + parser = argparse.ArgumentParser( + prog=f"python -m {__spec__.name}" if __spec__ else None, + description=( + "Run an effectful.handlers.llm script under the standard handler " + "stack. Flags other than the harness flags below are passed through " + "to the script unchanged." + ), + ) + parser.add_argument("script", help="Path to the script to run") + parser.add_argument( + "--model", + type=str, + default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), + help="LLM model to use", + ) + parser.add_argument( + "--num-retries", + type=int, + default=5, + help="Number of retries for malformed/failing LLM output", + ) + parser.add_argument( + "--langfuse", + action="store_true", + help="Whether to log LLM calls and metadata to Langfuse", + ) + parser.add_argument( + "--render", + action="store_true", + help="Live-render the streaming message history in the terminal", + ) + parser.add_argument( + "--dump-system-prompt", + type=str, + default=None, + metavar="PATH", + help="Dump the assembled system prompt to this Markdown file", + ) + return parser.parse_known_args(argv) + + +def main(argv: list[str] | None = None) -> None: + ns, script_args = _parse_args(sys.argv[1:] if argv is None else argv) + # The script should see only its own flags, under its own name. + sys.argv = [ns.script, *script_args] + with harness( + model=ns.model, + num_retries=ns.num_retries, + langfuse=ns.langfuse, + render=ns.render, + dump_system_prompt=ns.dump_system_prompt, + ): + runpy.run_path(ns.script, run_name="__main__") + + +if __name__ == "__main__": + main() From a3cf92bdb756c8ada20c8def2c739d1af2922ff5 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 14:55:13 -0400 Subject: [PATCH 033/155] standardize harness --- docs/source/llm_examples/async_concurrency.py | 45 +++------ docs/source/llm_examples/batch_translate.py | 70 -------------- docs/source/llm_examples/chat_memory.py | 33 ++----- docs/source/llm_examples/chat_search.py | 45 +++------ docs/source/llm_examples/decode_callable.py | 54 +++-------- docs/source/llm_examples/flight_booking.py | 49 +++------- docs/source/llm_examples/guardrails.py | 38 ++------ .../llm_examples/hanoi_solver_iterative.py | 33 ++----- .../llm_examples/hanoi_solver_recursive.py | 45 +++------ .../llm_examples/higher_order_function.py | 61 +++--------- docs/source/llm_examples/hitl.py | 80 ++++++---------- docs/source/llm_examples/image_input.py | 24 ++--- docs/source/llm_examples/image_tool.py | 29 ++---- docs/source/llm_examples/majority_vote.py | 31 ++----- docs/source/llm_examples/map_reduce.py | 50 ++-------- docs/source/llm_examples/multi_agent.py | 38 ++------ docs/source/llm_examples/prompt_templates.py | 86 ----------------- docs/source/llm_examples/rag.py | 45 +++------ docs/source/llm_examples/research_agent.py | 30 ++---- .../{retry_validation.py => retry.py} | 93 +++++++++---------- docs/source/llm_examples/retry_tool_errors.py | 91 ------------------ docs/source/llm_examples/structured_output.py | 24 ++--- docs/source/llm_examples/supervisor.py | 41 ++------ docs/source/llm_examples/tao_agent.py | 46 +++------ .../llm_examples/template_composition.py | 31 ++----- docs/source/llm_examples/text2sql.py | 53 +++-------- docs/source/llm_examples/thinking.py | 43 +++------ docs/source/llm_examples/tool_calling.py | 26 +----- 28 files changed, 296 insertions(+), 1038 deletions(-) delete mode 100644 docs/source/llm_examples/batch_translate.py delete mode 100644 docs/source/llm_examples/prompt_templates.py rename docs/source/llm_examples/{retry_validation.py => retry.py} (54%) delete mode 100644 docs/source/llm_examples/retry_tool_errors.py diff --git a/docs/source/llm_examples/async_concurrency.py b/docs/source/llm_examples/async_concurrency.py index b47170610..f49cfc1e9 100644 --- a/docs/source/llm_examples/async_concurrency.py +++ b/docs/source/llm_examples/async_concurrency.py @@ -5,15 +5,10 @@ - Using ``asyncio.to_thread`` to run synchronous template calls in parallel """ -import argparse import asyncio import functools -import os from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Async template @@ -24,7 +19,6 @@ def analyze_average_age(ages: list[int]) -> int: """Analyze the dataset of ages {ages} and return the average age of participants. Do not use any tools.""" - raise NotHandled # --------------------------------------------------------------------------- @@ -32,30 +26,21 @@ def analyze_average_age(ages: list[int]) -> int: # --------------------------------------------------------------------------- -async def main(provider: LiteLLMProvider): - analysis = functools.partial( - asyncio.to_thread, handler(provider)(analyze_average_age) - ) - results = await asyncio.gather( - analysis([25, 30, 35, 40]), - analysis([20, 28, 17, 30]), - analysis([22, 27, 31, 29]), - analysis([24, 26, 32, 38]), - analysis([21, 29, 33, 37]), - ) - for i, result in enumerate(results): - print(f"Group {i}: average age = {result}") +def main() -> None: + async def run() -> None: + analysis = functools.partial(asyncio.to_thread, analyze_average_age) + results = await asyncio.gather( + analysis([25, 30, 35, 40]), + analysis([20, 28, 17, 30]), + analysis([22, 27, 31, 29]), + analysis([24, 26, 32, 38]), + analysis([21, 29, 33, 37]), + ) + for i, result in enumerate(results): + print(f"Group {i}: average age = {result}") + + asyncio.run(run()) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Analyze average ages concurrently") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - asyncio.run(main(provider)) + main() diff --git a/docs/source/llm_examples/batch_translate.py b/docs/source/llm_examples/batch_translate.py deleted file mode 100644 index 66b4999f8..000000000 --- a/docs/source/llm_examples/batch_translate.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Batch translation with instruction injection. - -Demonstrates: -- ``@Template.define`` for a translation template with injected instructions -""" - -import argparse -import os - -from tenacity import stop_after_attempt - -from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.handlers.llm.evaluation import RestrictedEvalProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled - -# --------------------------------------------------------------------------- -# Translation template -# --------------------------------------------------------------------------- - - -@Template.define -def translate(target_language: str, instructions: str = "") -> Template[[str], str]: - """ - Write a `Template` that translates a string of English text into {target_language} - If any instructions are provided, include them in the prompt: {instructions} - """ - raise NotHandled - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Batch translation with instruction injection" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--max-steps", - type=int, - default=5, - help="Maximum number of steps before giving up", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed LLM output", - ) - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - handler(RestrictedEvalProvider()), - ): - translator = translate( - target_language="french", instructions="Use formal language." - ) - print(translator("hello, how are you? how is your day going?")) diff --git a/docs/source/llm_examples/chat_memory.py b/docs/source/llm_examples/chat_memory.py index b926cdff5..629371518 100644 --- a/docs/source/llm_examples/chat_memory.py +++ b/docs/source/llm_examples/chat_memory.py @@ -6,17 +6,12 @@ - Simple in-memory vector store with L2 distance """ -import argparse import dataclasses -import os import litellm import numpy as np from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Embedding helpers @@ -59,7 +54,6 @@ def respond_to_user( Continue the conversation. The last few messages were: {prev_messages} Older relevant context: {relevant_context}""" - raise NotHandled # --------------------------------------------------------------------------- @@ -102,23 +96,14 @@ def chat(self, user_input: str): # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Chat agent with embedding-based memory" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - args = parser.parse_args() - +def main() -> None: agent = ChatAgent() - provider = LiteLLMProvider(model=args.model) - with handler(provider): - agent.chat("Hello! How are you doing?") - agent.chat("Lovely! I'm having a great day.") - agent.chat("What is the capital of France?") - agent.chat("I didn't know that! That's amazing!") + agent.chat("Hello! How are you doing?") + agent.chat("Lovely! I'm having a great day.") + agent.chat("What is the capital of France?") + agent.chat("I didn't know that! That's amazing!") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/chat_search.py b/docs/source/llm_examples/chat_search.py index a2d7cecd5..46859c8ab 100644 --- a/docs/source/llm_examples/chat_search.py +++ b/docs/source/llm_examples/chat_search.py @@ -1,15 +1,10 @@ import argparse import dataclasses -import os import urllib.parse import requests -from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled @Tool.define @@ -68,19 +63,12 @@ def send(self, user_input: str) -> str: The user writes: {user_input} """ - raise NotHandled -if __name__ == "__main__": +def main() -> None: parser = argparse.ArgumentParser( description="LLM-guided research agent with web search" ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) parser.add_argument( "--name", type=str, @@ -92,25 +80,18 @@ def send(self, user_input: str) -> str: action="store_true", help="Run in interactive mode, allowing multiple back-and-forth messages", ) - parser.add_argument( - "--num-retries", - type=int, - default=4, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() chatbot = ChatBot(bot_name=args.name) - provider = LiteLLMProvider(model=args.model) - - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - if args.interactive: - while True: - print(chatbot.send(input("You: "))) - else: - print(chatbot.send("Hi! Can you tell me about the Statue of Liberty?")) - print(chatbot.send("Who designed it?")) - print(chatbot.send("What about the speed of light? How fast is it?")) + + if args.interactive: + while True: + print(chatbot.send(input("You: "))) + else: + print(chatbot.send("Hi! Can you tell me about the Statue of Liberty?")) + print(chatbot.send("Who designed it?")) + print(chatbot.send("What about the speed of light? How fast is it?")) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/decode_callable.py b/docs/source/llm_examples/decode_callable.py index 195167bce..1181f01c0 100644 --- a/docs/source/llm_examples/decode_callable.py +++ b/docs/source/llm_examples/decode_callable.py @@ -9,16 +9,9 @@ import argparse import inspect -import os from collections.abc import Callable -from tenacity import stop_after_attempt - from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Templates @@ -28,35 +21,19 @@ @Template.define def primes(first_digit: int) -> int: """Give a prime number with {first_digit} as the first digit. Do not use any tools.""" - raise NotHandled @Template.define def count_char(char: str) -> Callable[[str], int]: """Write a function which takes a string and counts the occurrances of '{char}'. Do not use any tools.""" - raise NotHandled # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Decode LLM responses to Python objects (incl. callables)" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed LLM output", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--first-digit", type=int, @@ -71,20 +48,17 @@ def count_char(char: str) -> Callable[[str], int]: ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) + prime = primes(args.first_digit) + assert type(prime) is int + print(f"Prime starting with {args.first_digit}: {prime}") - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - handler(UnsafeEvalProvider()), - ): - prime = primes(args.first_digit) - assert type(prime) is int - print(f"Prime starting with {args.first_digit}: {prime}") + counter = count_char(args.char) + assert callable(counter) + print("\nGenerated function:") + print(inspect.getsource(counter)) + print(f'counter("banana") == {counter("banana")}') + print(f'counter("cherry") == {counter("cherry")}') - counter = count_char(args.char) - assert callable(counter) - print("\nGenerated function:") - print(inspect.getsource(counter)) - print(f'counter("banana") == {counter("banana")}') - print(f'counter("cherry") == {counter("cherry")}') + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/flight_booking.py b/docs/source/llm_examples/flight_booking.py index d4de2e97a..a8314c698 100644 --- a/docs/source/llm_examples/flight_booking.py +++ b/docs/source/llm_examples/flight_booking.py @@ -12,15 +12,9 @@ import dataclasses import datetime import enum -import os from typing import Literal -from tenacity import stop_after_attempt - from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Structured output types @@ -85,7 +79,6 @@ def extract_flights(web_page_text: str) -> list[FlightDetails]: {web_page_text} """ - raise NotHandled # --------------------------------------------------------------------------- @@ -120,7 +113,6 @@ def find_flight( select the cheapest one that matches the origin, destination, and date exactly. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -141,7 +133,6 @@ def select_seat(self, user_input: str) -> SeatPreference: Row 1 is the front row with extra legroom. Rows 14 and 20 also have extra legroom. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -222,38 +213,22 @@ def book_flight( # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Flight booking with multi-agent delegation" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--interactive", action="store_true", help="Run in interactive mode with user prompts", ) - parser.add_argument( - "--num-retries", - type=int, - default=4, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - book_flight( - origin=Airport.SFO, - destination=Airport.ANC, - date=datetime.date(2025, 1, 10), - interactive=args.interactive, - ) + book_flight( + origin=Airport.SFO, + destination=Airport.ANC, + date=datetime.date(2025, 1, 10), + interactive=args.interactive, + ) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/guardrails.py b/docs/source/llm_examples/guardrails.py index 6ac800572..82e8b6d2f 100644 --- a/docs/source/llm_examples/guardrails.py +++ b/docs/source/llm_examples/guardrails.py @@ -5,15 +5,7 @@ - Simple control-flow gating based on LLM classification """ -import argparse -import os - -from tenacity import stop_after_attempt - from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Templates @@ -25,7 +17,6 @@ def travel_query(user_query: str) -> str: """ Produce a concise (<100 word) answer to: {user_query} """ - raise NotHandled # --------------------------------------------------------------------------- @@ -41,7 +32,6 @@ def is_safe_query(user_query: str) -> bool: """ Determine whether the user's query is purely related to travel advice: {user_query} """ - raise NotHandled if is_safe_query(user_query): return travel_query(user_query) @@ -53,26 +43,10 @@ def is_safe_query(user_query: str) -> bool: # Main # --------------------------------------------------------------------------- +def main() -> None: + print(answer_travel_query("What are great places to check out in NYC?")) + print(answer_travel_query("Should I buy apple stocks?")) + + if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Analyze average ages concurrently") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=4, - help="Number of retries for malformed LLM output", - ) - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - print(answer_travel_query("What are great places to check out in NYC?")) - print(answer_travel_query("Should I buy apple stocks?")) + main() diff --git a/docs/source/llm_examples/hanoi_solver_iterative.py b/docs/source/llm_examples/hanoi_solver_iterative.py index 0a5ecdbab..0b81573b4 100644 --- a/docs/source/llm_examples/hanoi_solver_iterative.py +++ b/docs/source/llm_examples/hanoi_solver_iterative.py @@ -11,15 +11,9 @@ import argparse import itertools -import os from dataclasses import dataclass, field -from tenacity import stop_after_attempt - from effectful.handlers.llm import Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Step model @@ -137,7 +131,6 @@ def predict(game_state: GameState) -> Step: rightmost tower). You MUST call get_valid_moves first to see which moves are legal, then pick the best one. Give a brief reasoning. """ - raise NotHandled return predict(state) @@ -170,14 +163,8 @@ def solve_hanoi(state: GameState, max_steps: int = 30): # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="LLM-guided Towers of Hanoi solver") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--game-size", type=int, @@ -190,18 +177,10 @@ def solve_hanoi(state: GameState, max_steps: int = 30): default=30, help="Maximum number of steps before giving up", ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) + solve_hanoi(GameState(size=args.game_size), max_steps=args.max_steps) - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - solve_hanoi(GameState(size=args.game_size), max_steps=args.max_steps) + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/hanoi_solver_recursive.py b/docs/source/llm_examples/hanoi_solver_recursive.py index b84387729..80c8eb2fc 100644 --- a/docs/source/llm_examples/hanoi_solver_recursive.py +++ b/docs/source/llm_examples/hanoi_solver_recursive.py @@ -25,17 +25,11 @@ """ import argparse -import os import typing from dataclasses import dataclass, field -from tenacity import stop_after_attempt - from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler from effectful.handlers.llm.template import IsRecursive -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Step model @@ -125,7 +119,6 @@ def solve( n_disks-1 disks from auxiliary to the target tower. 4. Return the concatenated list of all steps from (1), (2), and (3). """ - raise NotHandled # --------------------------------------------------------------------------- @@ -156,38 +149,22 @@ def validate_solution(size: int, steps: list[Step]) -> bool: # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Recursive LLM-based Towers of Hanoi solver" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--game-size", type=int, default=3, help="Number of disks in the Towers of Hanoi game", ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - n = args.game_size - print(f"Solving Tower of Hanoi with {n} disks...") - steps = solve(n_disks=n, source=0, target=n - 1, auxiliary=1) - print(f"\nLLM returned {len(steps)} steps. Validating...\n") - validate_solution(n, steps) + n = args.game_size + print(f"Solving Tower of Hanoi with {n} disks...") + steps = solve(n_disks=n, source=0, target=n - 1, auxiliary=1) + print(f"\nLLM returned {len(steps)} steps. Validating...\n") + validate_solution(n, steps) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/higher_order_function.py b/docs/source/llm_examples/higher_order_function.py index da2410959..5ffc26669 100644 --- a/docs/source/llm_examples/higher_order_function.py +++ b/docs/source/llm_examples/higher_order_function.py @@ -10,17 +10,10 @@ import argparse import inspect -import os from collections.abc import Callable from typing import Literal -from tenacity import stop_after_attempt - from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.handlers.llm.evaluation import UnsafeEvalProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Sub-templates the generated function may call @@ -30,13 +23,11 @@ @Template.define def write_chapter(chapter_number: int, chapter_name: str) -> str: """Write a short story about {chapter_number}. Do not use any tools.""" - raise NotHandled @Template.define def judge_chapter(story_so_far: str, chapter_number: int) -> bool: """Decide if the new chapter is coherent with the story so far. Do not use any tools.""" - raise NotHandled # --------------------------------------------------------------------------- @@ -46,30 +37,18 @@ def judge_chapter(story_so_far: str, chapter_number: int) -> bool: @Template.define def write_multi_chapter_story(style: Literal["moral", "funny"]) -> Callable[[str], str]: - """Generate a function that writes a story in style: {style} about the given topic. - - If you raise an exception, handle it yourself. - The program can use helper functions defined elsewhere (DO NOT REDEFINE THEM): - - write_chapter(chapter_number: int, chapter_name: str) -> str - - judge_chapter(story_so_far: str, chapter_number: int) -> bool """ - raise NotHandled + Generate a function that writes a story in style: {style} about the given topic. + """ # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Generate a higher-order function that calls sub-templates" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--topic", type=str, default="a curious cat", help="Story topic" ) @@ -80,26 +59,14 @@ def write_multi_chapter_story(style: Literal["moral", "funny"]) -> Callable[[str default="moral", help="Story style", ) - parser.add_argument( - "--num-retries", - type=int, - default=4, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - - print("Sub-templates available to write_multi_chapter_story:") - print(list(write_multi_chapter_story.tools.keys())) - - with ( - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - handler(provider), - handler(UnsafeEvalProvider()), - ): - print(f"\n=== Generating story function (style={args.style}) ===") - story_fn = write_multi_chapter_story(args.style) - print(inspect.getsource(story_fn)) - print(f"\n=== Running generated function on {args.topic!r} ===") - print(story_fn(args.topic)) + print(f"\n=== Generating story function (style={args.style}) ===") + story_fn = write_multi_chapter_story(args.style) + print(inspect.getsource(story_fn)) + print(f"\n=== Running generated function on {args.topic!r} ===") + print(story_fn(args.topic)) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/hitl.py b/docs/source/llm_examples/hitl.py index 540fc27c6..62ca5b512 100644 --- a/docs/source/llm_examples/hitl.py +++ b/docs/source/llm_examples/hitl.py @@ -11,14 +11,8 @@ import argparse import dataclasses import enum -import os - -from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Structured output @@ -39,29 +33,22 @@ class ProposedAction: details: str -# --------------------------------------------------------------------------- -# Simulated action execution -# --------------------------------------------------------------------------- - - -execution_log: list[str] = [] - - -@Tool.define -def execute_action(action: ActionType, details: str) -> str: - """Execute an approved action. Returns a confirmation message.""" - msg = f"[executed] {action}: {details}" - execution_log.append(msg) - return msg - - # --------------------------------------------------------------------------- # Planner agent # --------------------------------------------------------------------------- +@dataclasses.dataclass class Planner(Agent): """Agent that proposes actions one at a time for human approval.""" + execution_log: list[str] = dataclasses.field(default_factory=list) + + @Tool.define + def execute_action(self, action: ActionType, details: str) -> str: + """Execute an approved action. Returns a confirmation message.""" + msg = f"[executed] {action}: {details}" + self.execution_log.append(msg) + return msg @Template.define def propose_next(self, task: str, feedback: str) -> ProposedAction: @@ -78,7 +65,6 @@ def propose_next(self, task: str, feedback: str) -> ProposedAction: If a previous proposal was rejected, propose something different that addresses the feedback. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -113,28 +99,22 @@ def run_with_approval( approved = True if approved: - result = execute_action(proposal.action, proposal.details) + result = planner.execute_action(proposal.action, proposal.details) print(f" {result}") feedback = f"Approved and executed: {result}" else: print(f" [rejected] {answer}") feedback = f"Rejected: {answer}" - return list(execution_log) + return list(planner.execution_log) # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Human-in-the-loop task planner") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--interactive", action="store_true", @@ -146,32 +126,24 @@ def run_with_approval( default=5, help="Maximum number of action steps", ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - task = ( "Organize a team lunch for next Friday. " "Send an email to the team, create a shared document for " "restaurant suggestions, and schedule a meeting to finalize plans." ) - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - print(f"Task: {task}\n") - log = run_with_approval( - task, - interactive=args.interactive, - max_steps=args.max_steps, - ) - print(f"\nExecution log ({len(log)} actions):") - for entry in log: - print(f" {entry}") + print(f"Task: {task}\n") + log = run_with_approval( + task, + interactive=args.interactive, + max_steps=args.max_steps, + ) + print(f"\nExecution log ({len(log)} actions):") + for entry in log: + print(f" {entry}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/image_input.py b/docs/source/llm_examples/image_input.py index 14375b294..8444ef561 100644 --- a/docs/source/llm_examples/image_input.py +++ b/docs/source/llm_examples/image_input.py @@ -5,17 +5,12 @@ - Inline base64 image data so the script is self-contained """ -import argparse import base64 import io -import os from PIL import Image from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Inline image (32x32 yellow smiley face) @@ -39,25 +34,18 @@ def describe_image(image: Image.Image) -> str: """Return a short description of the following image. {image} """ - raise NotHandled # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Pass a PIL image to a template") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use (must support image inputs)", - ) - args = parser.parse_args() +def main() -> None: image = Image.open(io.BytesIO(base64.b64decode(IMAGE_BASE64))) - provider = LiteLLMProvider(model=args.model) - with handler(provider): - print(describe_image(image)) + print(describe_image(image)) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/image_tool.py b/docs/source/llm_examples/image_tool.py index 813058494..77df52255 100644 --- a/docs/source/llm_examples/image_tool.py +++ b/docs/source/llm_examples/image_tool.py @@ -1,15 +1,6 @@ -import argparse -import os - from PIL import Image from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import ( - LiteLLMProvider, - RetryLLMHandler, -) -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled class ImageTools(Agent): @@ -66,25 +57,17 @@ def _rotate_and_concat(self, i: int) -> int: the previous. """ - raise NotHandled def rotate_and_concat(self, i: Image.Image) -> Image.Image: return self._decode(self._rotate_and_concat(self._encode(i))) -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use (must support image inputs)", - ) - args = parser.parse_args() - +def main() -> None: image_agent = ImageTools() img = Image.open("../_static/img/chirho_logo_wide.png") - provider = LiteLLMProvider(model=args.model) - with handler(provider), handler(RetryLLMHandler()): - image_agent.rotate_and_concat(img).show() + image_agent.rotate_and_concat(img).show() + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/majority_vote.py b/docs/source/llm_examples/majority_vote.py index 25696ddcb..26606b56e 100644 --- a/docs/source/llm_examples/majority_vote.py +++ b/docs/source/llm_examples/majority_vote.py @@ -9,12 +9,8 @@ import collections import collections.abc import enum -import os from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Template @@ -32,7 +28,6 @@ def yes_or_no(question: str) -> Answer: """ Answer the following yes/no/maybe question: {question} """ - raise NotHandled # --------------------------------------------------------------------------- @@ -52,16 +47,8 @@ def majority_vote[Q]( # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Majority voting ensemble for yes/no questions" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--num-voters", type=int, default=3, help="Number of voters for majority vote" ) @@ -73,9 +60,11 @@ def majority_vote[Q]( ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - with handler(provider): - answer, count = majority_vote(yes_or_no, args.question, voters=args.num_voters) - print( - f"Question: {args.question}\nAnswer: {answer} (voted {count}/{args.num_voters})" - ) + answer, count = majority_vote(yes_or_no, args.question, voters=args.num_voters) + print( + f"Question: {args.question}\nAnswer: {answer} (voted {count}/{args.num_voters})" + ) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/map_reduce.py b/docs/source/llm_examples/map_reduce.py index 70a79c595..5f5bc5ad1 100644 --- a/docs/source/llm_examples/map_reduce.py +++ b/docs/source/llm_examples/map_reduce.py @@ -7,19 +7,12 @@ - Structured output with dataclasses """ -import argparse import asyncio import collections.abc import dataclasses import functools -import os - -from tenacity import stop_after_attempt from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Structured output @@ -52,7 +45,6 @@ def evaluate_resume(resume: str, job_description: str) -> Evaluation: Score from 1 (poor fit) to 10 (perfect fit). """ - raise NotHandled @Template.define @@ -70,7 +62,6 @@ def summarize_evaluations( Provide a brief summary: rank the candidates from best to worst, highlight the top candidate, and note any concerns. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -101,20 +92,12 @@ def summarize_evaluations( async def map_reduce_evaluate( - provider: LiteLLMProvider, resumes: list[str], job_description: str, ) -> str: """Evaluate resumes in parallel (map), then summarize (reduce).""" # Map: evaluate each resume concurrently - evaluate = functools.partial( - asyncio.to_thread, - handler(provider)( - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries)))( - evaluate_resume - ) - ), - ) + evaluate = functools.partial(asyncio.to_thread, evaluate_resume) evaluations: list[Evaluation] = list( await asyncio.gather(*(evaluate(resume, job_description) for resume in resumes)) ) @@ -126,35 +109,18 @@ async def map_reduce_evaluate( print(f" - {ev.weaknesses}") # Reduce: summarize all evaluations - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - return summarize_evaluations(job_description, evaluations) + return summarize_evaluations(job_description, evaluations) # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Map-reduce resume evaluation") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - +def main() -> None: print(f"Evaluating {len(RESUMES)} resumes for: {JOB_DESCRIPTION}\n") - summary = asyncio.run(map_reduce_evaluate(provider, RESUMES, JOB_DESCRIPTION)) + summary = asyncio.run(map_reduce_evaluate(RESUMES, JOB_DESCRIPTION)) print(f"\n{summary}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/multi_agent.py b/docs/source/llm_examples/multi_agent.py index 0389c6c87..caadfb8e8 100644 --- a/docs/source/llm_examples/multi_agent.py +++ b/docs/source/llm_examples/multi_agent.py @@ -10,14 +10,8 @@ import argparse import dataclasses import enum -import os - -from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Structured output @@ -73,7 +67,6 @@ def give_hint(self, guesser_response: str) -> str: The guesser's last response was: {guesser_response} """ - raise NotHandled class Guesser(Agent): @@ -89,7 +82,6 @@ def make_guess(self, hint: str) -> Guess: Review the conversation history for all previous hints. Make your best guess. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -130,26 +122,14 @@ def play_taboo( # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Multi-agent Taboo word guessing game") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--max-rounds", type=int, default=5, help="Maximum rounds per game", ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() games = [ @@ -157,12 +137,10 @@ def play_taboo( ("volcano", ["lava", "eruption", "mountain", "hot"]), ] - provider = LiteLLMProvider(model=args.model) + for secret, taboo in games: + print(f"\nGame: '{secret}' (taboo: {taboo})") + play_taboo(secret, taboo, max_rounds=args.max_rounds) - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - for secret, taboo in games: - print(f"\nGame: '{secret}' (taboo: {taboo})") - play_taboo(secret, taboo, max_rounds=args.max_rounds) + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/prompt_templates.py b/docs/source/llm_examples/prompt_templates.py deleted file mode 100644 index 74c1cf0d3..000000000 --- a/docs/source/llm_examples/prompt_templates.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Basic prompt templates and deterministic caching. - -Demonstrates: -- ``@Template.define`` for declaring an LLM-backed function -- Non-determinism: calling the same template twice yields different results -- ``functools.cache`` to make a template call deterministic in-process -- ``LiteLLMProvider(caching=True)`` with ``litellm.cache`` for cross-process caching -""" - -import argparse -import functools -import os - -import litellm -from litellm.caching.caching import Cache - -from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled - -# --------------------------------------------------------------------------- -# Templates -# --------------------------------------------------------------------------- - - -@Template.define -def limerick(theme: str) -> str: - """Write a limerick on the theme of {theme}. Do not use any tools.""" - raise NotHandled - - -@functools.cache -@Template.define -def haiku(theme: str) -> str: - """Write a haiku on the theme of {theme}. Do not use any tools.""" - raise NotHandled - - -@Template.define -def haiku_no_cache(theme: str) -> str: - """Write a haiku on the theme of {theme}. Do not use any tools.""" - raise NotHandled - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Basic prompt templates and deterministic caching" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument("--theme", type=str, default="fish", help="Theme for the poem") - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - - print("=== Non-deterministic limerick (two independent calls) ===") - with handler(provider): - print(limerick(args.theme)) - print("-" * 40) - print(limerick(args.theme)) - - print("\n=== functools.cache: same result on second call ===") - with handler(provider): - print(haiku(args.theme)) - print("-" * 40) - print(haiku(args.theme)) - - print("\n=== LiteLLMProvider(caching=True): backed by litellm.cache ===") - litellm.cache = Cache() - provider_cached = LiteLLMProvider(model=args.model, caching=True) - try: - with handler(provider_cached): - print(haiku_no_cache(args.theme)) - print("-" * 40) - print(haiku_no_cache(args.theme)) - finally: - litellm.cache = None diff --git a/docs/source/llm_examples/rag.py b/docs/source/llm_examples/rag.py index eca2b4507..c4751fd41 100644 --- a/docs/source/llm_examples/rag.py +++ b/docs/source/llm_examples/rag.py @@ -10,16 +10,11 @@ import argparse import dataclasses -import os import litellm import numpy as np -from tenacity import stop_after_attempt from effectful.handlers.llm import Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Embedding helpers @@ -142,40 +137,30 @@ def answer_question(question: str) -> str: Question: {question} """ - raise NotHandled # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Retrieval-augmented generation (RAG)") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--embedding-model", type=str, default="lm_studio/text-embedding-embeddinggemma-300m-qat", help="Embedding model to use", ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() # Offline: build the index index = build_index(DOCUMENTS, embedding_model=args.embedding_model) - # Create the retrieval tool bound to our index - retrieve: Tool = index.retrieve + # Create the retrieval tool bound to our index. `answer_question` is a + # module-level template, so the tool must be bound in module globals to be + # in its lexical scope. + global retrieve + retrieve = index.retrieve # Online: answer questions questions = [ @@ -184,13 +169,11 @@ def answer_question(question: str) -> str: "How many spectators could the Colosseum hold?", ] - provider = LiteLLMProvider(model=args.model) + for question in questions: + print(f"\nQ: {question}") + answer = answer_question(question) + print(f"A: {answer}") - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - for question in questions: - print(f"\nQ: {question}") - answer = answer_question(question) - print(f"A: {answer}") + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/research_agent.py b/docs/source/llm_examples/research_agent.py index 308d2df23..22c0cb863 100644 --- a/docs/source/llm_examples/research_agent.py +++ b/docs/source/llm_examples/research_agent.py @@ -8,17 +8,11 @@ """ import argparse -import os import urllib.parse import requests from effectful.handlers.llm import Template, Tool -from effectful.handlers.llm.completions import ( - LiteLLMProvider, -) -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Search effect + handler @@ -74,7 +68,6 @@ def search_web(query: str) -> str: def answer_question(question: str) -> str: """Acting as a research assistant that can search the web, construct an answer to the user's question: {question}.""" - raise NotHandled @Template.define @@ -82,7 +75,6 @@ def refine_answer(question: str, answer: str) -> str: """Acting as a research assistant that can search the web, given the user's original question ({question}), refine this previous answer: {answer}.""" - raise NotHandled @Template.define @@ -90,7 +82,6 @@ def is_question_answered(question: str, answer: str) -> bool: """Acting as a research assistant, decide if the user's question ({question}) is appropriately answered by: {answer}. Respond only true or false.""" - raise NotHandled # --------------------------------------------------------------------------- @@ -112,16 +103,8 @@ def research_agent(question: str, max_attempts: int = 3) -> str: # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="LLM-guided research agent with web search" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--question", type=str, @@ -130,8 +113,9 @@ def research_agent(question: str, max_attempts: int = 3) -> str: ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) + result = research_agent(args.question) + print(result) + - with handler(provider): - result = research_agent(args.question) - print(result) +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/retry_validation.py b/docs/source/llm_examples/retry.py similarity index 54% rename from docs/source/llm_examples/retry_validation.py rename to docs/source/llm_examples/retry.py index aab80df9e..515dfbc4b 100644 --- a/docs/source/llm_examples/retry_validation.py +++ b/docs/source/llm_examples/retry.py @@ -1,23 +1,21 @@ -"""Retrying when structured-output validation fails. +"""Retrying failed LLM output: validation errors and tool failures. Demonstrates: -- A pydantic dataclass with ``field_validator`` constraints - ``RetryLLMHandler`` feeding ``PydanticCustomError`` messages back to the LLM - so it can correct its output on a subsequent attempt + so it can correct structured output that fails validation +- ``RetryLLMHandler`` surfacing tool exceptions back to the LLM as tool messages, + so a flaky tool (``unstable_service``) can succeed after multiple attempts +- ``functools.cache`` to make a template call deterministic in-process """ import argparse -import os +import functools import pydantic from pydantic import field_validator from pydantic_core import PydanticCustomError -from tenacity import stop_after_attempt -from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled +from effectful.handlers.llm import Template, Tool # --------------------------------------------------------------------------- # Validated structured output @@ -53,55 +51,56 @@ def check_explanation_contains_score(cls, v, info): return v +@functools.cache +@Template.define +def give_rating_for_movie(movie_name: str) -> Rating: + """Give a rating for {movie_name}. The explanation MUST include the numeric score. Do not use any tools.""" + + # --------------------------------------------------------------------------- -# Template +# Flaky tool (unstable_service auto-captured from lexical scope) # --------------------------------------------------------------------------- +call_count = 0 +REQUIRED_RETRIES = 3 + + +@Tool.define +def unstable_service() -> str: + """Fetch data from an unstable external service. May require retries.""" + global call_count + call_count += 1 + if call_count < REQUIRED_RETRIES: + raise ConnectionError( + f"Service unavailable! Attempt {call_count}/{REQUIRED_RETRIES}. Please retry." + ) + return "{ 'status': 'ok', 'data': [1, 2, 3] }" + @Template.define -def give_rating_for_movie(movie_name: str) -> Rating: - """Give a rating for {movie_name}. The explanation MUST include the numeric score. Do not use any tools.""" - raise NotHandled +def fetch_data() -> str: + """Use the unstable_service tool to fetch data.""" # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Retry on pydantic validation errors in LLM responses" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--movie", type=str, default="Die Hard", help="Movie to rate") - parser.add_argument( - "--num-retries", - type=int, - default=4, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - - print("=== Without RetryLLMHandler ===") - with handler(provider): - try: - rating = give_rating_for_movie(args.movie) - print(f"Score: {rating.score}/5\nExplanation: {rating.explanation}") - except Exception as e: - print(f"Error: {e}") - - print("\n=== With RetryLLMHandler ===") - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - rating = give_rating_for_movie(args.movie) - print(f"Score: {rating.score}/5") - print(f"Explanation: {rating.explanation}") + print("=== Retrying structured-output validation ===") + rating = give_rating_for_movie(args.movie) + print(f"Score: {rating.score}/5") + print(f"Explanation: {rating.explanation}") + + print("\n=== Retrying tool execution failures ===") + result = fetch_data() + print(f"Result: {result} (after {call_count} tool attempts)") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/retry_tool_errors.py b/docs/source/llm_examples/retry_tool_errors.py deleted file mode 100644 index 7ce0b7b11..000000000 --- a/docs/source/llm_examples/retry_tool_errors.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Retrying tool execution failures. - -Demonstrates: -- ``RetryLLMHandler`` surfacing tool exceptions back to the LLM as tool messages -- A flaky tool (``unstable_service``) that succeeds only after multiple attempts -- The contrast between an unhandled failure and a retry-handled success -""" - -import argparse -import os - -from tenacity import stop_after_attempt - -from effectful.handlers.llm import Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled - -# --------------------------------------------------------------------------- -# Flaky tool -# --------------------------------------------------------------------------- - -call_count = 0 -REQUIRED_RETRIES = 3 - - -@Tool.define -def unstable_service() -> str: - """Fetch data from an unstable external service. May require retries.""" - global call_count - call_count += 1 - if call_count < REQUIRED_RETRIES: - raise ConnectionError( - f"Service unavailable! Attempt {call_count}/{REQUIRED_RETRIES}. Please retry." - ) - return "{ 'status': 'ok', 'data': [1, 2, 3] }" - - -# --------------------------------------------------------------------------- -# Template (unstable_service auto-captured from lexical scope) -# --------------------------------------------------------------------------- - - -@Template.define -def fetch_data() -> str: - """Use the unstable_service tool to fetch data.""" - raise NotHandled - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Retry LLM template calls when tools raise exceptions" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=4, - help="Number of retries for tool/decode failures", - ) - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - - print("=== Without RetryLLMHandler ===") - with handler(provider): - try: - result = fetch_data() - print(f"Result: {result}") - except Exception as e: - print(f"Error: {e}") - - # Reset for the retry-enabled run. - call_count = 0 - - print("\n=== With RetryLLMHandler ===") - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - result = fetch_data() - print(f"Result: {result} (after {call_count} tool attempts)") diff --git a/docs/source/llm_examples/structured_output.py b/docs/source/llm_examples/structured_output.py index 0f6c85f88..3273acdaa 100644 --- a/docs/source/llm_examples/structured_output.py +++ b/docs/source/llm_examples/structured_output.py @@ -7,12 +7,8 @@ import argparse import dataclasses -import os from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Structured output @@ -33,13 +29,11 @@ class KnockKnockJoke: @Template.define def write_joke(theme: str) -> KnockKnockJoke: """Write a knock-knock joke on the theme of {theme}. Do not use any tools.""" - raise NotHandled @Template.define def rate_joke(joke: KnockKnockJoke) -> bool: """Decide if {joke} is funny or not. Do not use any tools.""" - raise NotHandled # --------------------------------------------------------------------------- @@ -64,19 +58,15 @@ def do_comedy(theme: str) -> None: # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Structured output via dataclasses") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--theme", type=str, default="lizards", help="Theme for the joke" ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - with handler(provider): - do_comedy(args.theme) + do_comedy(args.theme) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/supervisor.py b/docs/source/llm_examples/supervisor.py index 29f258fe0..b26d9b955 100644 --- a/docs/source/llm_examples/supervisor.py +++ b/docs/source/llm_examples/supervisor.py @@ -8,16 +8,11 @@ import argparse import dataclasses -import os import urllib.parse import requests -from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Search tool @@ -90,7 +85,6 @@ def answer(self, question: str) -> str: Question: {question} """ - raise NotHandled # --------------------------------------------------------------------------- @@ -110,7 +104,6 @@ def judge_quality(question: str, answer: str) -> QualityJudgment: numbers) relevant to the question. Vague or generic answers should be rejected. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -144,16 +137,10 @@ def supervised_research(question: str, max_retries: int = 3) -> str: # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": +def main() -> None: parser = argparse.ArgumentParser( description="Supervised research agent with quality control" ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) parser.add_argument( "--question", type=str, @@ -166,22 +153,14 @@ def supervised_research(question: str, max_retries: int = 3) -> str: default=3, help="Maximum number of supervisor rejections before accepting", ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - result = supervised_research( - args.question, - max_retries=args.max_retries, - ) - print(f"\nFinal answer: {result}") + result = supervised_research( + args.question, + max_retries=args.max_retries, + ) + print(f"\nFinal answer: {result}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/tao_agent.py b/docs/source/llm_examples/tao_agent.py index 2a8a14717..93fe6a6a8 100644 --- a/docs/source/llm_examples/tao_agent.py +++ b/docs/source/llm_examples/tao_agent.py @@ -10,19 +10,11 @@ import argparse import dataclasses import enum -import os import urllib.parse import requests -from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template, Tool -from effectful.handlers.llm.completions import ( - LiteLLMProvider, - RetryLLMHandler, -) -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Search tool @@ -102,7 +94,6 @@ def think(self, query: str) -> AgentThought: ({query}) and prior conversation context, think about what action to take next. """ - raise NotHandled @Template.define def observe(self, action: str, action_input: str, action_result: str) -> str: @@ -116,7 +107,6 @@ def observe(self, action: str, action_input: str, action_result: str) -> str: Do not make decisions, just describe what you see. """ - raise NotHandled def run(self, query: str, max_steps: int = 5) -> str: result = "" @@ -145,38 +135,24 @@ def _act(self, action: AgentAction, action_input: str) -> str: # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="TAO chain-of-thought agent") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--max-steps", type=int, default=5, help="Maximum number of steps before giving up", ) - parser.add_argument( - "--num-retries", - type=int, - default=5, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - agent = TAOAgent() - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - answer = agent.run( - "How many tennis balls would fill an Olympic swimming pool?", - max_steps=args.max_steps, - ) - print("Answer:", answer) + answer = agent.run( + "How many tennis balls would fill an Olympic swimming pool?", + max_steps=args.max_steps, + ) + print("Answer:", answer) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/template_composition.py b/docs/source/llm_examples/template_composition.py index 5f8803078..3e87b71ed 100644 --- a/docs/source/llm_examples/template_composition.py +++ b/docs/source/llm_examples/template_composition.py @@ -7,12 +7,8 @@ """ import argparse -import os from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Sub-templates @@ -22,13 +18,11 @@ @Template.define def story_with_moral(topic: str) -> str: """Write a short story about {topic} and end with a moral lesson. Do not use any tools.""" - raise NotHandled @Template.define def story_funny(topic: str) -> str: """Write a funny, humorous story about {topic}. Do not use any tools.""" - raise NotHandled # --------------------------------------------------------------------------- @@ -42,35 +36,26 @@ def write_story(topic: str, style: str) -> str: Available styles: 'moral' for a story with a lesson, 'funny' for humor. Use story_funny for humor, story_with_moral for a story with a lesson. """ - raise NotHandled # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": +def main() -> None: parser = argparse.ArgumentParser( description="Template composition with auto-captured sub-templates" ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) parser.add_argument( "--topic", type=str, default="a curious cat", help="Story topic" ) args = parser.parse_args() - assert story_with_moral in write_story.tools.values() - assert story_funny in write_story.tools.values() - print("Sub-templates available to write_story:", list(write_story.tools.keys())) + print("\n=== Story with moral ===") + print(write_story(args.topic, "moral")) + print("\n=== Funny story ===") + print(write_story(args.topic, "funny")) - provider = LiteLLMProvider(model=args.model) - with handler(provider): - print("\n=== Story with moral ===") - print(write_story(args.topic, "moral")) - print("\n=== Funny story ===") - print(write_story(args.topic, "funny")) + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/text2sql.py b/docs/source/llm_examples/text2sql.py index a3e36f933..05d42f453 100644 --- a/docs/source/llm_examples/text2sql.py +++ b/docs/source/llm_examples/text2sql.py @@ -7,17 +7,10 @@ - ``@Tool.define`` to expose the database schema as a tool """ -import argparse -import os import sqlite3 import textwrap -from tenacity import stop_after_attempt - from effectful.handlers.llm import Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # In-memory database setup @@ -78,7 +71,6 @@ def generate_sql(question: str, db_schema: str) -> str: Return ONLY the SQL query, no explanation. """ - raise NotHandled @Template.define @@ -94,7 +86,6 @@ def fix_sql(question: str, db_schema: str, bad_sql: str, error: str) -> str: Write a corrected SQLite query. Return ONLY the SQL query. """ - raise NotHandled # --------------------------------------------------------------------------- @@ -133,26 +124,8 @@ def text_to_sql( # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Natural language to SQL with LLM-powered debug loop" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) - args = parser.parse_args() - +def main() -> None: conn = create_sample_db() - provider = LiteLLMProvider(model=args.model) questions = [ "What is the average salary by department?", @@ -160,15 +133,15 @@ def text_to_sql( "How many employees were hired after 2021?", ] - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - for question in questions: - print(f"\nQ: {question}") - try: - rows = text_to_sql(conn, question) - for row in rows: - print(f" => {row}") - except Exception as e: - print(f" FAILED: {e}") + for question in questions: + print(f"\nQ: {question}") + try: + rows = text_to_sql(conn, question) + for row in rows: + print(f" => {row}") + except Exception as e: + print(f" FAILED: {e}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/thinking.py b/docs/source/llm_examples/thinking.py index 058de61ef..bd26f9252 100644 --- a/docs/source/llm_examples/thinking.py +++ b/docs/source/llm_examples/thinking.py @@ -8,14 +8,8 @@ import argparse import dataclasses -import os - -from tenacity import stop_after_attempt from effectful.handlers.llm import Agent, Template -from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Structured output @@ -48,7 +42,6 @@ def think(self, problem: str) -> ThoughtStep: logical steps. Set is_final=true only when you have a complete, well-supported answer. """ - raise NotHandled def solve(self, problem: str, max_steps: int = 10) -> str: """Solve a problem by iterative chain-of-thought reasoning.""" @@ -65,14 +58,8 @@ def solve(self, problem: str, max_steps: int = 10) -> str: # Main # --------------------------------------------------------------------------- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Chain-of-thought reasoning agent") - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--max-steps", type=int, @@ -88,16 +75,8 @@ def solve(self, problem: str, max_steps: int = 10) -> str: ), help="The problem to solve", ) - parser.add_argument( - "--num-retries", - type=int, - default=3, - help="Number of retries for malformed LLM output", - ) args = parser.parse_args() - provider = LiteLLMProvider(model=args.model) - problems = [ args.problem, ( @@ -106,12 +85,12 @@ def solve(self, problem: str, max_steps: int = 10) -> str: ), ] - with ( - handler(provider), - handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))), - ): - for problem in problems: - thinker = Thinker() - print(f"\nProblem: {problem}") - answer = thinker.solve(problem, max_steps=args.max_steps) - print(f"Answer: {answer}") + for problem in problems: + thinker = Thinker() + print(f"\nProblem: {problem}") + answer = thinker.solve(problem, max_steps=args.max_steps) + print(f"Answer: {answer}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/tool_calling.py b/docs/source/llm_examples/tool_calling.py index f7d9b9f28..d87895106 100644 --- a/docs/source/llm_examples/tool_calling.py +++ b/docs/source/llm_examples/tool_calling.py @@ -7,13 +7,7 @@ - The model chains multiple tool calls to answer a multi-step query """ -import argparse -import os - from effectful.handlers.llm import Template, Tool -from effectful.handlers.llm.completions import LiteLLMProvider -from effectful.ops.semantics import handler -from effectful.ops.types import NotHandled # --------------------------------------------------------------------------- # Tools @@ -41,25 +35,15 @@ def weather(city: str) -> str: @Template.define def vacation() -> str: """Use the provided tools to suggest a city that has good weather. Use only the `cities` and `weather` tools provided.""" - raise NotHandled # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- +def main() -> None: + print(vacation()) + + if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Tool calling with auto-captured lexical scope" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("EFFECTFUL_LLM_MODEL", ""), - help="LLM model to use", - ) - args = parser.parse_args() - - provider = LiteLLMProvider(model=args.model) - with handler(provider): - print(vacation()) + main() From b0fd2cdca90d9b866fbb0250bdb49822c8b57d36 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 15:49:31 -0400 Subject: [PATCH 034/155] cleanup examples --- docs/source/llm_examples/chat_memory.py | 109 ------------ docs/source/llm_examples/chat_search.py | 97 ---------- docs/source/llm_examples/conversation.py | 70 ++++++++ docs/source/llm_examples/decode_callable.py | 64 ------- docs/source/llm_examples/error_recovery.py | 86 +++++++++ docs/source/llm_examples/flight_booking.py | 41 ++++- docs/source/llm_examples/guardrails.py | 19 +- .../llm_examples/hanoi_solver_recursive.py | 12 +- .../llm_examples/higher_order_function.py | 72 -------- docs/source/llm_examples/hitl.py | 16 +- docs/source/llm_examples/image_input.py | 16 +- docs/source/llm_examples/image_tool.py | 14 +- docs/source/llm_examples/lexical_scope.py | 111 ++++++++++++ docs/source/llm_examples/map_reduce.py | 15 +- docs/source/llm_examples/rag.py | 68 ++++--- docs/source/llm_examples/research_agent.py | 105 +++++++---- docs/source/llm_examples/retry.py | 106 ----------- docs/source/llm_examples/structured_output.py | 72 -------- docs/source/llm_examples/supervisor.py | 166 ------------------ .../llm_examples/{multi_agent.py => taboo.py} | 35 +++- docs/source/llm_examples/tao_agent.py | 62 ++++--- .../llm_examples/template_composition.py | 61 ------- docs/source/llm_examples/text2sql.py | 24 ++- .../{musr.py => theory_of_mind.py} | 0 docs/source/llm_examples/thinking.py | 96 ---------- docs/source/llm_examples/tool_calling.py | 49 ------ docs/source/llm_examples/typed_decoding.py | 111 ++++++++++++ 27 files changed, 683 insertions(+), 1014 deletions(-) delete mode 100644 docs/source/llm_examples/chat_memory.py delete mode 100644 docs/source/llm_examples/chat_search.py create mode 100644 docs/source/llm_examples/conversation.py delete mode 100644 docs/source/llm_examples/decode_callable.py create mode 100644 docs/source/llm_examples/error_recovery.py delete mode 100644 docs/source/llm_examples/higher_order_function.py create mode 100644 docs/source/llm_examples/lexical_scope.py delete mode 100644 docs/source/llm_examples/retry.py delete mode 100644 docs/source/llm_examples/structured_output.py delete mode 100644 docs/source/llm_examples/supervisor.py rename docs/source/llm_examples/{multi_agent.py => taboo.py} (82%) delete mode 100644 docs/source/llm_examples/template_composition.py rename docs/source/llm_examples/{musr.py => theory_of_mind.py} (100%) delete mode 100644 docs/source/llm_examples/thinking.py delete mode 100644 docs/source/llm_examples/tool_calling.py create mode 100644 docs/source/llm_examples/typed_decoding.py diff --git a/docs/source/llm_examples/chat_memory.py b/docs/source/llm_examples/chat_memory.py deleted file mode 100644 index 629371518..000000000 --- a/docs/source/llm_examples/chat_memory.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Chat agent with embedding-based memory. - -Demonstrates: -- A stateful chat agent that maintains conversation history -- Embedding-based retrieval of relevant past context -- Simple in-memory vector store with L2 distance -""" - -import dataclasses - -import litellm -import numpy as np - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Embedding helpers -# --------------------------------------------------------------------------- - - -def get_embedding(text: str) -> np.ndarray: - """Get an embedding vector for the given text using litellm.""" - response = litellm.embedding(model="text-embedding-ada-002", input=text) - return np.array(response.data[0]["embedding"], dtype=np.float32) - - -def find_closest( - index: list[tuple[str, np.ndarray]], phrase: str -) -> tuple[str, float] | None: - """Find the closest entry in the index to the given phrase.""" - if not index: - return None - phrase_embedding = get_embedding(phrase) - - def dist(a: np.ndarray, b: np.ndarray) -> float: - return float(((a - b) ** 2).sum()) - - return min( - ((msg, dist(embedding, phrase_embedding)) for msg, embedding in index), - key=lambda elt: elt[1], - ) - - -# --------------------------------------------------------------------------- -# Chat template -# --------------------------------------------------------------------------- - - -@Template.define -def respond_to_user( - user_message: str, relevant_context: str, prev_messages: str -) -> str: - """Given the user wrote: {user_message} - Continue the conversation. - The last few messages were: {prev_messages} - Older relevant context: {relevant_context}""" - - -# --------------------------------------------------------------------------- -# Chat agent -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass -class ChatAgent: - """A chat agent that compresses old messages into an embedding index.""" - - history: list[dict[str, str]] = dataclasses.field(default_factory=list) - index: list[tuple[str, np.ndarray]] = dataclasses.field(default_factory=list) - - def _compress(self): - """Move the oldest pair of messages into the embedding index.""" - oldest_pair, self.history = self.history[:2], self.history[2:] - text = "\n".join(m["content"] for m in oldest_pair) - self.index.append((text, get_embedding(text))) - - def _find_relevant(self, query: str) -> str: - result = find_closest(self.index, query) - return result[0] if result else "No relevant context." - - def chat(self, user_input: str): - relevant = self._find_relevant(user_input) - prev_messages = "\n".join( - f"{m['author']}: {m['content']}" for m in self.history - ) - response = respond_to_user(user_input, relevant, prev_messages) - self.history.append({"author": "user", "content": user_input}) - self.history.append({"author": "agent", "content": response}) - if len(self.history) > 6: - self._compress() - print(f"user: {user_input}") - print(f"agent: {response}") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - agent = ChatAgent() - - agent.chat("Hello! How are you doing?") - agent.chat("Lovely! I'm having a great day.") - agent.chat("What is the capital of France?") - agent.chat("I didn't know that! That's amazing!") - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/chat_search.py b/docs/source/llm_examples/chat_search.py deleted file mode 100644 index 46859c8ab..000000000 --- a/docs/source/llm_examples/chat_search.py +++ /dev/null @@ -1,97 +0,0 @@ -import argparse -import dataclasses -import urllib.parse - -import requests - -from effectful.handlers.llm import Agent, Template, Tool - - -@Tool.define -def search_web(query: str) -> str: - """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" - search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( - { - "action": "query", - "list": "search", - "srsearch": query, - "srlimit": 1, - "format": "json", - } - ) - search_data = requests.get( - search_url, headers={"User-Agent": "effectful-example/1.0"} - ).json() - results = search_data.get("query", {}).get("search", []) - if not results: - raise ValueError(f"No results found for: {query}") - title = results[0]["title"] - - summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( - { - "action": "query", - "titles": title, - "prop": "extracts", - "exintro": True, - "explaintext": True, - "format": "json", - } - ) - summary_data = requests.get( - summary_url, headers={"User-Agent": "effectful-example/1.0"} - ).json() - page = next(iter(summary_data["query"]["pages"].values())) - extract = page.get("extract", "No summary available.") - url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" - - return f"# {title}\n\n{extract}\n\nSource: {url}" - - -@dataclasses.dataclass -class ChatBot(Agent): - """Simple chat agent for testing history accumulation.""" - - bot_name: str = dataclasses.field(default="ChatBot") - - @Template.define - def send(self, user_input: str) -> str: - """ - You are a friendly and helpful AI assistant named {self.bot_name}. - If user input contains a question that you're not sure how to answer, - consider using the web search tool to find the answer and include it in your response. - - The user writes: - {user_input} - """ - - -def main() -> None: - parser = argparse.ArgumentParser( - description="LLM-guided research agent with web search" - ) - parser.add_argument( - "--name", - type=str, - default="Chatty McChatface", - help="The name of the chatbot", - ) - parser.add_argument( - "--interactive", - action="store_true", - help="Run in interactive mode, allowing multiple back-and-forth messages", - ) - args = parser.parse_args() - - chatbot = ChatBot(bot_name=args.name) - - if args.interactive: - while True: - print(chatbot.send(input("You: "))) - else: - print(chatbot.send("Hi! Can you tell me about the Statue of Liberty?")) - print(chatbot.send("Who designed it?")) - print(chatbot.send("What about the speed of light? How fast is it?")) - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/conversation.py b/docs/source/llm_examples/conversation.py new file mode 100644 index 000000000..28044728e --- /dev/null +++ b/docs/source/llm_examples/conversation.py @@ -0,0 +1,70 @@ +"""Conversational chat agent with persistent history. + +Demonstrates: +- An Agent subclass with automatic conversation history (Agent.__history__) +- Instance attributes available in prompts via {self.bot_name} +- Follow-up questions resolved from earlier turns via accumulated context +- An optional interactive REPL mode +""" + +import argparse +import dataclasses + +from effectful.handlers.llm import Agent, Template + + +@dataclasses.dataclass +class ChatBot(Agent): + """Conversational agent that remembers the conversation so far.""" + + bot_name: str = dataclasses.field(default="ChatBot") + + @Template.define + def send(self, user_input: str) -> str: + """ + You are a friendly and helpful AI assistant named {self.bot_name}. + + The user writes: + {user_input} + """ + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--name", + type=str, + default="Chatty McChatface", + help="The name of the chatbot", + ) + parser.add_argument( + "--interactive", + action="store_true", + help="Run in interactive mode, allowing multiple back-and-forth messages", + ) + parser.add_argument( + "--messages", + type=str, + nargs="+", + metavar="MESSAGE", + default=[ + "Hi! Can you tell me about the Statue of Liberty?", + "Who designed it?", + "What about the speed of light? How fast is it?", + ], + help="The sequence of user messages to send in non-interactive mode", + ) + args = parser.parse_args() + + chatbot = ChatBot(bot_name=args.name) + + if args.interactive: + while True: + print(chatbot.send(input("You: "))) + else: + for message in args.messages: + print(chatbot.send(message)) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/decode_callable.py b/docs/source/llm_examples/decode_callable.py deleted file mode 100644 index 1181f01c0..000000000 --- a/docs/source/llm_examples/decode_callable.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Decoding LLM responses into Python objects, including callables. - -Demonstrates: -- Primitive type decoding (``int``) from a template that returns a number -- Synthesizing a Python ``Callable`` from a template, executed via - ``UnsafeEvalProvider`` from ``effectful.handlers.llm.evaluation`` -- ``inspect.getsource`` on the synthesized function -""" - -import argparse -import inspect -from collections.abc import Callable - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Templates -# --------------------------------------------------------------------------- - - -@Template.define -def primes(first_digit: int) -> int: - """Give a prime number with {first_digit} as the first digit. Do not use any tools.""" - - -@Template.define -def count_char(char: str) -> Callable[[str], int]: - """Write a function which takes a string and counts the occurrances of '{char}'. Do not use any tools.""" - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--first-digit", - type=int, - default=6, - help="First digit of the prime to request", - ) - parser.add_argument( - "--char", - type=str, - default="a", - help="Character whose occurrences the synthesized function will count", - ) - args = parser.parse_args() - - prime = primes(args.first_digit) - assert type(prime) is int - print(f"Prime starting with {args.first_digit}: {prime}") - - counter = count_char(args.char) - assert callable(counter) - print("\nGenerated function:") - print(inspect.getsource(counter)) - print(f'counter("banana") == {counter("banana")}') - print(f'counter("cherry") == {counter("cherry")}') - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/error_recovery.py b/docs/source/llm_examples/error_recovery.py new file mode 100644 index 000000000..c8b90e647 --- /dev/null +++ b/docs/source/llm_examples/error_recovery.py @@ -0,0 +1,86 @@ +"""Recovering from failed LLM output: flaky tools and invalid structured output. + +A single task -- rate a movie after looking it up -- exercises both retry paths: + +Demonstrates: +- RetryLLMHandler surfacing tool exceptions back to the LLM as tool messages, so a + flaky tool (lookup_movie) can succeed after multiple attempts +- RetryLLMHandler feeding pydantic validation errors back to the LLM so it can + correct structured output (a Rating) that fails validation +""" + +import argparse +import dataclasses +import typing + +from effectful.handlers.llm import Template, Tool + +# --------------------------------------------------------------------------- +# Flaky tool (auto-captured into rate_movie's lexical scope) +# --------------------------------------------------------------------------- + +call_count = 0 +REQUIRED_RETRIES = 3 + + +@Tool.define +def lookup_movie(title: str) -> str: + """Look up facts about a movie from an (unreliable) database.""" + global call_count + call_count += 1 + if call_count < REQUIRED_RETRIES: + raise ConnectionError( + f"Movie database unavailable! Attempt {call_count}/{REQUIRED_RETRIES}. Please retry." + ) + return f"{title}: an acclaimed action film, widely regarded as a genre classic." + + +# --------------------------------------------------------------------------- +# Validated structured output +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Rating: + """ + A movie rating, with a score (an integer from 1 to 5) and an explanation. + The explanation MUST mention the score, otherwise it will be rejected as invalid. + """ + score: typing.Literal[1, 2, 3, 4, 5] + explanation: str + + def __post_init__(self): + if self.score < 1 or self.score > 5: + raise ValueError(f"score must be 1-5, got {self.score}") + if str(self.score) not in self.explanation: + raise ValueError(f"explanation must mention the score {self.score}, got '{self.explanation}'") + + +# --------------------------------------------------------------------------- +# Template: uses the flaky tool, returns validated structured output +# --------------------------------------------------------------------------- + + +@Template.define +def rate_movie(movie_name: str) -> Rating: + """Look up the movie {movie_name}, then give it a rating.""" + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--movie", type=str, default="Die Hard", help="Movie to rate") + args = parser.parse_args() + + rating = rate_movie(args.movie) + print(f"Rated {args.movie!r} after {call_count} tool attempts:") + print(f"Score: {rating.score}/5") + print(f"Explanation: {rating.explanation}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/flight_booking.py b/docs/source/llm_examples/flight_booking.py index a8314c698..c0530e094 100644 --- a/docs/source/llm_examples/flight_booking.py +++ b/docs/source/llm_examples/flight_booking.py @@ -49,6 +49,13 @@ class FlightDetails: @dataclasses.dataclass(frozen=True) class SeatPreference: + """ + User's seat preference extracted from natural language. + + Seats A and F are window seats. Seats C and D are aisle seats. + Row 1 is the front row with extra legroom. + Rows 14 and 20 also have extra legroom. + """ row: int # 1-30 seat: Literal["A", "B", "C", "D", "E", "F"] @@ -128,10 +135,6 @@ def select_seat(self, user_input: str) -> SeatPreference: """Extract the user's seat preference from their message. {user_input} - - Seats A and F are window seats. Seats C and D are aisle seats. - Row 1 is the front row with extra legroom. - Rows 14 and 20 also have extra legroom. """ @@ -215,6 +218,30 @@ def book_flight( def main() -> None: parser = argparse.ArgumentParser(description=__doc__) + airports = list(Airport) + parser.add_argument( + "--origin", + type=Airport, + choices=airports, + default=Airport.SFO, + metavar="CODE", + help="Origin airport code", + ) + parser.add_argument( + "--destination", + type=Airport, + choices=airports, + default=Airport.ANC, + metavar="CODE", + help="Destination airport code", + ) + parser.add_argument( + "--date", + type=datetime.date.fromisoformat, + default=datetime.date(2025, 1, 10), + metavar="YYYY-MM-DD", + help="Travel date (YYYY-MM-DD)", + ) parser.add_argument( "--interactive", action="store_true", @@ -223,9 +250,9 @@ def main() -> None: args = parser.parse_args() book_flight( - origin=Airport.SFO, - destination=Airport.ANC, - date=datetime.date(2025, 1, 10), + origin=args.origin, + destination=args.destination, + date=args.date, interactive=args.interactive, ) diff --git a/docs/source/llm_examples/guardrails.py b/docs/source/llm_examples/guardrails.py index 82e8b6d2f..90bf0c3e0 100644 --- a/docs/source/llm_examples/guardrails.py +++ b/docs/source/llm_examples/guardrails.py @@ -5,6 +5,8 @@ - Simple control-flow gating based on LLM classification """ +import argparse + from effectful.handlers.llm import Template # --------------------------------------------------------------------------- @@ -44,8 +46,21 @@ def is_safe_query(user_query: str) -> bool: # --------------------------------------------------------------------------- def main() -> None: - print(answer_travel_query("What are great places to check out in NYC?")) - print(answer_travel_query("Should I buy apple stocks?")) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--queries", + nargs="+", + default=[ + "What are great places to check out in NYC?", + "Should I buy apple stocks?", + ], + metavar="QUERY", + help="User queries to run through the travel-advice guardrail", + ) + args = parser.parse_args() + + for query in args.queries: + print(answer_travel_query(query)) if __name__ == "__main__": diff --git a/docs/source/llm_examples/hanoi_solver_recursive.py b/docs/source/llm_examples/hanoi_solver_recursive.py index 80c8eb2fc..4be6ebe9e 100644 --- a/docs/source/llm_examples/hanoi_solver_recursive.py +++ b/docs/source/llm_examples/hanoi_solver_recursive.py @@ -25,18 +25,16 @@ """ import argparse -import typing -from dataclasses import dataclass, field +import dataclasses from effectful.handlers.llm import Template -from effectful.handlers.llm.template import IsRecursive # --------------------------------------------------------------------------- # Step model # --------------------------------------------------------------------------- -@dataclass +@dataclasses.dataclass class Step: """A single move: take the top disk from tower ``start`` and place it on tower ``end``. Tower indices are zero-based.""" @@ -50,7 +48,7 @@ class Step: # --------------------------------------------------------------------------- -@dataclass +@dataclasses.dataclass class GameState: """State of a Towers of Hanoi game. @@ -60,7 +58,7 @@ class GameState: """ size: int - towers: tuple[tuple[int, ...], ...] = field(default=()) + towers: tuple[tuple[int, ...], ...] = dataclasses.field(default=()) def __post_init__(self): if self.size > 0 and not self.towers: @@ -105,7 +103,7 @@ def __str__(self) -> str: @Template.define def solve( n_disks: int, source: int, target: int, auxiliary: int -) -> typing.Annotated[list[Step], IsRecursive]: +) -> list[Step]: """Solve Tower of Hanoi: move {n_disks} disks from tower {source} to tower {target}, using tower {auxiliary} as temporary storage. diff --git a/docs/source/llm_examples/higher_order_function.py b/docs/source/llm_examples/higher_order_function.py deleted file mode 100644 index 5ffc26669..000000000 --- a/docs/source/llm_examples/higher_order_function.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Generating higher-order functions that call other templates. - -Demonstrates: -- A template returning a ``Callable``, evaluated via ``UnsafeEvalProvider`` -- The synthesized function calling sub-templates (``write_chapter``, - ``judge_chapter``) at runtime -- ``RetryLLMHandler`` to recover from transient validation/runtime errors -- ``inspect.getsource`` on the generated function -""" - -import argparse -import inspect -from collections.abc import Callable -from typing import Literal - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Sub-templates the generated function may call -# --------------------------------------------------------------------------- - - -@Template.define -def write_chapter(chapter_number: int, chapter_name: str) -> str: - """Write a short story about {chapter_number}. Do not use any tools.""" - - -@Template.define -def judge_chapter(story_so_far: str, chapter_number: int) -> bool: - """Decide if the new chapter is coherent with the story so far. Do not use any tools.""" - - -# --------------------------------------------------------------------------- -# Orchestrator template returning a callable -# --------------------------------------------------------------------------- - - -@Template.define -def write_multi_chapter_story(style: Literal["moral", "funny"]) -> Callable[[str], str]: - """ - Generate a function that writes a story in style: {style} about the given topic. - """ - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--topic", type=str, default="a curious cat", help="Story topic" - ) - parser.add_argument( - "--style", - type=str, - choices=["moral", "funny"], - default="moral", - help="Story style", - ) - args = parser.parse_args() - - print(f"\n=== Generating story function (style={args.style}) ===") - story_fn = write_multi_chapter_story(args.style) - print(inspect.getsource(story_fn)) - print(f"\n=== Running generated function on {args.topic!r} ===") - print(story_fn(args.topic)) - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/hitl.py b/docs/source/llm_examples/hitl.py index 62ca5b512..5dd693ddb 100644 --- a/docs/source/llm_examples/hitl.py +++ b/docs/source/llm_examples/hitl.py @@ -126,13 +126,19 @@ def main() -> None: default=5, help="Maximum number of action steps", ) + parser.add_argument( + "--task", + type=str, + default=( + "Organize a team lunch for next Friday. " + "Send an email to the team, create a shared document for " + "restaurant suggestions, and schedule a meeting to finalize plans." + ), + help="The goal for the planner to accomplish", + ) args = parser.parse_args() - task = ( - "Organize a team lunch for next Friday. " - "Send an email to the team, create a shared document for " - "restaurant suggestions, and schedule a meeting to finalize plans." - ) + task = args.task print(f"Task: {task}\n") log = run_with_approval( diff --git a/docs/source/llm_examples/image_input.py b/docs/source/llm_examples/image_input.py index 8444ef561..b2527ef3c 100644 --- a/docs/source/llm_examples/image_input.py +++ b/docs/source/llm_examples/image_input.py @@ -5,6 +5,7 @@ - Inline base64 image data so the script is self-contained """ +import argparse import base64 import io @@ -42,7 +43,20 @@ def describe_image(image: Image.Image) -> str: def main() -> None: - image = Image.open(io.BytesIO(base64.b64decode(IMAGE_BASE64))) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--image", + type=str, + default=None, + metavar="PATH", + help="Path to an image file to describe (defaults to a built-in 32x32 smiley face)", + ) + args = parser.parse_args() + + if args.image is not None: + image = Image.open(args.image) + else: + image = Image.open(io.BytesIO(base64.b64decode(IMAGE_BASE64))) print(describe_image(image)) diff --git a/docs/source/llm_examples/image_tool.py b/docs/source/llm_examples/image_tool.py index 77df52255..5325eeb8b 100644 --- a/docs/source/llm_examples/image_tool.py +++ b/docs/source/llm_examples/image_tool.py @@ -1,3 +1,5 @@ +import argparse + from PIL import Image from effectful.handlers.llm import Agent, Template, Tool @@ -63,8 +65,18 @@ def rotate_and_concat(self, i: Image.Image) -> Image.Image: def main() -> None: + parser = argparse.ArgumentParser(description=__doc__ or ImageTools.__doc__) + parser.add_argument( + "--image", + type=str, + default="../_static/img/chirho_logo_wide.png", + metavar="PATH", + help="Path to the input image to rotate-and-concatenate.", + ) + args = parser.parse_args() + image_agent = ImageTools() - img = Image.open("../_static/img/chirho_logo_wide.png") + img = Image.open(args.image) image_agent.rotate_and_concat(img).show() diff --git a/docs/source/llm_examples/lexical_scope.py b/docs/source/llm_examples/lexical_scope.py new file mode 100644 index 000000000..2a8630cd1 --- /dev/null +++ b/docs/source/llm_examples/lexical_scope.py @@ -0,0 +1,111 @@ +"""Composition via lexical scope: auto-captured sub-templates, invoked two ways. + +Demonstrates: +- Module-level @Template.define sub-templates auto-captured into other templates' + lexical scope, with no explicit registration +- An Agent grouping @Tool.define tools with a @Template.define orchestrator that + calls those tools and the sub-templates directly (model-driven composition) +- A template returning a Callable: the model synthesizes a function that calls the + same sub-templates when run (code-driven composition), via the eval provider +- inspect.getsource on the synthesized function +""" + +import argparse +import inspect +from collections.abc import Callable +from typing import Literal + +from effectful.handlers.llm import Agent, Template, Tool + +# --------------------------------------------------------------------------- +# Sub-templates (module-level; auto-captured into the scopes below) +# --------------------------------------------------------------------------- + + +@Template.define +def story_with_moral(topic: str) -> str: + """Write a short story about {topic} and end with a moral lesson.""" + + +@Template.define +def story_funny(topic: str) -> str: + """Write a funny, humorous story about {topic}.""" + + +# --------------------------------------------------------------------------- +# (1) Model-driven composition: an orchestrator template calls tools and +# sub-templates directly during its own turn. +# --------------------------------------------------------------------------- + + +class TripPlanner(Agent): + """Plans a trip to a city with good weather and tells a story about visiting it.""" + + @Tool.define + def cities(self) -> list[str]: + """Return a list of candidate destination cities.""" + return ["Chicago", "New York", "Barcelona"] + + @Tool.define + def weather(self, city: str) -> str: + """Given a city name, return a short description of its weather.""" + status = {"Chicago": "cold", "New York": "wet", "Barcelona": "sunny"} + return status.get(city, "unknown") + + @Template.define + def plan_trip_story(self, style: str) -> str: + """Use the relevant tools to identify a city that has good (sunny) + weather. Then write a short story about visiting that city in the requested + style: {style}""" + + +# --------------------------------------------------------------------------- +# (2) Code-driven composition: a template synthesizes a function that calls the +# same sub-templates when executed. +# --------------------------------------------------------------------------- + + +@Template.define +def write_story_fn(style: Literal["moral", "funny"]) -> Callable[[str], str]: + """Generate a Python function that takes a topic string and returns a story + about it in the {style} style. The function should delegate the writing to the + `story_funny` sub-template for humor, or `story_with_moral` for a lesson.""" + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--style", + type=str, + choices=["moral", "funny"], + default="funny", + help="Style of the story to produce", + ) + parser.add_argument( + "--topic", + type=str, + default="a curious cat", + help="Topic for the synthesized story function to run on", + ) + args = parser.parse_args() + + # (1) Model-driven: the orchestrator template calls tools and sub-templates. + print("=== Orchestrator template (model-driven composition) ===") + planner = TripPlanner() + print(planner.plan_trip_story(args.style)) + + # (2) Code-driven: the model synthesizes a function that calls the sub-templates. + print(f"\n=== Synthesized higher-order function (style={args.style}) ===") + story_fn = write_story_fn(args.style) + print(inspect.getsource(story_fn)) + print(f"\n=== Running it on {args.topic!r} ===") + print(story_fn(args.topic)) + + +if __name__ == "__main__": + main() diff --git a/docs/source/llm_examples/map_reduce.py b/docs/source/llm_examples/map_reduce.py index 5f5bc5ad1..7dfeb1826 100644 --- a/docs/source/llm_examples/map_reduce.py +++ b/docs/source/llm_examples/map_reduce.py @@ -7,6 +7,7 @@ - Structured output with dataclasses """ +import argparse import asyncio import collections.abc import dataclasses @@ -117,8 +118,18 @@ async def map_reduce_evaluate( # --------------------------------------------------------------------------- def main() -> None: - print(f"Evaluating {len(RESUMES)} resumes for: {JOB_DESCRIPTION}\n") - summary = asyncio.run(map_reduce_evaluate(RESUMES, JOB_DESCRIPTION)) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--job-description", + type=str, + metavar="TEXT", + default=JOB_DESCRIPTION, + help="Job description to evaluate resumes against.", + ) + args = parser.parse_args() + + print(f"Evaluating {len(RESUMES)} resumes for: {args.job_description}\n") + summary = asyncio.run(map_reduce_evaluate(RESUMES, args.job_description)) print(f"\n{summary}") diff --git a/docs/source/llm_examples/rag.py b/docs/source/llm_examples/rag.py index c4751fd41..498cb40d8 100644 --- a/docs/source/llm_examples/rag.py +++ b/docs/source/llm_examples/rag.py @@ -14,7 +14,7 @@ import litellm import numpy as np -from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm import Agent, Template, Tool # --------------------------------------------------------------------------- # Embedding helpers @@ -45,8 +45,7 @@ def add(self, text: str) -> None: self.chunks.append(text) self.embeddings.append(get_embedding(text, model=self.model)) - @Tool.define - def retrieve(self, query: str, top_k: int = 3) -> list[str]: + def search(self, query: str, top_k: int = 3) -> list[str]: """Return the top-k most similar chunks to the query.""" if not self.embeddings: return [] @@ -123,26 +122,39 @@ def build_index(documents: list[str], embedding_model: str) -> VectorIndex: # --------------------------------------------------------------------------- -# RAG query (online phase) +# RAG agent (online phase): the `retrieve` tool and `answer_question` template +# share one instance, so the tool is auto-captured from lexical scope. # --------------------------------------------------------------------------- -@Template.define -def answer_question(question: str) -> str: - """You are a helpful assistant. Answer the user's question using ONLY - information retrieved from the knowledge base via the retrieve tool. +@dataclasses.dataclass +class RAGAgent(Agent): + """Answers a question grounded in the vector index via a retrieval tool.""" + + index: VectorIndex + + @Tool.define + def retrieve(self, query: str, top_k: int = 3) -> list[str]: + """Return the top-k most similar chunks to the query.""" + return self.index.search(query, top_k) - If the retrieved information doesn't contain the answer, say so. - Always cite which document your information comes from. + @Template.define + def answer_question(self, question: str) -> str: + """You are a helpful assistant. Answer the user's question using ONLY + information retrieved from the knowledge base via the retrieve tool. - Question: {question} - """ + If the retrieved information doesn't contain the answer, say so. + Always cite which document your information comes from. + + Question: {question} + """ # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -151,27 +163,27 @@ def main() -> None: default="lm_studio/text-embedding-embeddinggemma-300m-qat", help="Embedding model to use", ) + parser.add_argument( + "--questions", + type=str, + nargs="+", + metavar="QUESTION", + default=[ + "How tall is the Eiffel Tower?", + "When was the Great Wall of China built?", + "How many spectators could the Colosseum hold?", + ], + help="Questions to answer against the indexed documents", + ) args = parser.parse_args() - # Offline: build the index + # Offline: build the index once. index = build_index(DOCUMENTS, embedding_model=args.embedding_model) - # Create the retrieval tool bound to our index. `answer_question` is a - # module-level template, so the tool must be bound in module globals to be - # in its lexical scope. - global retrieve - retrieve = index.retrieve - - # Online: answer questions - questions = [ - "How tall is the Eiffel Tower?", - "When was the Great Wall of China built?", - "How many spectators could the Colosseum hold?", - ] - - for question in questions: + # Online: answer each question with a fresh (stateless) agent over that index. + for question in args.questions: print(f"\nQ: {question}") - answer = answer_question(question) + answer = RAGAgent(index=index).answer_question(question) print(f"A: {answer}") diff --git a/docs/source/llm_examples/research_agent.py b/docs/source/llm_examples/research_agent.py index 22c0cb863..76d69a736 100644 --- a/docs/source/llm_examples/research_agent.py +++ b/docs/source/llm_examples/research_agent.py @@ -1,21 +1,23 @@ -"""Research agent with web search. +"""Research agent with web search and LLM quality control. Demonstrates: -- ``@defop`` + ``ObjectInterpretation`` to define a pluggable web search effect -- ``@Template.define`` for LLM-implemented answer/refine/judge templates -- Handler composition: stacking a search provider alongside an LLM provider -- Iterative refinement loop: answer → judge → refine → judge → ... +- @Tool.define web-search tool, auto-captured into templates from lexical scope +- An Agent subclass with persistent conversation history +- One Template judging another's output, returning a structured QualityJudgment + (a bool plus written feedback) +- A feedback-driven refinement loop: answer -> judge -> refine -> judge -> ... """ import argparse +import dataclasses import urllib.parse import requests -from effectful.handlers.llm import Template, Tool +from effectful.handlers.llm import Agent, Template, Tool # --------------------------------------------------------------------------- -# Search effect + handler +# Search tool # --------------------------------------------------------------------------- @@ -60,42 +62,74 @@ def search_web(query: str) -> str: # --------------------------------------------------------------------------- -# Templates (auto-capture `search_web` from lexical scope) +# Structured output for quality judgment # --------------------------------------------------------------------------- -@Template.define -def answer_question(question: str) -> str: - """Acting as a research assistant that can search the web, - construct an answer to the user's question: {question}.""" +@dataclasses.dataclass(frozen=True) +class QualityJudgment: + is_acceptable: bool + feedback: str -@Template.define -def refine_answer(question: str, answer: str) -> str: - """Acting as a research assistant that can search the web, - given the user's original question ({question}), - refine this previous answer: {answer}.""" +# --------------------------------------------------------------------------- +# Research agent (persistent history; search_web auto-captured from scope) +# --------------------------------------------------------------------------- + + +class Researcher(Agent): + """Agent that answers research questions using web search, refining on feedback.""" + + @Template.define + def answer(self, question: str) -> str: + """You are a research assistant. Use the search tool to find accurate, + specific information, then answer the question: {question}""" + + @Template.define + def refine(self, question: str, feedback: str) -> str: + """A reviewer rejected your previous answer to the question ({question}) + with this feedback: {feedback}. Use the search tool as needed and provide + an improved answer that addresses the feedback.""" + + +# --------------------------------------------------------------------------- +# Supervisor (quality judge) +# --------------------------------------------------------------------------- @Template.define -def is_question_answered(question: str, answer: str) -> bool: - """Acting as a research assistant, decide if the user's question - ({question}) is appropriately answered by: {answer}. - Respond only true or false.""" +def judge_quality(question: str, answer: str) -> QualityJudgment: + """You are a strict quality reviewer. Evaluate whether this answer adequately + addresses the question with accurate, specific information. + + Question: {question} + Answer: {answer} + + An answer is acceptable if it contains specific facts (names, dates, numbers) + relevant to the question. Vague or generic answers should be rejected; when + rejecting, explain in the feedback what is missing. + """ # --------------------------------------------------------------------------- -# Agent loop +# Supervised agent loop # --------------------------------------------------------------------------- -def research_agent(question: str, max_attempts: int = 3) -> str: - """Answer a question, iteratively refining until satisfactory.""" - answer = answer_question(question) - for _ in range(max_attempts): - if is_question_answered(question, answer): - break - answer = refine_answer(question, answer) +def research_agent(question: str, max_retries: int = 3) -> str: + """Answer a question, refining on supervisor feedback until it is acceptable.""" + researcher = Researcher() + answer = researcher.answer(question) + + for attempt in range(1, max_retries + 1): + judgment = judge_quality(question, answer) + if judgment.is_acceptable: + print(f"[supervisor] Accepted on attempt {attempt}") + return answer + print(f"[supervisor] Rejected attempt {attempt}: {judgment.feedback}") + answer = researcher.refine(question, judgment.feedback) + + print("[supervisor] Returning best effort after max retries") return answer @@ -103,18 +137,25 @@ def research_agent(question: str, max_attempts: int = 3) -> str: # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--question", type=str, - default="What is the meaning of life?", + default="What year was the Eiffel Tower completed and how tall is it?", help="The question to research", ) + parser.add_argument( + "--max-retries", + type=int, + default=3, + help="Maximum number of supervisor rejections before returning best effort", + ) args = parser.parse_args() - result = research_agent(args.question) - print(result) + result = research_agent(args.question, max_retries=args.max_retries) + print(f"\nFinal answer: {result}") if __name__ == "__main__": diff --git a/docs/source/llm_examples/retry.py b/docs/source/llm_examples/retry.py deleted file mode 100644 index 515dfbc4b..000000000 --- a/docs/source/llm_examples/retry.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Retrying failed LLM output: validation errors and tool failures. - -Demonstrates: -- ``RetryLLMHandler`` feeding ``PydanticCustomError`` messages back to the LLM - so it can correct structured output that fails validation -- ``RetryLLMHandler`` surfacing tool exceptions back to the LLM as tool messages, - so a flaky tool (``unstable_service``) can succeed after multiple attempts -- ``functools.cache`` to make a template call deterministic in-process -""" - -import argparse -import functools - -import pydantic -from pydantic import field_validator -from pydantic_core import PydanticCustomError - -from effectful.handlers.llm import Template, Tool - -# --------------------------------------------------------------------------- -# Validated structured output -# --------------------------------------------------------------------------- - - -@pydantic.dataclasses.dataclass -class Rating: - score: int - explanation: str - - @field_validator("score") - @classmethod - def check_score(cls, v): - if v < 1 or v > 5: - raise PydanticCustomError( - "invalid_score", - "score must be 1–5, got {v}", - {"v": v}, - ) - return v - - @field_validator("explanation") - @classmethod - def check_explanation_contains_score(cls, v, info): - score = info.data.get("score", None) - if score is not None and str(score) not in v: - raise PydanticCustomError( - "invalid_explanation", - "explanation must mention the score {score}, got '{explanation}'", - {"score": score, "explanation": v}, - ) - return v - - -@functools.cache -@Template.define -def give_rating_for_movie(movie_name: str) -> Rating: - """Give a rating for {movie_name}. The explanation MUST include the numeric score. Do not use any tools.""" - - -# --------------------------------------------------------------------------- -# Flaky tool (unstable_service auto-captured from lexical scope) -# --------------------------------------------------------------------------- - -call_count = 0 -REQUIRED_RETRIES = 3 - - -@Tool.define -def unstable_service() -> str: - """Fetch data from an unstable external service. May require retries.""" - global call_count - call_count += 1 - if call_count < REQUIRED_RETRIES: - raise ConnectionError( - f"Service unavailable! Attempt {call_count}/{REQUIRED_RETRIES}. Please retry." - ) - return "{ 'status': 'ok', 'data': [1, 2, 3] }" - - -@Template.define -def fetch_data() -> str: - """Use the unstable_service tool to fetch data.""" - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--movie", type=str, default="Die Hard", help="Movie to rate") - args = parser.parse_args() - - print("=== Retrying structured-output validation ===") - rating = give_rating_for_movie(args.movie) - print(f"Score: {rating.score}/5") - print(f"Explanation: {rating.explanation}") - - print("\n=== Retrying tool execution failures ===") - result = fetch_data() - print(f"Result: {result} (after {call_count} tool attempts)") - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/structured_output.py b/docs/source/llm_examples/structured_output.py deleted file mode 100644 index 3273acdaa..000000000 --- a/docs/source/llm_examples/structured_output.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Structured output via dataclasses. - -Demonstrates: -- Dataclass return types decoded from constrained LLM generation -- Round-tripping a dataclass: one template produces it, another consumes it -""" - -import argparse -import dataclasses - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Structured output -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass -class KnockKnockJoke: - whos_there: str - punchline: str - - -# --------------------------------------------------------------------------- -# Templates -# --------------------------------------------------------------------------- - - -@Template.define -def write_joke(theme: str) -> KnockKnockJoke: - """Write a knock-knock joke on the theme of {theme}. Do not use any tools.""" - - -@Template.define -def rate_joke(joke: KnockKnockJoke) -> bool: - """Decide if {joke} is funny or not. Do not use any tools.""" - - -# --------------------------------------------------------------------------- -# Helper -# --------------------------------------------------------------------------- - - -def do_comedy(theme: str) -> None: - joke = write_joke(theme) - print("> You are onstage at a comedy club. You tell the following joke:") - print( - f"Knock knock.\nWho's there?\n{joke.whos_there}.\n" - f"{joke.whos_there} who?\n{joke.punchline}" - ) - if rate_joke(joke): - print("> The crowd laughs politely.") - else: - print("> The crowd stares in stony silence.") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--theme", type=str, default="lizards", help="Theme for the joke" - ) - args = parser.parse_args() - - do_comedy(args.theme) - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/supervisor.py b/docs/source/llm_examples/supervisor.py deleted file mode 100644 index b26d9b955..000000000 --- a/docs/source/llm_examples/supervisor.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Supervisor quality-control wrapper. - -Demonstrates: -- Wrapping an agent's output with a quality-control check -- Using one ``Template`` to judge another's output -- Retry loop driven by LLM-based evaluation -""" - -import argparse -import dataclasses -import urllib.parse - -import requests - -from effectful.handlers.llm import Agent, Template, Tool - -# --------------------------------------------------------------------------- -# Search tool -# --------------------------------------------------------------------------- - - -@Tool.define -def search_web(query: str) -> str: - """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" - search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( - { - "action": "query", - "list": "search", - "srsearch": query, - "srlimit": 1, - "format": "json", - } - ) - search_data = requests.get( - search_url, headers={"User-Agent": "effectful-example/1.0"} - ).json() - results = search_data.get("query", {}).get("search", []) - if not results: - return f"No results found for: {query}" - title = results[0]["title"] - - summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( - { - "action": "query", - "titles": title, - "prop": "extracts", - "exintro": True, - "explaintext": True, - "format": "json", - } - ) - summary_data = requests.get( - summary_url, headers={"User-Agent": "effectful-example/1.0"} - ).json() - page = next(iter(summary_data["query"]["pages"].values())) - extract = page.get("extract", "No summary available.") - url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" - - return f"# {title}\n\n{extract}\n\nSource: {url}" - - -# --------------------------------------------------------------------------- -# Structured output for quality judgment -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass(frozen=True) -class QualityJudgment: - is_acceptable: bool - feedback: str - - -# --------------------------------------------------------------------------- -# Research agent -# --------------------------------------------------------------------------- - - -class Researcher(Agent): - """Agent that answers research questions using web search.""" - - @Template.define - def answer(self, question: str) -> str: - """You are a research assistant. Answer the following question using - the search tool to find accurate information. - - Question: {question} - """ - - -# --------------------------------------------------------------------------- -# Supervisor (quality judge) -# --------------------------------------------------------------------------- - - -@Template.define -def judge_quality(question: str, answer: str) -> QualityJudgment: - """You are a strict quality reviewer. Evaluate whether this answer - adequately addresses the question with accurate, specific information. - - Question: {question} - Answer: {answer} - - An answer is acceptable if it contains specific facts (names, dates, - numbers) relevant to the question. Vague or generic answers should - be rejected. - """ - - -# --------------------------------------------------------------------------- -# Supervised agent loop -# --------------------------------------------------------------------------- - - -def supervised_research(question: str, max_retries: int = 3) -> str: - """Answer a question with quality-control supervision. - - The researcher agent answers, the supervisor judges quality, - and if rejected the researcher tries again with feedback. - """ - researcher = Researcher() - - for attempt in range(max_retries + 1): - answer = researcher.answer(question) - judgment = judge_quality(question, answer) - - if judgment.is_acceptable: - print(f"[supervisor] Accepted on attempt {attempt + 1}") - return answer - - print(f"[supervisor] Rejected attempt {attempt + 1}: {judgment.feedback}") - - print("[supervisor] Returning best effort after max retries") - return answer - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser( - description="Supervised research agent with quality control" - ) - parser.add_argument( - "--question", - type=str, - default="What year was the Eiffel Tower completed and how tall is it?", - help="Research question to answer", - ) - parser.add_argument( - "--max-retries", - type=int, - default=3, - help="Maximum number of supervisor rejections before accepting", - ) - args = parser.parse_args() - - result = supervised_research( - args.question, - max_retries=args.max_retries, - ) - print(f"\nFinal answer: {result}") - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/multi_agent.py b/docs/source/llm_examples/taboo.py similarity index 82% rename from docs/source/llm_examples/multi_agent.py rename to docs/source/llm_examples/taboo.py index caadfb8e8..677ce3fc7 100644 --- a/docs/source/llm_examples/multi_agent.py +++ b/docs/source/llm_examples/taboo.py @@ -18,7 +18,7 @@ # --------------------------------------------------------------------------- -class Confidence(enum.Enum): +class Confidence(enum.StrEnum): LOW = "low" MEDIUM = "medium" HIGH = "high" @@ -39,8 +39,8 @@ class Guess: class Hinter(Agent): """Agent that gives hints about a secret word without saying it.""" - secret_word: str = dataclasses.field(default="") - taboo_words: list[str] = dataclasses.field(default_factory=list) + secret_word: str + taboo_words: list[str] @Tool.define def is_taboo(self, hint: str) -> bool: @@ -130,12 +130,33 @@ def main() -> None: default=5, help="Maximum rounds per game", ) + parser.add_argument( + "--secret-word", + type=str, + default=None, + metavar="WORD", + help="Secret word to guess (used with --taboo-words for a single custom game)", + ) + parser.add_argument( + "--taboo-words", + nargs="+", + type=str, + default=None, + metavar="WORD", + help="Taboo words the hinter may not say (used with --secret-word)", + ) args = parser.parse_args() - games = [ - ("piano", ["music", "keys", "instrument", "play"]), - ("volcano", ["lava", "eruption", "mountain", "hot"]), - ] + if (args.secret_word is None) != (args.taboo_words is None): + parser.error("--secret-word and --taboo-words must be given together") + + if args.secret_word is not None: + games = [(args.secret_word, args.taboo_words)] + else: + games = [ + ("piano", ["music", "keys", "instrument", "play"]), + ("volcano", ["lava", "eruption", "mountain", "hot"]), + ] for secret, taboo in games: print(f"\nGame: '{secret}' (taboo: {taboo})") diff --git a/docs/source/llm_examples/tao_agent.py b/docs/source/llm_examples/tao_agent.py index 93fe6a6a8..6a58276de 100644 --- a/docs/source/llm_examples/tao_agent.py +++ b/docs/source/llm_examples/tao_agent.py @@ -1,10 +1,12 @@ -"""Think-Act-Observe chain-of-thought agent. +"""Think-Act-Observe agent: structured chain-of-thought with optional tool use. Demonstrates: -- ``Agent`` mixin for persistent conversation history -- Structured output with Pydantic models (``AgentThought``) -- A think → act → observe reasoning loop -- Pattern matching for action dispatch +- Agent mixin for persistent conversation history (the LLM sees its own prior + reasoning across steps) +- Structured output with an AgentThought dataclass carrying an is_final flag +- A think -> act -> observe loop that continues until the agent is done +- Pattern-matching action dispatch: reason straight to an answer, or call a + web-search tool when a fact is missing """ import argparse @@ -68,7 +70,6 @@ def search_web(query: str) -> str: class AgentAction(enum.StrEnum): search_the_web = "search_the_web" - calculate = "calculate" answer = "answer" @@ -91,8 +92,10 @@ class TAOAgent(Agent): @Template.define def think(self, query: str) -> AgentThought: """You are an AI assistant solving a problem. Based on the user's query - ({query}) and prior conversation context, think about what action to - take next. + ({query}) and your own prior reasoning in the conversation history, think + about what to do next: either `search_the_web` for a fact you are missing, + or `answer` once you can conclude. Set is_final=true when your action is + the final answer. """ @Template.define @@ -110,8 +113,9 @@ def observe(self, action: str, action_input: str, action_result: str) -> str: def run(self, query: str, max_steps: int = 5) -> str: result = "" - for _ in range(max_steps): + for i in range(max_steps): thought = self.think(query) + print(f" [step {i + 1}] {thought.thinking}") result = self._act(thought.action, thought.action_input) self.observe(str(thought.action), thought.action_input, result) if thought.is_final: @@ -122,11 +126,6 @@ def _act(self, action: AgentAction, action_input: str) -> str: match action: case AgentAction.search_the_web: return search_web(action_input) - case AgentAction.calculate: - try: - return action_input # eval(action_input)) # noqa: S307 - except Exception as e: - return str(e) case AgentAction.answer: return action_input @@ -135,23 +134,42 @@ def _act(self, action: AgentAction, action_input: str) -> str: # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--max-steps", type=int, default=5, - help="Maximum number of steps before giving up", + help="Maximum think-act-observe steps per problem", + ) + parser.add_argument( + "--problem", + dest="problems", + metavar="PROBLEM", + nargs="+", + default=[ + ( + "A farmer has 17 sheep. All but 9 run away. " + "Then he buys 5 more. How many sheep does he have now?" + ), + "What year was the Eiffel Tower completed, and how tall is it?", + ], + help=( + "One or more problems to solve (a pure-reasoning puzzle and a " + "web-lookup question by default)" + ), ) args = parser.parse_args() - agent = TAOAgent() - - answer = agent.run( - "How many tennis balls would fill an Olympic swimming pool?", - max_steps=args.max_steps, - ) - print("Answer:", answer) + # By default, one puzzle the agent can reason through with no tools, and one + # that needs a web lookup -- the same loop handles both via its action + # dispatch. + for problem in args.problems: + agent = TAOAgent() + print(f"\nProblem: {problem}") + answer = agent.run(problem, max_steps=args.max_steps) + print(f"Answer: {answer}") if __name__ == "__main__": diff --git a/docs/source/llm_examples/template_composition.py b/docs/source/llm_examples/template_composition.py deleted file mode 100644 index 3e87b71ed..000000000 --- a/docs/source/llm_examples/template_composition.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Template composition: templates can call other templates. - -Demonstrates: -- Sub-templates auto-captured into an orchestrator template's lexical scope -- Inspecting ``write_story.tools`` to confirm sub-templates are exposed to the LLM -- The orchestrator dispatches to the right sub-template based on a style argument -""" - -import argparse - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Sub-templates -# --------------------------------------------------------------------------- - - -@Template.define -def story_with_moral(topic: str) -> str: - """Write a short story about {topic} and end with a moral lesson. Do not use any tools.""" - - -@Template.define -def story_funny(topic: str) -> str: - """Write a funny, humorous story about {topic}. Do not use any tools.""" - - -# --------------------------------------------------------------------------- -# Orchestrator template -# --------------------------------------------------------------------------- - - -@Template.define -def write_story(topic: str, style: str) -> str: - """Write a story about {topic} in the style: {style}. - Available styles: 'moral' for a story with a lesson, 'funny' for humor. - Use story_funny for humor, story_with_moral for a story with a lesson. - """ - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser( - description="Template composition with auto-captured sub-templates" - ) - parser.add_argument( - "--topic", type=str, default="a curious cat", help="Story topic" - ) - args = parser.parse_args() - - print("\n=== Story with moral ===") - print(write_story(args.topic, "moral")) - print("\n=== Funny story ===") - print(write_story(args.topic, "funny")) - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/text2sql.py b/docs/source/llm_examples/text2sql.py index 05d42f453..2b5e8794b 100644 --- a/docs/source/llm_examples/text2sql.py +++ b/docs/source/llm_examples/text2sql.py @@ -7,6 +7,7 @@ - ``@Tool.define`` to expose the database schema as a tool """ +import argparse import sqlite3 import textwrap @@ -125,15 +126,22 @@ def text_to_sql( # --------------------------------------------------------------------------- def main() -> None: - conn = create_sample_db() - - questions = [ - "What is the average salary by department?", - "Who is the highest paid employee?", - "How many employees were hired after 2021?", - ] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--questions", + nargs="+", + metavar="QUESTION", + default=[ + "What is the average salary by department?", + "Who is the highest paid employee?", + "How many employees were hired after 2021?", + ], + help="Natural-language questions to answer against the sample database", + ) + args = parser.parse_args() - for question in questions: + conn = create_sample_db() + for question in args.questions: print(f"\nQ: {question}") try: rows = text_to_sql(conn, question) diff --git a/docs/source/llm_examples/musr.py b/docs/source/llm_examples/theory_of_mind.py similarity index 100% rename from docs/source/llm_examples/musr.py rename to docs/source/llm_examples/theory_of_mind.py diff --git a/docs/source/llm_examples/thinking.py b/docs/source/llm_examples/thinking.py deleted file mode 100644 index bd26f9252..000000000 --- a/docs/source/llm_examples/thinking.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Chain-of-thought reasoning with structured self-loop. - -Demonstrates: -- Structured output with a ``ThoughtStep`` dataclass -- An ``Agent`` that loops until it decides it has a final answer -- The LLM sees its own prior reasoning via ``Agent.__history__`` -""" - -import argparse -import dataclasses - -from effectful.handlers.llm import Agent, Template - -# --------------------------------------------------------------------------- -# Structured output -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass(frozen=True) -class ThoughtStep: - reasoning: str - conclusion: str - is_final: bool - - -# --------------------------------------------------------------------------- -# Chain-of-thought agent -# --------------------------------------------------------------------------- - - -class Thinker(Agent): - """Agent that reasons step-by-step until it reaches a final answer.""" - - @Template.define - def think(self, problem: str) -> ThoughtStep: - """You are solving a problem step by step. - - Problem: {problem} - - Review the conversation history for any prior reasoning steps. - Continue from where you left off. Break the problem into small, - logical steps. Set is_final=true only when you have a complete, - well-supported answer. - """ - - def solve(self, problem: str, max_steps: int = 10) -> str: - """Solve a problem by iterative chain-of-thought reasoning.""" - for i in range(max_steps): - step = self.think(problem) - print(f" [step {i + 1}] {step.reasoning}") - if step.is_final: - return step.conclusion - - return step.conclusion - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--max-steps", - type=int, - default=10, - help="Maximum reasoning steps before stopping", - ) - parser.add_argument( - "--problem", - type=str, - default=( - "A farmer has 17 sheep. All but 9 run away. " - "Then he buys 5 more. How many sheep does he have now?" - ), - help="The problem to solve", - ) - args = parser.parse_args() - - problems = [ - args.problem, - ( - "If you have a 3-gallon jug and a 5-gallon jug, " - "how do you measure exactly 4 gallons of water?" - ), - ] - - for problem in problems: - thinker = Thinker() - print(f"\nProblem: {problem}") - answer = thinker.solve(problem, max_steps=args.max_steps) - print(f"Answer: {answer}") - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/tool_calling.py b/docs/source/llm_examples/tool_calling.py deleted file mode 100644 index d87895106..000000000 --- a/docs/source/llm_examples/tool_calling.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Tool calling: templates invoke Python callables exposed via ``@Tool.define``. - -Demonstrates: -- ``@Tool.define`` for exposing a Python function to the model -- Lexical-scope auto-capture: tools defined alongside a template are made - available to the LLM without explicit registration -- The model chains multiple tool calls to answer a multi-step query -""" - -from effectful.handlers.llm import Template, Tool - -# --------------------------------------------------------------------------- -# Tools -# --------------------------------------------------------------------------- - - -@Tool.define -def cities() -> list[str]: - """Return a list of cities that can be passed to `weather`.""" - return ["Chicago", "New York", "Barcelona"] - - -@Tool.define -def weather(city: str) -> str: - """Given a city name, return a description of the weather in that city.""" - status = {"Chicago": "cold", "New York": "wet", "Barcelona": "sunny"} - return status.get(city, "unknown") - - -# --------------------------------------------------------------------------- -# Template (cities and weather are auto-captured from lexical scope) -# --------------------------------------------------------------------------- - - -@Template.define -def vacation() -> str: - """Use the provided tools to suggest a city that has good weather. Use only the `cities` and `weather` tools provided.""" - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - print(vacation()) - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/typed_decoding.py b/docs/source/llm_examples/typed_decoding.py new file mode 100644 index 000000000..08cb2d57f --- /dev/null +++ b/docs/source/llm_examples/typed_decoding.py @@ -0,0 +1,111 @@ +"""Type-driven decoding: turning model output into typed Python values. + +Demonstrates: +- Primitive decoding (int, bool) from templates that return a number / a decision +- Dataclass return types decoded from constrained generation +- Round-tripping a dataclass: one template produces it, others consume it as prompt input +- Synthesizing an executable Callable from a template (run via the eval provider) +- inspect.getsource on the synthesized function + +The thread tying these together: you declare a Python return type and the model's +output is decoded into a real value of that type -- an int, a bool, a dataclass, or +an executable function -- as an auto-grader that poses a problem, solves it, and +checks its own work. +""" + +import argparse +import dataclasses +import inspect +from collections.abc import Callable + +from effectful.handlers.llm import Template + +# --------------------------------------------------------------------------- +# Structured output +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Problem: + title: str + description: str + example_input: str + example_output: str + + +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- + + +@Template.define +def pose_problem(topic: str) -> Problem: + """Invent a small self-contained string-processing coding problem about {topic}. + + The problem must be solvable by a single Python function taking one string and + returning one string. Fill in a short title, a clear description, and one + worked example (``example_input`` and its correct ``example_output``). Do not + use any tools.""" + + +@Template.define +def estimate_difficulty(problem: Problem) -> int: + """Rate the difficulty of {problem} from 1 (trivial) to 5 (very hard), + returning just the integer. Do not use any tools.""" + + +@Template.define +def write_solution(problem: Problem) -> Callable[[str], str]: + """Write a Python function that solves {problem}: it takes the input string and + returns the required output string. It must reproduce the worked example.""" + + +@Template.define +def judge(problem: Problem, output: str) -> bool: + """For {problem}, the candidate solution produced {output} when run on the + example input. Decide whether that matches the expected ``example_output``. + Do not use any tools.""" + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--topic", + type=str, + default="text processing", + help="Topic the generated coding problem should be about", + ) + args = parser.parse_args() + + # Dataclass return type, decoded from constrained generation. + problem = pose_problem(args.topic) + assert isinstance(problem, Problem) + print(f"# {problem.title}\n{problem.description}") + print(f"example: {problem.example_input!r} -> {problem.example_output!r}") + + # Primitive int decode; the dataclass is round-tripped back in as prompt input. + difficulty = estimate_difficulty(problem) + assert isinstance(difficulty, int) + print(f"\nDifficulty: {difficulty}/5") + + # Synthesize an executable Callable and inspect its source. + solution = write_solution(problem) + assert callable(solution) + print("\nGenerated solution:") + print(inspect.getsource(solution)) + + # Run the synthesized function, then decode a bool verdict from the judge. + output = solution(problem.example_input) + print(f"solution({problem.example_input!r}) == {output!r}") + verdict = judge(problem, output) + assert isinstance(verdict, bool) + print("PASS" if verdict else "FAIL") + + +if __name__ == "__main__": + main() From 234c05998f8408e61613be99345cda5014bec655 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 16:43:21 -0400 Subject: [PATCH 035/155] nits --- docs/source/llm_examples/async_concurrency.py | 46 ------------- docs/source/llm_examples/conversation.py | 2 +- docs/source/llm_examples/error_recovery.py | 5 +- docs/source/llm_examples/flight_booking.py | 45 +++++-------- docs/source/llm_examples/guardrails.py | 1 + .../llm_examples/hanoi_solver_iterative.py | 1 + .../llm_examples/hanoi_solver_recursive.py | 5 +- docs/source/llm_examples/hitl.py | 2 + docs/source/llm_examples/image_input.py | 27 ++------ docs/source/llm_examples/lexical_scope.py | 66 ++++++++----------- docs/source/llm_examples/majority_vote.py | 1 + docs/source/llm_examples/map_reduce.py | 4 +- docs/source/llm_examples/taboo.py | 1 + docs/source/llm_examples/text2sql.py | 1 + effectful/handlers/llm/harness.py | 11 ++-- 15 files changed, 75 insertions(+), 143 deletions(-) delete mode 100644 docs/source/llm_examples/async_concurrency.py diff --git a/docs/source/llm_examples/async_concurrency.py b/docs/source/llm_examples/async_concurrency.py deleted file mode 100644 index f49cfc1e9..000000000 --- a/docs/source/llm_examples/async_concurrency.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Fork/join async concurrency with templates. - -Demonstrates: -- Running multiple LLM template calls concurrently with ``asyncio.gather`` -- Using ``asyncio.to_thread`` to run synchronous template calls in parallel -""" - -import asyncio -import functools - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Async template -# --------------------------------------------------------------------------- - - -@Template.define -def analyze_average_age(ages: list[int]) -> int: - """Analyze the dataset of ages {ages} and return the average age of - participants. Do not use any tools.""" - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -def main() -> None: - async def run() -> None: - analysis = functools.partial(asyncio.to_thread, analyze_average_age) - results = await asyncio.gather( - analysis([25, 30, 35, 40]), - analysis([20, 28, 17, 30]), - analysis([22, 27, 31, 29]), - analysis([24, 26, 32, 38]), - analysis([21, 29, 33, 37]), - ) - for i, result in enumerate(results): - print(f"Group {i}: average age = {result}") - - asyncio.run(run()) - - -if __name__ == "__main__": - main() diff --git a/docs/source/llm_examples/conversation.py b/docs/source/llm_examples/conversation.py index 28044728e..7b99e244d 100644 --- a/docs/source/llm_examples/conversation.py +++ b/docs/source/llm_examples/conversation.py @@ -17,7 +17,7 @@ class ChatBot(Agent): """Conversational agent that remembers the conversation so far.""" - bot_name: str = dataclasses.field(default="ChatBot") + bot_name: str @Template.define def send(self, user_input: str) -> str: diff --git a/docs/source/llm_examples/error_recovery.py b/docs/source/llm_examples/error_recovery.py index c8b90e647..8146419e9 100644 --- a/docs/source/llm_examples/error_recovery.py +++ b/docs/source/llm_examples/error_recovery.py @@ -46,6 +46,7 @@ class Rating: A movie rating, with a score (an integer from 1 to 5) and an explanation. The explanation MUST mention the score, otherwise it will be rejected as invalid. """ + score: typing.Literal[1, 2, 3, 4, 5] explanation: str @@ -53,7 +54,9 @@ def __post_init__(self): if self.score < 1 or self.score > 5: raise ValueError(f"score must be 1-5, got {self.score}") if str(self.score) not in self.explanation: - raise ValueError(f"explanation must mention the score {self.score}, got '{self.explanation}'") + raise ValueError( + f"explanation must mention the score {self.score}, got '{self.explanation}'" + ) # --------------------------------------------------------------------------- diff --git a/docs/source/llm_examples/flight_booking.py b/docs/source/llm_examples/flight_booking.py index c0530e094..e3f19dfff 100644 --- a/docs/source/llm_examples/flight_booking.py +++ b/docs/source/llm_examples/flight_booking.py @@ -14,7 +14,7 @@ import enum from typing import Literal -from effectful.handlers.llm import Agent, Template, Tool +from effectful.handlers.llm import Agent, Template # --------------------------------------------------------------------------- # Structured output types @@ -28,14 +28,14 @@ class Airport(enum.StrEnum): JNU = "JNU" NYC = "NYC" LAX = "LAX" - CHI = "CHI" + ORD = "ORD" MIA = "MIA" BOS = "BOS" SEA = "SEA" DFW = "DFW" DEN = "DEN" ATL = "ATL" - HOU = "HOU" + IAH = "IAH" @dataclasses.dataclass(frozen=True) @@ -51,11 +51,12 @@ class FlightDetails: class SeatPreference: """ User's seat preference extracted from natural language. - + Seats A and F are window seats. Seats C and D are aisle seats. Row 1 is the front row with extra legroom. Rows 14 and 20 also have extra legroom. """ + row: int # 1-30 seat: Literal["A", "B", "C", "D", "E", "F"] @@ -68,11 +69,11 @@ class SeatPreference: 1. Flight SFO-AK123 - $350 - San Francisco (SFO) to Anchorage (ANC) - 2025-01-10 2. Flight SFO-AK456 - $370 - San Francisco (SFO) to Fairbanks (FAI) - 2025-01-10 3. Flight SFO-AK789 - $400 - San Francisco (SFO) to Juneau (JNU) - 2025-01-20 -4. Flight NYC-LA101 - $250 - San Francisco (SFO) to Anchorage (ANC) - 2025-01-10 -5. Flight CHI-MIA202 - $200 - Chicago (ORD) to Miami (MIA) - 2025-01-12 -6. Flight BOS-SEA303 - $120 - Boston (BOS) to Anchorage (ANC) - 2025-01-12 +4. Flight NYC-LA101 - $250 - New York (NYC) to Los Angeles (LAX) - 2025-01-10 +5. Flight ORD-MIA202 - $200 - Chicago (ORD) to Miami (MIA) - 2025-01-12 +6. Flight BOS-SEA303 - $120 - Boston (BOS) to Seattle (SEA) - 2025-01-12 7. Flight DFW-DEN404 - $150 - Dallas (DFW) to Denver (DEN) - 2025-01-10 -8. Flight ATL-HOU505 - $180 - Atlanta (ATL) to Houston (IAH) - 2025-01-10 +8. Flight ATL-IAH505 - $180 - Atlanta (ATL) to Houston (IAH) - 2025-01-10 """ # --------------------------------------------------------------------------- @@ -88,37 +89,26 @@ def extract_flights(web_page_text: str) -> list[FlightDetails]: """ -# --------------------------------------------------------------------------- -# Tool that delegates to the extraction template -# --------------------------------------------------------------------------- - -# The tool is defined at module scope so that FlightFinder's template -# captures it via lexical scope (same pattern as search_web in other examples). - - -@Tool.define -def get_available_flights() -> list[FlightDetails]: - """Retrieve all available flights from the booking page.""" - return extract_flights(FLIGHTS_PAGE) - - # --------------------------------------------------------------------------- # Flight search agent # --------------------------------------------------------------------------- +@dataclasses.dataclass class FlightFinder(Agent): """Agent that finds flights matching user criteria.""" + available_flights: list[FlightDetails] + @Template.define def find_flight( self, origin: Airport, destination: Airport, date: datetime.date ) -> FlightDetails: - """Find the cheapest flight from {origin} to {destination} on {date}. + """ + Find the cheapest flight from {origin} to {destination} on {date}. - Use the get_available_flights tool to retrieve all flights, then - select the cheapest one that matches the origin, destination, - and date exactly. + List of available flights (from the web page): + {self.available_flights} """ @@ -170,7 +160,7 @@ def book_flight( max_retries: int = 3, ) -> None: """End-to-end flight booking with search, validation, and seat selection.""" - searcher = FlightFinder() + searcher = FlightFinder(available_flights=extract_flights(FLIGHTS_PAGE)) # --- Search with validation retry --- flight = None @@ -216,6 +206,7 @@ def book_flight( # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) airports = list(Airport) diff --git a/docs/source/llm_examples/guardrails.py b/docs/source/llm_examples/guardrails.py index 90bf0c3e0..e23f0701d 100644 --- a/docs/source/llm_examples/guardrails.py +++ b/docs/source/llm_examples/guardrails.py @@ -45,6 +45,7 @@ def is_safe_query(user_query: str) -> bool: # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/hanoi_solver_iterative.py b/docs/source/llm_examples/hanoi_solver_iterative.py index 0b81573b4..2a5e069d6 100644 --- a/docs/source/llm_examples/hanoi_solver_iterative.py +++ b/docs/source/llm_examples/hanoi_solver_iterative.py @@ -163,6 +163,7 @@ def solve_hanoi(state: GameState, max_steps: int = 30): # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/hanoi_solver_recursive.py b/docs/source/llm_examples/hanoi_solver_recursive.py index 4be6ebe9e..c19401dcc 100644 --- a/docs/source/llm_examples/hanoi_solver_recursive.py +++ b/docs/source/llm_examples/hanoi_solver_recursive.py @@ -101,9 +101,7 @@ def __str__(self) -> str: @Template.define -def solve( - n_disks: int, source: int, target: int, auxiliary: int -) -> list[Step]: +def solve(n_disks: int, source: int, target: int, auxiliary: int) -> list[Step]: """Solve Tower of Hanoi: move {n_disks} disks from tower {source} to tower {target}, using tower {auxiliary} as temporary storage. @@ -147,6 +145,7 @@ def validate_solution(size: int, steps: list[Step]) -> bool: # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/hitl.py b/docs/source/llm_examples/hitl.py index 5dd693ddb..8c9d64aed 100644 --- a/docs/source/llm_examples/hitl.py +++ b/docs/source/llm_examples/hitl.py @@ -41,6 +41,7 @@ class ProposedAction: @dataclasses.dataclass class Planner(Agent): """Agent that proposes actions one at a time for human approval.""" + execution_log: list[str] = dataclasses.field(default_factory=list) @Tool.define @@ -113,6 +114,7 @@ def run_with_approval( # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/image_input.py b/docs/source/llm_examples/image_input.py index b2527ef3c..68f0d5bee 100644 --- a/docs/source/llm_examples/image_input.py +++ b/docs/source/llm_examples/image_input.py @@ -13,22 +13,6 @@ from effectful.handlers.llm import Template -# --------------------------------------------------------------------------- -# Inline image (32x32 yellow smiley face) -# --------------------------------------------------------------------------- - -IMAGE_BASE64 = ( - "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAhElEQVR4nO2W4QqA" - "MAiEVXr/VzYWDGoMdk7Cgrt/sUs/DqZTd3EplFU2JwATYAJMoOlAB4bq89s95+Mg" - "+gyAchsKAYplBBBA43hFhfxnUixDjdEUUL8hpr7R0KLdt9qElzcyiu8As+Kr8zQA" - "mgLavAl+kIzFZyCRxtsAmWb/voZvqRzgBE1sIDuVFX4eAAAAAElFTkSuQmCC" -) - - -# --------------------------------------------------------------------------- -# Template -# --------------------------------------------------------------------------- - @Template.define def describe_image(image: Image.Image) -> str: @@ -37,11 +21,6 @@ def describe_image(image: Image.Image) -> str: """ -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -56,6 +35,12 @@ def main() -> None: if args.image is not None: image = Image.open(args.image) else: + IMAGE_BASE64 = ( + "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAhElEQVR4nO2W4QqA" + "MAiEVXr/VzYWDGoMdk7Cgrt/sUs/DqZTd3EplFU2JwATYAJMoOlAB4bq89s95+Mg" + "+gyAchsKAYplBBBA43hFhfxnUixDjdEUUL8hpr7R0KLdt9qElzcyiu8As+Kr8zQA" + "mgLavAl+kIzFZyCRxtsAmWb/voZvqRzgBE1sIDuVFX4eAAAAAElFTkSuQmCC" + ) image = Image.open(io.BytesIO(base64.b64decode(IMAGE_BASE64))) print(describe_image(image)) diff --git a/docs/source/llm_examples/lexical_scope.py b/docs/source/llm_examples/lexical_scope.py index 2a8630cd1..e61f23d01 100644 --- a/docs/source/llm_examples/lexical_scope.py +++ b/docs/source/llm_examples/lexical_scope.py @@ -17,10 +17,6 @@ from effectful.handlers.llm import Agent, Template, Tool -# --------------------------------------------------------------------------- -# Sub-templates (module-level; auto-captured into the scopes below) -# --------------------------------------------------------------------------- - @Template.define def story_with_moral(topic: str) -> str: @@ -32,12 +28,6 @@ def story_funny(topic: str) -> str: """Write a funny, humorous story about {topic}.""" -# --------------------------------------------------------------------------- -# (1) Model-driven composition: an orchestrator template calls tools and -# sub-templates directly during its own turn. -# --------------------------------------------------------------------------- - - class TripPlanner(Agent): """Plans a trip to a city with good weather and tells a story about visiting it.""" @@ -59,24 +49,6 @@ def plan_trip_story(self, style: str) -> str: style: {style}""" -# --------------------------------------------------------------------------- -# (2) Code-driven composition: a template synthesizes a function that calls the -# same sub-templates when executed. -# --------------------------------------------------------------------------- - - -@Template.define -def write_story_fn(style: Literal["moral", "funny"]) -> Callable[[str], str]: - """Generate a Python function that takes a topic string and returns a story - about it in the {style} style. The function should delegate the writing to the - `story_funny` sub-template for humor, or `story_with_moral` for a lesson.""" - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -92,19 +64,35 @@ def main() -> None: default="a curious cat", help="Topic for the synthesized story function to run on", ) + parser.add_argument( + "--method", + type=str, + choices=["model", "code"], + default="model", + help="Whether to run the model-driven or code-driven composition", + ) args = parser.parse_args() - # (1) Model-driven: the orchestrator template calls tools and sub-templates. - print("=== Orchestrator template (model-driven composition) ===") - planner = TripPlanner() - print(planner.plan_trip_story(args.style)) - - # (2) Code-driven: the model synthesizes a function that calls the sub-templates. - print(f"\n=== Synthesized higher-order function (style={args.style}) ===") - story_fn = write_story_fn(args.style) - print(inspect.getsource(story_fn)) - print(f"\n=== Running it on {args.topic!r} ===") - print(story_fn(args.topic)) + if args.method == "model": + # (1) Model-driven: the orchestrator template calls tools and sub-templates. + print("=== Orchestrator template (model-driven composition) ===") + planner = TripPlanner() + print(planner.plan_trip_story(args.style)) + + elif args.method == "code": + + @Template.define + def write_story_fn(style: Literal["moral", "funny"]) -> Callable[[str], str]: + """Generate a Python function that takes a topic string and returns a story + about it in the {style} style. The function should delegate the writing to the + `story_funny` sub-template for humor, or `story_with_moral` for a lesson.""" + + # (2) Code-driven: the model synthesizes a function that calls the sub-templates. + print(f"\n=== Synthesized higher-order function (style={args.style}) ===") + story_fn = write_story_fn(args.style) + print(inspect.getsource(story_fn)) + print(f"\n=== Running it on {args.topic!r} ===") + print(story_fn(args.topic)) if __name__ == "__main__": diff --git a/docs/source/llm_examples/majority_vote.py b/docs/source/llm_examples/majority_vote.py index 26606b56e..31041a963 100644 --- a/docs/source/llm_examples/majority_vote.py +++ b/docs/source/llm_examples/majority_vote.py @@ -47,6 +47,7 @@ def majority_vote[Q]( # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/map_reduce.py b/docs/source/llm_examples/map_reduce.py index 7dfeb1826..61ce44826 100644 --- a/docs/source/llm_examples/map_reduce.py +++ b/docs/source/llm_examples/map_reduce.py @@ -97,7 +97,8 @@ async def map_reduce_evaluate( job_description: str, ) -> str: """Evaluate resumes in parallel (map), then summarize (reduce).""" - # Map: evaluate each resume concurrently + # Map: fork/join -- evaluate each resume concurrently via asyncio.gather + + # asyncio.to_thread (sync template calls run in parallel threads). evaluate = functools.partial(asyncio.to_thread, evaluate_resume) evaluations: list[Evaluation] = list( await asyncio.gather(*(evaluate(resume, job_description) for resume in resumes)) @@ -117,6 +118,7 @@ async def map_reduce_evaluate( # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/taboo.py b/docs/source/llm_examples/taboo.py index 677ce3fc7..ffb7f02d0 100644 --- a/docs/source/llm_examples/taboo.py +++ b/docs/source/llm_examples/taboo.py @@ -122,6 +122,7 @@ def play_taboo( # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/docs/source/llm_examples/text2sql.py b/docs/source/llm_examples/text2sql.py index 2b5e8794b..528483982 100644 --- a/docs/source/llm_examples/text2sql.py +++ b/docs/source/llm_examples/text2sql.py @@ -125,6 +125,7 @@ def text_to_sql( # Main # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/effectful/handlers/llm/harness.py b/effectful/handlers/llm/harness.py index ec80bf453..d75dab4b5 100644 --- a/effectful/handlers/llm/harness.py +++ b/effectful/handlers/llm/harness.py @@ -37,7 +37,6 @@ def main() -> None: from effectful.handlers.llm.completions import ( LangfuseTracer, - LexicalReaders, LiteLLMProvider, PythonRepl, RetryLLMHandler, @@ -86,8 +85,8 @@ def __init__( render: bool = False, dump_system_prompt: str | os.PathLike[str] | None = None, tool_choice: str = "required", - api_base: str = "http://localhost:8030/v1", - api_key: str = "", + api_base: str | None = None, + api_key: str | None = None, ) -> None: self.model = model self.num_retries = num_retries @@ -122,7 +121,7 @@ def __enter__(self) -> "harness": stack.enter_context( handler(RetryLLMHandler(stop=tenacity.stop_after_attempt(self.num_retries))) ) - stack.enter_context(handler(LexicalReaders())) + # stack.enter_context(handler(LexicalReaders())) if self.langfuse: stack.enter_context(handler(LangfuseTracer())) self._stack = stack @@ -185,6 +184,10 @@ def main(argv: list[str] | None = None) -> None: langfuse=ns.langfuse, render=ns.render, dump_system_prompt=ns.dump_system_prompt, + api_base="http://localhost:8030/v1" + if ns.model == "openai/deepseek-v4-flash" + else None, + api_key="" if ns.model == "openai/deepseek-v4-flash" else None, ): runpy.run_path(ns.script, run_name="__main__") From 9902b8445b4dfbd03940944148a459516efdff17 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 17:38:04 -0400 Subject: [PATCH 036/155] fix docstring --- effectful/handlers/llm/template.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index e4870ffda..41ee0beb2 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -40,7 +40,6 @@ def weather(city: str) -> str: @Template.define # cities and weather auto-captured from lexical scope def vacation() -> str: \"\"\"Use the `cities` and `weather` tools to suggest a city that has good weather.\"\"\" - raise NotHandled ``` Class methods may be used as templates, in which case any other methods @@ -101,7 +100,7 @@ class Template[**P, T](Tool[P, T]): ## Constructing Templates Apply `Template.define` as a decorator to a fully type-annotated function or - method whose body is `raise NotHandled`. The docstring is a + method whose body is either empty or `raise NotHandled`. The docstring is a [format string](https://docs.python.org/3/library/string.html#format-string-syntax) prompt: its `{...}` fields are filled at call time (see *Prompt assembly* below) and the LLM's response is decoded to the return type. @@ -122,7 +121,6 @@ class Template[**P, T](Tool[P, T]): >>> @Template.define ... def limerick(theme: str) -> str: ... \"\"\"Write a limerick on the theme of {theme}. Do not use any tools.\"\"\" - ... raise NotHandled ## Structured output @@ -134,7 +132,6 @@ class Template[**P, T](Tool[P, T]): >>> @Template.define ... def primes(first_digit: int) -> int: ... \"\"\"Give a prime number with {first_digit} as the first digit. Do not use any tools.\"\"\" - ... raise NotHandled Structured generation is used to constrain the LLM to return values that can be decoded without error. @@ -149,7 +146,6 @@ class Template[**P, T](Tool[P, T]): >>> @Template.define ... def write_joke(theme: str) -> KnockKnockJoke: ... \"\"\"Write a knock-knock joke on the theme of {theme}. Do not use any tools.\"\"\" - ... raise NotHandled Many common Python data types are decodable without additional effort. To register a decoder for a custom type, see `effectful.handlers.llm.encoding.type_to_encodable_type`. @@ -390,7 +386,6 @@ class ChatBot(Agent): @Template.define def send(self, user_input: str) -> str: \"""Friendly bot named {self.bot_name}. User writes: {user_input}\""" - raise NotHandled provider = LiteLLMProvider() chatbot = ChatBot() From 2cc4c3531f73687d28781dbdc393928c9ae0ff5a Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 17:43:21 -0400 Subject: [PATCH 037/155] tool choice in harness --- effectful/handlers/llm/harness.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/effectful/handlers/llm/harness.py b/effectful/handlers/llm/harness.py index d75dab4b5..81e5974ca 100644 --- a/effectful/handlers/llm/harness.py +++ b/effectful/handlers/llm/harness.py @@ -84,7 +84,7 @@ def __init__( langfuse: bool = False, render: bool = False, dump_system_prompt: str | os.PathLike[str] | None = None, - tool_choice: str = "required", + tool_choice: str = "auto", api_base: str | None = None, api_key: str | None = None, ) -> None: @@ -171,6 +171,13 @@ def _parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: metavar="PATH", help="Dump the assembled system prompt to this Markdown file", ) + parser.add_argument( + "--tool-choice", + type=str, + default="auto", + choices=["required", "auto", "none"], + help="Whether to require, allow, or disable tool calls (none means disabled)", + ) return parser.parse_known_args(argv) @@ -184,6 +191,7 @@ def main(argv: list[str] | None = None) -> None: langfuse=ns.langfuse, render=ns.render, dump_system_prompt=ns.dump_system_prompt, + tool_choice=ns.tool_choice, api_base="http://localhost:8030/v1" if ns.model == "openai/deepseek-v4-flash" else None, From d0f0d4b3e5bec053acf29e9cea779d9dcdc1e5ec Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 19:18:57 -0400 Subject: [PATCH 038/155] contextvar --- effectful/internals/runtime.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/effectful/internals/runtime.py b/effectful/internals/runtime.py index 4c9ebd7b1..e9d925c35 100644 --- a/effectful/internals/runtime.py +++ b/effectful/internals/runtime.py @@ -1,36 +1,27 @@ import contextlib -import dataclasses +import contextvars import functools import inspect from collections.abc import Callable, Mapping -from threading import local from effectful.ops.types import Interpretation, Operation +_INTERPRETATION: "contextvars.ContextVar[Interpretation]" = contextvars.ContextVar( + "effectful_interpretation", default={} +) -@dataclasses.dataclass -class Runtime[S, T](local): - interpretation: "Interpretation[S, T]" - -@functools.lru_cache(maxsize=1) -def get_runtime() -> Runtime: - return Runtime(interpretation={}) - - -def get_interpretation(): - return get_runtime().interpretation +def get_interpretation() -> "Interpretation": + return _INTERPRETATION.get() @contextlib.contextmanager def interpreter(intp: "Interpretation"): - r = get_runtime() - old_intp = r.interpretation + token = _INTERPRETATION.set(intp) try: - old_intp, r.interpretation = r.interpretation, dict(intp) yield intp finally: - r.interpretation = old_intp + _INTERPRETATION.reset(token) @Operation.define From ad8afa8e425cc495688358c4c092aeb541e97efb Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 19:34:56 -0400 Subject: [PATCH 039/155] remove dumb example --- docs/source/llm_examples/tao_agent.py | 176 -------------------------- 1 file changed, 176 deletions(-) delete mode 100644 docs/source/llm_examples/tao_agent.py diff --git a/docs/source/llm_examples/tao_agent.py b/docs/source/llm_examples/tao_agent.py deleted file mode 100644 index 6a58276de..000000000 --- a/docs/source/llm_examples/tao_agent.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Think-Act-Observe agent: structured chain-of-thought with optional tool use. - -Demonstrates: -- Agent mixin for persistent conversation history (the LLM sees its own prior - reasoning across steps) -- Structured output with an AgentThought dataclass carrying an is_final flag -- A think -> act -> observe loop that continues until the agent is done -- Pattern-matching action dispatch: reason straight to an answer, or call a - web-search tool when a fact is missing -""" - -import argparse -import dataclasses -import enum -import urllib.parse - -import requests - -from effectful.handlers.llm import Agent, Template, Tool - -# --------------------------------------------------------------------------- -# Search tool -# --------------------------------------------------------------------------- - - -@Tool.define -def search_web(query: str) -> str: - """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question.""" - search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( - { - "action": "query", - "list": "search", - "srsearch": query, - "srlimit": 1, - "format": "json", - } - ) - search_data = requests.get( - search_url, headers={"User-Agent": "effectful-example/1.0"} - ).json() - results = search_data.get("query", {}).get("search", []) - if not results: - return f"No results found for: {query}" - title = results[0]["title"] - - summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode( - { - "action": "query", - "titles": title, - "prop": "extracts", - "exintro": True, - "explaintext": True, - "format": "json", - } - ) - summary_data = requests.get( - summary_url, headers={"User-Agent": "effectful-example/1.0"} - ).json() - page = next(iter(summary_data["query"]["pages"].values())) - extract = page.get("extract", "No summary available.") - url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}" - - return f"# {title}\n\n{extract}\n\nSource: {url}" - - -# --------------------------------------------------------------------------- -# Structured output types -# --------------------------------------------------------------------------- - - -class AgentAction(enum.StrEnum): - search_the_web = "search_the_web" - answer = "answer" - - -@dataclasses.dataclass(frozen=True) -class AgentThought: - thinking: str - action: AgentAction - action_input: str - is_final: bool - - -# --------------------------------------------------------------------------- -# TAO Agent -# --------------------------------------------------------------------------- - - -class TAOAgent(Agent): - """Think-Act-Observe agent that reasons step by step.""" - - @Template.define - def think(self, query: str) -> AgentThought: - """You are an AI assistant solving a problem. Based on the user's query - ({query}) and your own prior reasoning in the conversation history, think - about what to do next: either `search_the_web` for a fact you are missing, - or `answer` once you can conclude. Set is_final=true when your action is - the final answer. - """ - - @Template.define - def observe(self, action: str, action_input: str, action_result: str) -> str: - """You are an observer. Provide a concise, objective observation of this result. - - Action: {action} - Action input: {action_input} - Action result: {action_result} - - - Do not make decisions, just describe what you see. - - """ - - def run(self, query: str, max_steps: int = 5) -> str: - result = "" - for i in range(max_steps): - thought = self.think(query) - print(f" [step {i + 1}] {thought.thinking}") - result = self._act(thought.action, thought.action_input) - self.observe(str(thought.action), thought.action_input, result) - if thought.is_final: - break - return result - - def _act(self, action: AgentAction, action_input: str) -> str: - match action: - case AgentAction.search_the_web: - return search_web(action_input) - case AgentAction.answer: - return action_input - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--max-steps", - type=int, - default=5, - help="Maximum think-act-observe steps per problem", - ) - parser.add_argument( - "--problem", - dest="problems", - metavar="PROBLEM", - nargs="+", - default=[ - ( - "A farmer has 17 sheep. All but 9 run away. " - "Then he buys 5 more. How many sheep does he have now?" - ), - "What year was the Eiffel Tower completed, and how tall is it?", - ], - help=( - "One or more problems to solve (a pure-reasoning puzzle and a " - "web-lookup question by default)" - ), - ) - args = parser.parse_args() - - # By default, one puzzle the agent can reason through with no tools, and one - # that needs a web lookup -- the same loop handles both via its action - # dispatch. - for problem in args.problems: - agent = TAOAgent() - print(f"\nProblem: {problem}") - answer = agent.run(problem, max_steps=args.max_steps) - print(f"Answer: {answer}") - - -if __name__ == "__main__": - main() From bcb5a0c8a8482fae744af282d18cfecc71af8af5 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 19:40:13 -0400 Subject: [PATCH 040/155] image --- docs/source/llm_examples/image_tool.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/llm_examples/image_tool.py b/docs/source/llm_examples/image_tool.py index 5325eeb8b..afe736aee 100644 --- a/docs/source/llm_examples/image_tool.py +++ b/docs/source/llm_examples/image_tool.py @@ -1,4 +1,5 @@ import argparse +import pathlib from PIL import Image @@ -65,11 +66,18 @@ def rotate_and_concat(self, i: Image.Image) -> Image.Image: def main() -> None: + DEFAULT_IMAGE = ( + pathlib.Path(__file__).resolve().parent.parent + / "_static" + / "img" + / "chirho_logo_wide.png" + ) + parser = argparse.ArgumentParser(description=__doc__ or ImageTools.__doc__) parser.add_argument( "--image", type=str, - default="../_static/img/chirho_logo_wide.png", + default=str(DEFAULT_IMAGE), metavar="PATH", help="Path to the input image to rotate-and-concatenate.", ) From 1c5cdc671ea5bdb795c4dd19e4a3ece717c8b73a Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 20:00:18 -0400 Subject: [PATCH 041/155] fix synthesisfinaltool tool collection bug --- effectful/handlers/llm/completions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 26037a9bd..07dc0efa4 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -260,7 +260,7 @@ def call_assistant[T]( includes the raw assistant message for retry handling. """ name2tool = {t.__name__: t for t in tools} - assert len(name2tool) == len(tools) + assert len(tools) == len(name2tool), "Tool name collision detected" env = {_TOOLS_KEY: name2tool, **env} tool_specs = [] for name, t in sorted(name2tool.items()): @@ -819,6 +819,8 @@ def _apply[**P, T]( tool = self._SynthesisFinalTool.define(template, bound_args) def _add_synthesis_tool(env, response_type, tools=frozenset(), anchor=None): + if any(isinstance(t, self._SynthesisFinalTool) for t in tools): + return fwd() return fwd(env, response_type, tools | {tool}, anchor=anchor) with handler({call_assistant: _add_synthesis_tool}): From 969c7035f63367a3499942fbee41c15d546693d8 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 20:14:52 -0400 Subject: [PATCH 042/155] remove hanoi hints --- .../llm_examples/hanoi_solver_recursive.py | 38 +------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/docs/source/llm_examples/hanoi_solver_recursive.py b/docs/source/llm_examples/hanoi_solver_recursive.py index c19401dcc..755b4346d 100644 --- a/docs/source/llm_examples/hanoi_solver_recursive.py +++ b/docs/source/llm_examples/hanoi_solver_recursive.py @@ -1,28 +1,4 @@ -"""Recursive LLM-based Towers of Hanoi solver. - -Adapted from https://github.com/BasisResearch/effectful/pull/404 - -Demonstrates: -- ``IsRecursive`` annotation to let a template call itself as a tool -- Recursive problem decomposition via LLM tool calls -- Post-hoc validation of the LLM-generated move sequence - -The classic recursive algorithm for Tower of Hanoi is: - - hanoi(n, source, target, auxiliary): - if n == 1: move disk from source to target - else: - hanoi(n-1, source, auxiliary, target) # move n-1 disks out of the way - move largest disk from source to target # move the bottom disk - hanoi(n-1, auxiliary, target, source) # move n-1 disks to target - -This solver defines a recursive ``Template`` that can call itself as a tool. -The LLM decomposes the n-disk problem into three sub-steps, making recursive -tool calls for the (n-1)-disk sub-problems, and returns the concatenated -list of moves. - -See: https://en.wikipedia.org/wiki/Tower_of_Hanoi -""" +"""Recursive LLM-based Towers of Hanoi solver.""" import argparse import dataclasses @@ -102,18 +78,8 @@ def __str__(self) -> str: @Template.define def solve(n_disks: int, source: int, target: int, auxiliary: int) -> list[Step]: - """Solve Tower of Hanoi: move {n_disks} disks from tower {source} to + """Solve Tower of Hanoi using recursion: move {n_disks} disks from tower {source} to tower {target}, using tower {auxiliary} as temporary storage. - - Recursive strategy: - - Base case (n_disks == 1): return [Step(start=source, end=target)] - - Recursive case (n_disks > 1): - 1. Call solve(n_disks - 1, source, auxiliary, target) to move the - top n_disks-1 disks out of the way onto the auxiliary tower. - 2. Move the largest disk: Step(start=source, end=target). - 3. Call solve(n_disks - 1, auxiliary, target, source) to move the - n_disks-1 disks from auxiliary to the target tower. - 4. Return the concatenated list of all steps from (1), (2), and (3). """ From e6f5c71047fdc9d1e7c9c51711afdc620562683c Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 21:30:40 -0400 Subject: [PATCH 043/155] fix bug --- effectful/handlers/llm/completions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 07dc0efa4..5a13d7d73 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -9,6 +9,7 @@ import json import pathlib import re +import sys import time import traceback import types @@ -1369,8 +1370,13 @@ class TerminalRenderer(ObjectInterpretation): :func:`litellm.stream_chunk_builder` so the rest of the pipeline is unchanged. """ + # Pin the console to the process's original stdout rather than the live + # ``sys.stdout``. Otherwise, when a nested ``completion`` renders while stdout + # is redirected -- inside ``exec_code`` (``redirect_stdout``) or ``run_doctests`` + # (doctest's ``_SpoofOut``) -- the rendered panels are captured and fed back + # into the model's context. ``sys.__stdout__`` is immune to those rebindings. console: rich.console.Console = dataclasses.field( - default_factory=rich.console.Console + default_factory=lambda: rich.console.Console(file=sys.__stdout__) ) @implements(completion) From 6e640286105fc22f261f7ec1293b07fecade8ab4 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 21:42:41 -0400 Subject: [PATCH 044/155] nit --- docs/source/llm_examples/map_reduce.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/llm_examples/map_reduce.py b/docs/source/llm_examples/map_reduce.py index 61ce44826..3efca2697 100644 --- a/docs/source/llm_examples/map_reduce.py +++ b/docs/source/llm_examples/map_reduce.py @@ -12,6 +12,7 @@ import collections.abc import dataclasses import functools +import typing from effectful.handlers.llm import Template @@ -26,7 +27,7 @@ class Evaluation: qualified: bool strengths: str weaknesses: str - score: int # 1-10 + score: typing.Literal[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] # --------------------------------------------------------------------------- @@ -43,8 +44,6 @@ def evaluate_resume(resume: str, job_description: str) -> Evaluation: Resume: {resume} - - Score from 1 (poor fit) to 10 (perfect fit). """ From ed6de7d088e9039802083da0780e915c8a48d924 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 21:57:42 -0400 Subject: [PATCH 045/155] comment about nasty bug --- effectful/handlers/llm/completions.py | 36 +++++++++++++++++++++++++++ effectful/handlers/llm/encoding.py | 19 +++++++------- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index 5a13d7d73..eea2c1af5 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -751,6 +751,42 @@ class SynthesizeAndCall(ObjectInterpretation): be compiled and executed. """ + # TODO FIX THIS!!! + # KNOWN BUG -- the synthesized function is NOT type-checked against the + # Template. + # + # The docstring above (and PR #706) promise that the synthesized code is + # "type-checked": `decode` (encoding.py) splices the generated function into + # the Template's own module source and runs mypy on it, so the body is checked + # in its real lexical scope. That splice only happens when the enclosing + # Template's underlying function rides in the decode context under + # `TYPE_CHECK_ANCHOR_KEY` (see `decode`, encoding.py, and `_serialize`'s + # `if anchor is not None:` guard). + # + # That anchor is threaded on exactly ONE of the two decode paths in + # `call_assistant`: + # - Direct result (model answers with content): decoded with + # `context={**env, TYPE_CHECK_ANCHOR_KEY: anchor}` -> spliced + mypy-checked. + # - Tool call (model calls a tool): decoded with `context=env` -- no anchor + # -> splice skipped. + # + # `submit_solution` is a `FinalTool`, so the model always reaches it via the + # *tool-call* path. Its `implementation` argument is therefore decoded WITHOUT + # the anchor, so the mypy splice never runs for it. The synthesized function -- + # the whole product of this handler -- is only validated structurally + # (`_validate_signature_callable`: parameter count/return match), compiled, and + # executed (plus its own doctests). Its body escapes the type check entirely. + # + # The tool-call path drops the anchor deliberately: a Callable passed to an + # *arbitrary* tool is contracted by that tool's parameter type, not by the + # Template's body, so anchoring it to the Template would be wrong. But + # `submit_solution` is special -- its Callable argument *is* the Template's + # body -- so for this FinalTool the anchor should be the enclosing Template's + # `__default__`. The fix is to thread that anchor into the decode context when + # decoding a synthesis FinalTool's argument (rather than using the generic + # `context=env` tool-call path), so `submit_solution`'s implementation is + # spliced into the Template and mypy-checked like a directly-returned result. + @typing.final class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): """## Code synthesis diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index a6956f247..7419645c7 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -17,7 +17,6 @@ Mapping, MutableMapping, ) -from typing import Any import litellm import pydantic @@ -284,7 +283,7 @@ def _pydantic_type_str[T](ty: type[T]) -> type[T]: @TypeToPydanticType.register(object) -def _pydantic_type_base(ty: type) -> Any: +def _pydantic_type_base(ty: type) -> typing.Any: return ty @@ -533,7 +532,7 @@ def _class_template(self) -> Template[..., T] | None: else: return None - def _method_instance(self, other: Template) -> Any | None: + def _method_instance(self, other: Template) -> typing.Any | None: """The instance ``op`` is bound to, if ``op`` is this synthesized Agent-method on *some* instance; otherwise ``None``. """ @@ -680,8 +679,8 @@ def _validate_signature_callable( @TypeToPydanticType.register(Callable) def _pydantic_callable( - callable_type: Any, metadata: _SynthesisSpec | None = None -) -> Any: + callable_type: typing.Any, metadata: _SynthesisSpec | None = None +) -> typing.Any: """Create a Pydantic-compatible Annotated type for a parameterized Callable. Usage: PydanticCallable(Callable[[int, str], bool]) @@ -711,7 +710,7 @@ def _pydantic_callable( else: expected_params = None - def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: + def _validate(value: typing.Any, info: pydantic.ValidationInfo) -> Callable: if callable(value) and not isinstance(value, dict): return value if isinstance(value, SynthesizedFunction): @@ -743,7 +742,7 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable: if spliced is not None: evaluation.type_check(*spliced) - g: MutableMapping[str, Any] = {} + g: MutableMapping[str, typing.Any] = {} g.update({k: v for k, v in ctx.items() if k.isidentifier()}) bytecode: types.CodeType = evaluation.compile(module, filename) @@ -858,7 +857,7 @@ def _validate_tool( def _serialize_tool(value: Tool) -> ChatCompletionToolParam: - fields: dict[str, Any] = { + fields: dict[str, typing.Any] = { name: TypeToPydanticType().evaluate(param.annotation) for name, param in inspect.signature(value).parameters.items() } @@ -919,7 +918,7 @@ def _validate_tool_call( f"Unexpected argument {name} for tool {tool.__name__}" ) param = sig.parameters[name] - arg_enc: pydantic.TypeAdapter[Any] = pydantic.TypeAdapter( + arg_enc: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( Encodable[param.annotation] # type: ignore[name-defined] ) decoded_args[name] = arg_enc.validate_python(raw_arg, context=ctx) @@ -937,7 +936,7 @@ def _serialize_tool_call( ctx = info.context or {} encoded_args = {} for k, v in value.bound_args.arguments.items(): - v_enc: pydantic.TypeAdapter[Any] = pydantic.TypeAdapter( + v_enc: pydantic.TypeAdapter[typing.Any] = pydantic.TypeAdapter( Encodable[nested_type(v).value] # type: ignore[misc] ) encoded_args[k] = v_enc.dump_python(v, mode="json", context=ctx) From bb67580855c4be4b13271245cae25d7d784503a2 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 22:42:57 -0400 Subject: [PATCH 046/155] Move Encodable to template.py and fix lint --- docs/source/llm.ipynb | 8 ------ effectful/handlers/llm/__init__.py | 3 +- effectful/handlers/llm/completions.py | 2 +- effectful/handlers/llm/encoding.py | 40 ++------------------------- effectful/handlers/llm/template.py | 36 ++++++++++++++++++++++++ tests/test_handlers_llm_encoding.py | 3 +- tests/test_handlers_llm_evaluation.py | 2 +- tests/test_handlers_llm_provider.py | 2 +- tests/test_handlers_llm_template.py | 2 +- 9 files changed, 45 insertions(+), 53 deletions(-) diff --git a/docs/source/llm.ipynb b/docs/source/llm.ipynb index c738adcae..891e74aa7 100644 --- a/docs/source/llm.ipynb +++ b/docs/source/llm.ipynb @@ -528,11 +528,6 @@ " raise NotHandled\n", "\n", "\n", - "# Verify sub-templates are captured in write_story's lexical context\n", - "assert story_with_moral in write_story.tools.values()\n", - "assert story_funny in write_story.tools.values()\n", - "print(\"Sub-templates available to write_story:\", write_story.tools.keys())\n", - "\n", "with handler(provider):\n", " print(\"=== Story with moral ===\")\n", " print(write_story(\"a curious cat\", \"moral\"))\n", @@ -781,9 +776,6 @@ " raise NotHandled\n", "\n", "\n", - "# Verify sub-templates are captured in write_story's lexical context\n", - "print(\"Sub-templates available to write_story:\", write_multi_chapter_story.tools.keys())\n", - "\n", "with (\n", " handler(RetryLLMHandler()),\n", " handler(provider),\n", diff --git a/effectful/handlers/llm/__init__.py b/effectful/handlers/llm/__init__.py index c8f2e5316..72ebf2d09 100644 --- a/effectful/handlers/llm/__init__.py +++ b/effectful/handlers/llm/__init__.py @@ -47,7 +47,6 @@ observed, logged, or overridden by installing additional handlers. """ -from .encoding import Encodable -from .template import Agent, Template, Tool +from .template import Agent, Encodable, Template, Tool __all__ = ["Agent", "Template", "Tool", "Encodable"] diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index eea2c1af5..2a7f92605 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -41,7 +41,6 @@ _TOOLS_KEY, TYPE_CHECK_ANCHOR_KEY, DecodedToolCall, - Encodable, _callable_type_from_signature, _SynthesisSpec, format_as_content_blocks, @@ -50,6 +49,7 @@ from effectful.handlers.llm.evaluation import ReplSession from effectful.handlers.llm.template import ( Agent, + Encodable, FinalTool, Template, Tool, diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 7419645c7..11c14e83f 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -34,7 +34,7 @@ from PIL import Image import effectful.handlers.llm.evaluation as evaluation -from effectful.handlers.llm.template import Template, Tool +from effectful.handlers.llm.template import Encodable, Template, Tool from effectful.internals.unification import GenericAlias, TypeEvaluator, nested_type from effectful.ops.semantics import fwd, handler from effectful.ops.types import Operation, Term @@ -201,40 +201,6 @@ def result_type(self) -> type[T]: return inspect.signature(self.tool).return_annotation -if typing.TYPE_CHECKING: - type Encodable[T] = typing.Annotated[T, "encoded"] -else: - - class Encodable: - """The type-driven JSON bridge between Python values and the LLM. - - `Encodable[T]` maps a Python type `T` to a Pydantic-compatible type - whose JSON schema and (de)serialization the harness uses to move - values across the model boundary in both directions: - - - **Encoding (Python -> model):** argument and tool-result *values* - spliced into prompts are serialized to JSON via `Encodable[type]`, - so the model sees a faithful, schema-shaped rendering of each value - (including non-text values such as images, emitted as content - blocks). - - **Decoding (model -> Python):** a `Template`'s structured return - value and the arguments of every tool call are validated and - decoded from the model's JSON back into real Python objects through - the same `Encodable[type]` schema, so the value handed to your code - already has the declared type. - - Custom types register their JSON representation with - `TypeToPydanticType`; see - `effectful.handlers.llm.encoding.type_to_encodable_type`. Because the - encoding is derived from the *type*, it is the single source of truth - for both the schema shown to the model and the validation applied to - its output. - """ - - def __class_getitem__(cls, item): - return TypeToPydanticType().evaluate(item) - - class TypeToPydanticType(TypeEvaluator): """Substitute custom types with their Pydantic Annotated equivalents. @@ -654,7 +620,7 @@ def _create_typed_synthesized_function( def _validate_signature_callable( func: Callable, expected_params: list[type] | None, - expected_return: type, + expected_return: type | None, ) -> None: """Validate the function signature from runtime callable after execution. @@ -727,7 +693,7 @@ def _validate(value: typing.Any, info: pydantic.ValidationInfo) -> Callable: ctx = info.context or {} filename = f"" - module: ast.AST = evaluation.parse(encoded.module_code, filename) + module: ast.Module = evaluation.parse(encoded.module_code, filename) # The anchor (Template's underlying function) rides in the decoding context # under TYPE_CHECK_ANCHOR_KEY; absent for tool-argument decoding, whose diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 41ee0beb2..b3a1439f0 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -431,3 +431,39 @@ def simulate(chatbot, advisor) -> str: @functools.cached_property def __history__(self) -> collections.OrderedDict[str, Mapping[str, typing.Any]]: return collections.OrderedDict() + + +if typing.TYPE_CHECKING: + type Encodable[T] = typing.Annotated[T, "encoded"] +else: + + class Encodable: + """The type-driven JSON bridge between Python values and the LLM. + + `Encodable[T]` maps a Python type `T` to a Pydantic-compatible type + whose JSON schema and (de)serialization the harness uses to move + values across the model boundary in both directions: + + - **Encoding (Python -> model):** argument and tool-result *values* + spliced into prompts are serialized to JSON via `Encodable[type]`, + so the model sees a faithful, schema-shaped rendering of each value + (including non-text values such as images, emitted as content + blocks). + - **Decoding (model -> Python):** a `Template`'s structured return + value and the arguments of every tool call are validated and + decoded from the model's JSON back into real Python objects through + the same `Encodable[type]` schema, so the value handed to your code + already has the declared type. + + Custom types register their JSON representation with + `TypeToPydanticType`; see + `effectful.handlers.llm.encoding.type_to_encodable_type`. Because the + encoding is derived from the *type*, it is the single source of truth + for both the schema shown to the model and the validation applied to + its output. + """ + + def __class_getitem__(cls, item): + from effectful.handlers.llm.encoding import TypeToPydanticType + + return TypeToPydanticType().evaluate(item) diff --git a/tests/test_handlers_llm_encoding.py b/tests/test_handlers_llm_encoding.py index 5b3433da3..705dfe367 100644 --- a/tests/test_handlers_llm_encoding.py +++ b/tests/test_handlers_llm_encoding.py @@ -25,7 +25,6 @@ CONTENT_BLOCK_TYPES, TYPE_CHECK_ANCHOR_KEY, DecodedToolCall, - Encodable, SynthesizedFunction, to_content_blocks, ) @@ -33,7 +32,7 @@ RestrictedEvalProvider, UnsafeEvalProvider, ) -from effectful.handlers.llm.template import Tool +from effectful.handlers.llm.template import Encodable, Tool from effectful.internals.unification import nested_type from effectful.ops.semantics import handler from effectful.ops.types import Operation, Term diff --git a/tests/test_handlers_llm_evaluation.py b/tests/test_handlers_llm_evaluation.py index 32e8805ad..ff3465133 100644 --- a/tests/test_handlers_llm_evaluation.py +++ b/tests/test_handlers_llm_evaluation.py @@ -17,7 +17,6 @@ from effectful.handlers.llm.encoding import ( TYPE_CHECK_ANCHOR_KEY, - Encodable, SynthesizedFunction, ) from effectful.handlers.llm.evaluation import ( @@ -32,6 +31,7 @@ from effectful.handlers.llm.evaluation import compile as compile_op from effectful.handlers.llm.evaluation import exec as exec_op from effectful.handlers.llm.evaluation import parse as parse_op +from effectful.handlers.llm.template import Encodable from effectful.ops.semantics import handler # ============================================================================ diff --git a/tests/test_handlers_llm_provider.py b/tests/test_handlers_llm_provider.py index c9026fe6c..7285af20b 100644 --- a/tests/test_handlers_llm_provider.py +++ b/tests/test_handlers_llm_provider.py @@ -45,8 +45,8 @@ call_tool, completion, ) -from effectful.handlers.llm.encoding import Encodable from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.handlers.llm.template import Encodable from effectful.ops.semantics import fwd, handler from effectful.ops.syntax import ObjectInterpretation, implements from effectful.ops.types import NotHandled diff --git a/tests/test_handlers_llm_template.py b/tests/test_handlers_llm_template.py index 754f3e419..449f8fc3b 100644 --- a/tests/test_handlers_llm_template.py +++ b/tests/test_handlers_llm_template.py @@ -1664,8 +1664,8 @@ def test_tool_forward_ref(): from effectful.handlers.llm.completions import ( PythonRepl, ) -from effectful.handlers.llm.encoding import Encodable from effectful.handlers.llm.evaluation import UnsafeEvalProvider +from effectful.handlers.llm.template import Encodable from tests.conftest import offered_tools, template_tools From 8c7b9b0d524de8747c2424a12df0622db5fc6fcb Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 22:46:37 -0400 Subject: [PATCH 047/155] nit --- effectful/handlers/llm/template.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index b3a1439f0..8c48a1b19 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -84,15 +84,6 @@ class FinalTool[**P, T](Tool[P, T]): `effectful.handlers.llm.completions.RetryLLMHandler`). """ - @classmethod - def define(cls, *args, **kwargs) -> "FinalTool[P, T]": - """Define a final tool. - - See `effectful.ops.types.Operation.define` for more information on - the use of `FinalTool.define`. - """ - return typing.cast("FinalTool[P, T]", super().define(*args, **kwargs)) - class Template[**P, T](Tool[P, T]): """A `Template` is a function that is implemented by a large language model. From 4cf864e166ff8269a28da039dba1ef7578ab3bac Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 22:49:12 -0400 Subject: [PATCH 048/155] revert --- effectful/handlers/llm/template.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index 8c48a1b19..b3a1439f0 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -84,6 +84,15 @@ class FinalTool[**P, T](Tool[P, T]): `effectful.handlers.llm.completions.RetryLLMHandler`). """ + @classmethod + def define(cls, *args, **kwargs) -> "FinalTool[P, T]": + """Define a final tool. + + See `effectful.ops.types.Operation.define` for more information on + the use of `FinalTool.define`. + """ + return typing.cast("FinalTool[P, T]", super().define(*args, **kwargs)) + class Template[**P, T](Tool[P, T]): """A `Template` is a function that is implemented by a large language model. From 958415d0e9d09a0e5d19fd2a55e1585532376de3 Mon Sep 17 00:00:00 2001 From: Eli Date: Sun, 19 Jul 2026 22:59:29 -0400 Subject: [PATCH 049/155] minor --- docs/source/llm_examples/rag.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/source/llm_examples/rag.py b/docs/source/llm_examples/rag.py index 498cb40d8..7e4cab7f9 100644 --- a/docs/source/llm_examples/rag.py +++ b/docs/source/llm_examples/rag.py @@ -16,17 +16,6 @@ from effectful.handlers.llm import Agent, Template, Tool -# --------------------------------------------------------------------------- -# Embedding helpers -# --------------------------------------------------------------------------- - - -def get_embedding(text: str, model: str) -> np.ndarray: - """Get an embedding vector for the given text using litellm.""" - response = litellm.embedding(model=model, input=text) - return np.array(response.data[0]["embedding"], dtype=np.float32) - - # --------------------------------------------------------------------------- # Vector index # --------------------------------------------------------------------------- @@ -40,16 +29,21 @@ class VectorIndex: chunks: list[str] = dataclasses.field(default_factory=list) embeddings: list[np.ndarray] = dataclasses.field(default_factory=list) + def get_embedding(self, text: str) -> np.ndarray: + """Get an embedding vector for the given text using litellm.""" + response = litellm.embedding(model=self.model, input=text) + return np.array(response.data[0]["embedding"], dtype=np.float32) + def add(self, text: str) -> None: """Add a text chunk to the index.""" self.chunks.append(text) - self.embeddings.append(get_embedding(text, model=self.model)) + self.embeddings.append(self.get_embedding(text)) def search(self, query: str, top_k: int = 3) -> list[str]: """Return the top-k most similar chunks to the query.""" if not self.embeddings: return [] - query_emb = get_embedding(query, model=self.model) + query_emb = self.get_embedding(query) distances = [float(((emb - query_emb) ** 2).sum()) for emb in self.embeddings] indices = sorted(range(len(distances)), key=lambda i: distances[i]) return [self.chunks[i] for i in indices[:top_k]] From 22bd549cb02a792558f56d974edf0b3bd2a6a0da Mon Sep 17 00:00:00 2001 From: Eli Date: Mon, 20 Jul 2026 00:22:12 -0400 Subject: [PATCH 050/155] repl_history --- effectful/handlers/llm/completions.py | 18 +++++++++++++----- effectful/handlers/llm/encoding.py | 19 +++++++++++-------- effectful/handlers/llm/evaluation.py | 20 -------------------- effectful/handlers/llm/harness.py | 6 ++++-- tests/test_handlers_llm_encoding.py | 17 ++++++++++------- 5 files changed, 38 insertions(+), 42 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index e9c63188f..dd69be089 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -47,7 +47,7 @@ format_as_content_blocks, to_content_blocks, ) -from effectful.handlers.llm.evaluation import ReplSession, _repl_session +from effectful.handlers.llm.evaluation import ReplSession from effectful.handlers.llm.template import ( Agent, Encodable, @@ -873,9 +873,10 @@ class PythonRepl(ObjectInterpretation): single Template invocation. Scoping mirrors how `__history__` is managed for Template calls: `PythonRepl` - handles `Template.__apply__` to introduce a fresh `_repl_session` handler for - the duration of the call, and intercepts `call_assistant` to inject an - `exec_code` Tool routed to that session. The session is therefore introduced and + handles `Template.__apply__` to introduce fresh session-bound handlers (`exec_code`, + `read_lexical_variable`, `repl_history`) for the duration of the call, and intercepts + `call_assistant` to inject an `exec_code` Tool routed to that session. The session is + therefore introduced and eliminated by its own handler, bounded to the Template call by construction -- there is no global registry of sessions, and nested Template calls get their own isolated sessions. @@ -913,6 +914,13 @@ def read_lexical_variable(cls, name: str) -> typing.Any: """ raise NotImplementedError("No handler") + @typing.final + @Operation.define + @classmethod + def repl_history(cls) -> list[str]: + """This REPL session's error-free executed snippets, in order.""" + raise NotImplementedError("No handler") + @implements(call_system) def _call_system(self, template, tool_types=frozenset()): return fwd(template, tool_types=tool_types | {self._ReplInteractionTool}) @@ -929,7 +937,7 @@ def _apply[**P, T]( { self.exec_code: session.exec_code, self.read_lexical_variable: env.get, - _repl_session: lambda _: session, + self.repl_history: lambda: session.prior_snippets, } ): return fwd() diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index f105d0768..7398d8585 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -324,17 +324,20 @@ def validate(value: object, info: pydantic.ValidationInfo) -> types.CodeType: # Type-check the snippet in its execution context, exactly as a synthesized # `Callable` is (see `_pydantic_callable`): when the enclosing Template is the - # type-check anchor in the decode context, splice the accumulated REPL session (the - # `_repl_session` op is in scope during the response decode) plus this snippet into - # the Template body and check it. A type error raises here -> the tool-call decode - # fails -> `RetryLLMHandler` retries, so ill-typed code never reaches `runcode`. + # type-check anchor in the decode context, splice the accumulated REPL session + # (`PythonRepl.repl_history` returns the prior snippets of the session in scope) + # plus this snippet into the Template body and check it. A type error raises here + # -> the tool-call decode fails -> `RetryLLMHandler` retries, so ill-typed code + # never reaches `runcode`. ctx = info.context or {} anchor = ctx.get(REPL_ANCHOR_KEY) if anchor is not None: - # Pass an empty env (not `ctx`): the managed session ignores it, and a fresh - # fallback session must not be seeded from the decode context (which holds tool - # names and the anchor key). The decoder only reads `prior_snippets`. - prior = evaluation._repl_session({}).prior_snippets + # Imported lazily (not at module load) to avoid an import cycle: `completions` + # imports this module. `repl_history` returns the managed session's prior + # snippets, or `[]` when no REPL is in scope. + from effectful.handlers.llm.completions import PythonRepl + + prior = PythonRepl.repl_history() checked = evaluation._splice_repl(prior, value, anchor) if checked is not None: evaluation.type_check(*checked, lenient=True) diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index c6c33680b..c39cde71a 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -28,7 +28,6 @@ from RestrictedPython.PrintCollector import PrintCollector from effectful.ops.syntax import ObjectInterpretation, defop, implements -from effectful.ops.types import Operation @defop @@ -728,22 +727,3 @@ def exec_code(self, code: types.CodeType) -> str: ): self.runcode(code) return self.stdout.getvalue()[out_start:] + self.stderr.getvalue()[err_start:] - - -@Operation.define -def _repl_session( - env: collections.abc.MutableMapping[str, typing.Any], -) -> "ReplSession": - """Return the REPL session for the current Template call, seeded from `env`. - - `PythonRepl` (in completions.py) installs a fresh handler for this inside each - `Template.__apply__` (mirroring how `__history__` is managed), giving the session a - lifetime of exactly one Template call. Outside such a scope there is no managed - session, so this falls back to a fresh one -- e.g. when tools are listed outside a - Template call, or when a code object is decoded with no REPL in scope. - - Defined here (not with `PythonRepl`) so the `Encodable[CodeType]` decoder can reach the - session -- and its accumulated `prior_snippets` -- at decode time without importing - `completions` (which would be a cycle). - """ - return ReplSession(env) diff --git a/effectful/handlers/llm/harness.py b/effectful/handlers/llm/harness.py index 81e5974ca..5fde40d11 100644 --- a/effectful/handlers/llm/harness.py +++ b/effectful/handlers/llm/harness.py @@ -192,10 +192,12 @@ def main(argv: list[str] | None = None) -> None: render=ns.render, dump_system_prompt=ns.dump_system_prompt, tool_choice=ns.tool_choice, - api_base="http://localhost:8030/v1" + api_base=os.environ.get("DS4_OPENAI_API_BASE", None) + if ns.model == "openai/deepseek-v4-flash" + else None, + api_key=os.environ.get("DS4_OPENAI_API_KEY", None) if ns.model == "openai/deepseek-v4-flash" else None, - api_key="" if ns.model == "openai/deepseek-v4-flash" else None, ): runpy.run_path(ns.script, run_name="__main__") diff --git a/tests/test_handlers_llm_encoding.py b/tests/test_handlers_llm_encoding.py index 705dfe367..c74d6fef7 100644 --- a/tests/test_handlers_llm_encoding.py +++ b/tests/test_handlers_llm_encoding.py @@ -25,7 +25,6 @@ CONTENT_BLOCK_TYPES, TYPE_CHECK_ANCHOR_KEY, DecodedToolCall, - SynthesizedFunction, to_content_blocks, ) from effectful.handlers.llm.evaluation import ( @@ -789,11 +788,17 @@ def _int_pair_anchor() -> Callable[[int, int], int]: # Callable error cases: (type, ctx, source, exc_type, anchor) +# +# Sources are passed as raw ``{"module_code": ...}`` dicts, not pre-built +# ``SynthesizedFunction`` instances: structurally-invalid code (e.g. a non-function +# last statement) is rejected by ``SynthesizedFunction``'s own field validator, so +# building it eagerly here would raise at collection. A dict defers that validation +# to the decoder (``model_validate``), which is the real path an LLM's JSON takes. CALLABLE_ERROR_CASES = [ pytest.param( Callable[..., int], {}, - SynthesizedFunction(module_code="x = 42"), + {"module_code": "x = 42"}, ValueError, None, id="non-function-last-stmt", @@ -801,7 +806,7 @@ def _int_pair_anchor() -> Callable[[int, int], int]: pytest.param( Callable[[int, int], int], {}, - SynthesizedFunction(module_code="def add(a: int) -> int:\n return a"), + {"module_code": "def add(a: int) -> int:\n return a"}, ValueError, None, id="wrong-param-count", @@ -809,9 +814,7 @@ def _int_pair_anchor() -> Callable[[int, int], int]: pytest.param( Callable[[int, int], int], {}, - SynthesizedFunction( - module_code="def add(a: int, b: int) -> str:\n return str(a + b)" - ), + {"module_code": "def add(a: int, b: int) -> str:\n return str(a + b)"}, TypeError, _int_pair_anchor, id="wrong-return-type", @@ -819,7 +822,7 @@ def _int_pair_anchor() -> Callable[[int, int], int]: pytest.param( Callable[[int, int], int], {}, - SynthesizedFunction(module_code="def add(a: int, b: int):\n return a + b"), + {"module_code": "def add(a: int, b: int):\n return a + b"}, ValueError, None, id="missing-return-annotation", From a6236375a5579fa1cc4e864895aa55dd43905cd2 Mon Sep 17 00:00:00 2001 From: Eli Date: Thu, 23 Jul 2026 11:03:40 -0400 Subject: [PATCH 051/155] cleanup --- effectful/handlers/llm/encoding.py | 151 ++++++++++------------------- 1 file changed, 52 insertions(+), 99 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index c017b30b8..c229a7dc0 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -15,7 +15,6 @@ from collections.abc import ( Callable, Mapping, - MutableMapping, ) import litellm @@ -584,7 +583,6 @@ def _validate_module_code(cls, value: str) -> str: f"got {type(last_stmt).__name__}" ) - # Check that the function has type annotations for all parameters for arg in last_stmt.args.args: if arg.annotation is None: raise ValueError( @@ -592,20 +590,17 @@ def _validate_module_code(cls, value: str) -> str: f"parameter '{arg.arg}' is missing an annotation" ) - # Check that the function has a return type annotation if last_stmt.returns is None: raise ValueError( "decode() requires the function to have a return type annotation" ) - # no __future__ imports are allowed for stmt in module.body: if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__": raise ValueError( "decode() does not allow __future__ imports in the module code" ) - # no star imports are allowed for stmt in module.body: if isinstance(stmt, ast.ImportFrom) and stmt.names: for alias in stmt.names: @@ -616,51 +611,52 @@ def _validate_module_code(cls, value: str) -> str: return value + @classmethod + def _create_typed_synthesized_function( + cls, callable_type: type[Callable] + ) -> type[typing.Self]: + """Create a SynthesizedFunction subclass with type signature in the model description. -def _create_typed_synthesized_function( - callable_type: type[Callable], -) -> type[SynthesizedFunction]: - """Create a SynthesizedFunction subclass with type signature in the model description. + Uses pydantic.create_model to ensure the description is included in the JSON schema + sent to the LLM, informing it of the expected function signature. + """ + if not typing.get_args(callable_type): + type_signature = "Callable" + # Callable[[arg1, arg2, ...], return_type] + elif len(typing.get_args(callable_type)) >= 2: + param_types = typing.get_args(callable_type)[0] + return_type = typing.get_args(callable_type)[-1] + + if param_types is ...: + params_str = "..." + elif isinstance(param_types, list | tuple): + params_str = ", ".join( + getattr(t, "__name__", str(t)) for t in param_types + ) + else: + params_str = str(param_types) - Uses pydantic.create_model to ensure the description is included in the JSON schema - sent to the LLM, informing it of the expected function signature. - """ - if not typing.get_args(callable_type): - type_signature = "Callable" - # Callable[[arg1, arg2, ...], return_type] - elif len(typing.get_args(callable_type)) >= 2: - param_types = typing.get_args(callable_type)[0] - return_type = typing.get_args(callable_type)[-1] - - if param_types is ...: - params_str = "..." - elif isinstance(param_types, list | tuple): - params_str = ", ".join(getattr(t, "__name__", str(t)) for t in param_types) + return_str = getattr(return_type, "__name__", str(return_type)) + type_signature = f"Callable[[{params_str}], {return_str}]" else: - params_str = str(param_types) - - return_str = getattr(return_type, "__name__", str(return_type)) - type_signature = f"Callable[[{params_str}], {return_str}]" - else: - type_signature = str(callable_type) + type_signature = str(callable_type) - return pydantic.create_model( - "TypedSynthesizedFunction", - __base__=SynthesizedFunction, - __doc__=f"""Python function with signature {type_signature}""", - ) + return pydantic.create_model( + "TypedSynthesizedFunction", + __base__=cls, + __doc__=f"""Python function with signature {type_signature}""", + ) -def _validate_signature_callable( - func: Callable, - expected_params: list[type] | None, - expected_return: type | None, -) -> None: +def _validate_signature_callable(func: Callable, ty: type[Callable]) -> None: """Validate the function signature from runtime callable after execution. The synthesized function must have type annotations for parameters and return type. """ sig = inspect.signature(func) + type_args = typing.get_args(ty) + expected_params = type_args[0] if type_args else None + expected_return = type_args[-1] if type_args else None if expected_params is not None: actual_params = list(sig.parameters.values()) @@ -684,62 +680,23 @@ def _validate_signature_callable( @TypeToPydanticType.register(Callable) def _pydantic_callable( - callable_type: typing.Any, metadata: _SynthesisSpec | None = None + ty: typing.Any, metadata: _SynthesisSpec | None = None ) -> typing.Any: - """Create a Pydantic-compatible Annotated type for a parameterized Callable. + """Create a Pydantic-compatible Annotated type for a parameterized Callable.""" - Usage: PydanticCallable(Callable[[int, str], bool]) - """ - type_args = typing.get_args(callable_type) - - if not type_args: - typed_enc = _create_typed_synthesized_function(Callable[..., typing.Any]) # type: ignore[arg-type] - expected_params = None - expected_return = None - else: - if len(type_args) < 2: - raise pydantic.errors.PydanticSchemaGenerationError( - f"Callable type signature incomplete: {callable_type}. " - "Expected Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType]." - ) - if type_args[1] is None: - raise pydantic.errors.PydanticSchemaGenerationError( - "Cannot decode/synthesize callable without a concrete type signature. " - "Use Callable[[ParamTypes...], ReturnType] or Callable[..., ReturnType] " - "with a concrete return type (not Any)." - ) - param_types, expected_return = type_args[0], type_args[1] - typed_enc = _create_typed_synthesized_function(callable_type) - if param_types is not ... and isinstance(param_types, list | tuple): - expected_params = list(param_types) - else: - expected_params = None - - def _validate(value: typing.Any, info: pydantic.ValidationInfo) -> Callable: - if callable(value) and not isinstance(value, dict): - return value - if isinstance(value, SynthesizedFunction): - encoded = value - elif isinstance(value, dict): - encoded = typed_enc.model_validate(value) - elif isinstance(value, str): - encoded = typed_enc.model_validate_json(value) - else: - raise ValueError( - f"Expected callable, SynthesizedFunction dict, or JSON string, " - f"got {type(value)}" - ) + typed_enc = SynthesizedFunction._create_typed_synthesized_function( + Callable[..., typing.Any] if not typing.get_args(ty) else ty # type: ignore[arg-type] + ) + def _validate( + value: SynthesizedFunction | dict, info: pydantic.ValidationInfo + ) -> Callable: + if isinstance(value, dict): + value = typed_enc.model_validate(value) ctx = info.context or {} - filename = f"" - module: ast.Module = evaluation.parse(encoded.module_code, filename) - - # The anchor (Template's underlying function) rides in the decoding context - # under TYPE_CHECK_ANCHOR_KEY; absent for tool-argument decoding, whose - # synthesized Callables are contracted by the tool param's type, not the - # Template's return type, so the Template anchor doesn't apply. When - # present, the code is spliced into the Template body, so first reject - # constructs illegal once nested (star / `__future__` imports), then check. + filename = f"" + module: ast.Module = evaluation.parse(value.module_code, filename) + anchor = ctx.get(TYPE_CHECK_ANCHOR_KEY) if anchor is not None: evaluation.scan_non_nestable(module) @@ -747,14 +704,13 @@ def _validate(value: typing.Any, info: pydantic.ValidationInfo) -> Callable: if spliced is not None: evaluation.type_check(*spliced) - g: MutableMapping[str, typing.Any] = {} - g.update({k: v for k, v in ctx.items() if k.isidentifier()}) - bytecode: types.CodeType = evaluation.compile(module, filename) + + g: dict[str, typing.Any] = {k: v for k, v in ctx.items() if k.isidentifier()} evaluation.exec(bytecode, g) result = g[module.body[-1].name] # type: ignore - _validate_signature_callable(result, expected_params, expected_return) + _validate_signature_callable(result, ty) if metadata is not None: if metadata._class_template is not None: @@ -791,9 +747,6 @@ def _doctest_apply(op, *args, **kwargs): return result def _serialize(value: Callable) -> dict: - if not callable(value): - raise TypeError(f"Expected callable, got {type(value)}") - try: source = inspect.getsource(value) except (OSError, TypeError): @@ -822,7 +775,7 @@ def _serialize(value: Callable) -> dict: return typed_enc(module_code=stub_code).model_dump() return typing.Annotated[ - callable_type, + ty, pydantic.PlainValidator(_validate), pydantic.PlainSerializer(_serialize), # Distinct schemas per direction. Validation (the model *produces* a From 62ebd45b6994a8496359cd8ab0cf93893a59f2f5 Mon Sep 17 00:00:00 2001 From: Eli Date: Thu, 23 Jul 2026 15:49:20 -0400 Subject: [PATCH 052/155] force type checking of submit_solution --- effectful/handlers/llm/completions.py | 36 -------------------- effectful/handlers/llm/encoding.py | 48 ++++++--------------------- 2 files changed, 11 insertions(+), 73 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index dd69be089..d9e038eaf 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -752,42 +752,6 @@ class SynthesizeAndCall(ObjectInterpretation): be compiled and executed. """ - # TODO FIX THIS!!! - # KNOWN BUG -- the synthesized function is NOT type-checked against the - # Template. - # - # The docstring above (and PR #706) promise that the synthesized code is - # "type-checked": `decode` (encoding.py) splices the generated function into - # the Template's own module source and runs mypy on it, so the body is checked - # in its real lexical scope. That splice only happens when the enclosing - # Template's underlying function rides in the decode context under - # `TYPE_CHECK_ANCHOR_KEY` (see `decode`, encoding.py, and `_serialize`'s - # `if anchor is not None:` guard). - # - # That anchor is threaded on exactly ONE of the two decode paths in - # `call_assistant`: - # - Direct result (model answers with content): decoded with - # `context={**env, TYPE_CHECK_ANCHOR_KEY: anchor}` -> spliced + mypy-checked. - # - Tool call (model calls a tool): decoded with `context=env` -- no anchor - # -> splice skipped. - # - # `submit_solution` is a `FinalTool`, so the model always reaches it via the - # *tool-call* path. Its `implementation` argument is therefore decoded WITHOUT - # the anchor, so the mypy splice never runs for it. The synthesized function -- - # the whole product of this handler -- is only validated structurally - # (`_validate_signature_callable`: parameter count/return match), compiled, and - # executed (plus its own doctests). Its body escapes the type check entirely. - # - # The tool-call path drops the anchor deliberately: a Callable passed to an - # *arbitrary* tool is contracted by that tool's parameter type, not by the - # Template's body, so anchoring it to the Template would be wrong. But - # `submit_solution` is special -- its Callable argument *is* the Template's - # body -- so for this FinalTool the anchor should be the enclosing Template's - # `__default__`. The fix is to thread that anchor into the decode context when - # decoding a synthesis FinalTool's argument (rather than using the generic - # `context=env` tool-call path), so `submit_solution`'s implementation is - # spliced into the Template and mypy-checked like a directly-returned result. - @typing.final class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): """## Code synthesis diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index c229a7dc0..4f0ce5e95 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -304,7 +304,9 @@ def _pydantic_type_code(ty): `linecache`, which carries everything the source string did. """ - def validate(value: object, info: pydantic.ValidationInfo) -> types.CodeType: + def validate( + value: types.CodeType | str, info: pydantic.ValidationInfo + ) -> types.CodeType: if isinstance(value, types.CodeType): return value if not isinstance(value, str): @@ -648,36 +650,6 @@ def _create_typed_synthesized_function( ) -def _validate_signature_callable(func: Callable, ty: type[Callable]) -> None: - """Validate the function signature from runtime callable after execution. - - The synthesized function must have type annotations for parameters and return type. - """ - sig = inspect.signature(func) - type_args = typing.get_args(ty) - expected_params = type_args[0] if type_args else None - expected_return = type_args[-1] if type_args else None - - if expected_params is not None: - actual_params = list(sig.parameters.values()) - if len(actual_params) != len(expected_params): - params_str = ", ".join( - getattr(t, "__name__", str(t)) for t in expected_params - ) - return_str = getattr(expected_return, "__name__", str(expected_return)) - raise ValueError( - f"synthesized function must match Callable[[{params_str}], {return_str}] " - f"-- exactly {len(expected_params)} parameter(s) -- " - f"but got {len(actual_params)}" - ) - - actual_return = sig.return_annotation - if actual_return is inspect.Parameter.empty: - raise ValueError( - "decode() requires synthesized function to have a return type annotation" - ) - - @TypeToPydanticType.register(Callable) def _pydantic_callable( ty: typing.Any, metadata: _SynthesisSpec | None = None @@ -697,20 +669,22 @@ def _validate( filename = f"" module: ast.Module = evaluation.parse(value.module_code, filename) - anchor = ctx.get(TYPE_CHECK_ANCHOR_KEY) - if anchor is not None: - evaluation.scan_non_nestable(module) - spliced = evaluation.splice_into_source(module, anchor) + if ctx.get(TYPE_CHECK_ANCHOR_KEY) is not None: + spliced = evaluation.splice_into_source(module, ctx[TYPE_CHECK_ANCHOR_KEY]) if spliced is not None: evaluation.type_check(*spliced) + elif ctx.get(REPL_ANCHOR_KEY) is not None: + spliced = evaluation._splice_repl( + [], value.module_code, ctx[REPL_ANCHOR_KEY] + ) + if spliced is not None: + evaluation.type_check(*spliced, lenient=True) bytecode: types.CodeType = evaluation.compile(module, filename) g: dict[str, typing.Any] = {k: v for k, v in ctx.items() if k.isidentifier()} evaluation.exec(bytecode, g) - result = g[module.body[-1].name] # type: ignore - _validate_signature_callable(result, ty) if metadata is not None: if metadata._class_template is not None: From fda9049264a4f2ceb687288a9fab521660bb3f37 Mon Sep 17 00:00:00 2001 From: Eli Date: Thu, 23 Jul 2026 18:49:05 -0400 Subject: [PATCH 053/155] remove trivial majority_vote --- docs/source/llm_examples/majority_vote.py | 71 ----------------------- 1 file changed, 71 deletions(-) delete mode 100644 docs/source/llm_examples/majority_vote.py diff --git a/docs/source/llm_examples/majority_vote.py b/docs/source/llm_examples/majority_vote.py deleted file mode 100644 index 31041a963..000000000 --- a/docs/source/llm_examples/majority_vote.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Majority voting ensemble. - -Demonstrates: -- Running the same template multiple times and taking a majority vote -- ``collections.Counter`` for tallying responses -""" - -import argparse -import collections -import collections.abc -import enum - -from effectful.handlers.llm import Template - -# --------------------------------------------------------------------------- -# Template -# --------------------------------------------------------------------------- - - -class Answer(enum.StrEnum): - yes = "yes" - no = "no" - maybe = "maybe" - - -@Template.define -def yes_or_no(question: str) -> Answer: - """ - Answer the following yes/no/maybe question: {question} - """ - - -# --------------------------------------------------------------------------- -# Majority vote -# --------------------------------------------------------------------------- - - -def majority_vote[Q]( - oracle: collections.abc.Callable[[Q], Answer], query: Q, voters: int = 3 -) -> tuple[Answer, int]: - """Call ``oracle(query)`` multiple times and return the most common answer.""" - counter = collections.Counter(oracle(query) for _ in range(voters)) - return counter.most_common(1)[0] - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--num-voters", type=int, default=3, help="Number of voters for majority vote" - ) - parser.add_argument( - "--question", - type=str, - default="Is Paris the capital of France?", - help="Yes/no question to ask", - ) - args = parser.parse_args() - - answer, count = majority_vote(yes_or_no, args.question, voters=args.num_voters) - print( - f"Question: {args.question}\nAnswer: {answer} (voted {count}/{args.num_voters})" - ) - - -if __name__ == "__main__": - main() From 867939d4d53bd85819c3a105dd0188c51f04b14d Mon Sep 17 00:00:00 2001 From: Eli Date: Thu, 23 Jul 2026 23:05:27 -0400 Subject: [PATCH 054/155] many fixes --- effectful/handlers/llm/completions.py | 30 +- effectful/handlers/llm/encoding.py | 510 +++++++++++++++++++------- effectful/handlers/llm/evaluation.py | 208 +++++++++-- tests/test_handlers_llm_evaluation.py | 31 +- 4 files changed, 584 insertions(+), 195 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index d9e038eaf..ee84d70ac 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -42,8 +42,9 @@ REPL_ANCHOR_KEY, TYPE_CHECK_ANCHOR_KEY, DecodedToolCall, + MethodTemplateBody, + TemplateBody, _callable_type_from_signature, - _SynthesisSpec, format_as_content_blocks, to_content_blocks, ) @@ -242,7 +243,7 @@ def call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), - anchor: types.FunctionType | None = None, + anchor: "Template | None" = None, ) -> AssistantResult[T]: """Low-level LLM request. Handlers may log/modify requests and delegate via fwd(). @@ -692,7 +693,7 @@ def _call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), - anchor: types.FunctionType | None = None, + anchor: "Template | None" = None, ) -> AssistantResult[T]: readers: set[Tool] = set(tools) taken = {t.__name__ for t in tools} @@ -781,24 +782,25 @@ def define( template: Template[..., T], bound_args: inspect.BoundArguments, ) -> FinalTool[[collections.abc.Callable[..., T]], T]: - # Synthesize a drop-in syntactic replacement for the Template body, so the - # function carries the Template's full signature -- including `self` for - # Agent-method Templates (whose `__default__` is a bound method). if isinstance(template.__default__, types.MethodType): signature = inspect.signature(template.__default__.__func__) args, kwargs = ( (template.__default__.__self__,) + bound_args.args, bound_args.kwargs, ) + body_type = MethodTemplateBody[ # type: ignore + typing.get_args(_callable_type_from_signature(signature)) + ] + return_type = signature.return_annotation else: signature = inspect.signature(template) args, kwargs = bound_args.args, bound_args.kwargs + body_type = TemplateBody[ # type: ignore + typing.get_args(_callable_type_from_signature(signature)) + ] + return_type = signature.return_annotation - callable_type = _callable_type_from_signature(signature) - callable_type = typing.Annotated[callable_type, _SynthesisSpec(template)] # type: ignore - return_type = signature.return_annotation - - def submit_solution(implementation: callable_type) -> return_type: # type: ignore + def submit_solution(implementation: body_type) -> return_type: # type: ignore """ Submit your final answer as a Python function implementing the task. The function must have the required signature; it is applied to the @@ -912,7 +914,7 @@ def _call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), - anchor: types.FunctionType | None = None, + anchor: "Template | None" = None, ) -> AssistantResult[T]: return fwd( env, @@ -987,7 +989,7 @@ def _call_assistant[T]( env: collections.abc.Mapping[str, typing.Any], response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), - anchor: types.FunctionType | None = None, + anchor: "Template | None" = None, ) -> AssistantResult[T]: _message_sequence = _get_history().copy() @@ -1518,7 +1520,7 @@ def _call[**P, T]( env, template.__signature__.return_annotation, _tools_in_scope(env) - {template}, - anchor=template.__default__, + anchor=template, ) if tool_calls: for tool_call in tool_calls: diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 4f0ce5e95..a14545710 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -44,18 +44,21 @@ # Deliberately not a valid Python identifier, so it can never collide with a # lexical variable name sharing the context (e.g. a reader named after its var). _TOOLS_KEY: typing.Literal["$TOOLS"] = "$TOOLS" -# Reserved key under which the type-check anchor (the enclosing Template's -# underlying function) rides in the Pydantic decoding context, alongside the -# lexical environment. `decode` reads it to type-check a synthesized function -# against the Template's source; absent (tool-argument decoding) means skip. -# Deliberately not a valid identifier so `LexicalReaders` skips it (no tool leak) -# and it can never collide with a lexical name. +# Reserved key under which the type-check anchor -- the enclosing `Template` +# itself -- rides in the Pydantic decoding context, alongside the lexical +# environment. `decode` reads it to type-check a synthesized function against the +# Template's source (recovered from the Template via `inspect.unwrap`); absent +# (tool-argument decoding) means skip. Deliberately not a valid identifier so +# `LexicalReaders` skips it (no tool leak) and it can never collide with a lexical +# name. TYPE_CHECK_ANCHOR_KEY = "" -# Type-check anchor for REPL `exec_code` snippets, separate from the Callable/result -# synthesis anchor (TYPE_CHECK_ANCHOR_KEY): the two decoders check against different -# contracts -- a REPL snippet against the Template body, a synthesized Callable tool -# argument against its own parameter type. +# Anchor for REPL `exec_code` snippets and synthesized tool arguments (including a +# `TemplateBody`), separate from the structured-output-result synthesis anchor +# (TYPE_CHECK_ANCHOR_KEY): the two decoders check against different contracts -- a +# REPL snippet or a `TemplateBody` against the Template body, a synthesized general +# `Callable` tool argument against its own parameter type. Both keys carry the +# enclosing `Template`. REPL_ANCHOR_KEY = "" CONTENT_BLOCK_TYPES: frozenset[str] = frozenset( @@ -227,14 +230,6 @@ def register(cls, *args, **kwargs): return cls._registry.register(*args, **kwargs) def evaluate(self, ty): - if typing.get_origin(ty) is typing.Annotated and any( - isinstance(m, _SynthesisSpec) for m in ty.__metadata__ - ): - inner, *meta = typing.get_args(ty) - return self._registry.dispatch(typing.get_origin(inner) or inner)( - inner, *meta - ) - app = super().evaluate(ty) origin = typing.get_origin(app) # Only dispatch on regular types. Special forms (Literal, Annotated, @@ -338,8 +333,13 @@ def validate( # snippets, or `[]` when no REPL is in scope. from effectful.handlers.llm.completions import PythonRepl + # Prepend the already-run (type-clean) session snippets so their bindings + # resolve; `value` is the current snippet. The whole cumulative body is + # spliced and checked. prior = PythonRepl.repl_history() - checked = evaluation._splice_repl(prior, value, anchor) + prior_src = "".join(s if s.endswith("\n") else s + "\n" for s in prior) + session = ast.parse(prior_src + value) + checked = evaluation.splice_repl_code_into_body(session, anchor) if checked is not None: evaluation.type_check(*checked, lenient=True) try: @@ -522,31 +522,71 @@ def _callable_type_from_signature( return collections.abc.Callable[param_types, return_type] # type: ignore -@dataclasses.dataclass(frozen=True) -class _SynthesisSpec[T]: - template: Template[..., T] +class TemplateBody: + """The synthesized *body* of a `Template`, as opposed to a general `Callable`. - @property - def _class_template(self) -> Template[..., T] | None: - if isinstance(self.template.__default__, types.MethodType): - return self.template.__default__.__func__.__wrapped__ # type: ignore[attr-defined] - else: - return None + Used only as the type of `submit_solution`'s ``implementation`` parameter (see + `effectful.handlers.llm.completions.SynthesizeAndCall`). A `TemplateBody[[P], + R]` carries the Template's parameter and return types exactly like a + `Callable`, but gets its own `TypeToPydanticType` case (`_pydantic_template_body`) + so the synthesized function is type-checked against the enclosing Template's + source and its doctests run with self/recursive calls routed to the synthesized + implementation. The enclosing `Template` is recovered from the decode context + (the ``anchor``), so no state rides on the type itself. + """ - def _method_instance(self, other: Template) -> typing.Any | None: - """The instance ``op`` is bound to, if ``op`` is this synthesized - Agent-method on *some* instance; otherwise ``None``. - """ - if ( - self._class_template is not None - and _SynthesisSpec(other)._class_template is self._class_template - ): - return other.__default__.__self__ # type: ignore[attr-defined] - else: - return None + def __class_getitem__(cls, item): + return types.GenericAlias(cls, item) + + +class MethodTemplateBody(TemplateBody): + """A `TemplateBody` for an *instance-method* Template. + + Carries the method/free distinction on the type's origin (context-free schema + generation reads it) so `submit_solution`'s description names the leading + receiver ``self`` and the receiver is exempt from the annotation requirement -- + the model no longer has to reverse-engineer that the first parameter is ``self``. + The Template's real signature (which includes the receiver) remains the + type-check contract; see `splice_template_body`. + """ + + +def _class_template_of(op: typing.Any) -> typing.Any | None: + """The class-level `Template` underlying an Agent-method Template ``op``. + + Returns ``None`` for a free-function template (whose ``__default__`` is a plain + function rather than a bound method). + """ + default = getattr(op, "__default__", None) + if isinstance(default, types.MethodType): + return default.__func__.__wrapped__ # type: ignore[attr-defined] + return None + + +def _method_instance(op: typing.Any, class_template: typing.Any) -> typing.Any | None: + """The instance ``op`` is bound to, if ``op`` is ``class_template`` on *some* + instance; otherwise ``None``. + """ + if class_template is not None and _class_template_of(op) is class_template: + return op.__default__.__self__ + return None + + +# The *serialization* view of a synthesized callable: the shape the model reads +# when a function is handed to it as a value (e.g. a tool's return) -- just the +# source, with none of the synthesis instructions the `SynthesizedFunction` subtype +# carries for the generation direction. Its JSON schema (docstring included, since +# pydantic renders it as the schema `description`) is the ``mode="serialization"`` +# schema of every synthesized-callable encoding, so keep the docstring model-facing. +class EncodedFunction(pydantic.BaseModel): + """A function, encoded as a string of its complete Python source.""" + + module_code: str = pydantic.Field( + ..., description="Python source defining the function." + ) -class SynthesizedFunction(pydantic.BaseModel): +class SynthesizedFunction(EncodedFunction): """ Structured output for function synthesis. """ @@ -615,12 +655,16 @@ def _validate_module_code(cls, value: str) -> str: @classmethod def _create_typed_synthesized_function( - cls, callable_type: type[Callable] + cls, callable_type: type[Callable], *, method: bool = False ) -> type[typing.Self]: """Create a SynthesizedFunction subclass with type signature in the model description. Uses pydantic.create_model to ensure the description is included in the JSON schema sent to the LLM, informing it of the expected function signature. + + When ``method``, the leading parameter is the instance receiver: it is + rendered as ``self`` (rather than by its type) so the model writes it + explicitly, and a note records that it may be left unannotated. """ if not typing.get_args(callable_type): type_signature = "Callable" @@ -632,9 +676,12 @@ def _create_typed_synthesized_function( if param_types is ...: params_str = "..." elif isinstance(param_types, list | tuple): - params_str = ", ".join( - getattr(t, "__name__", str(t)) for t in param_types - ) + names = [getattr(t, "__name__", str(t)) for t in param_types] + # The receiver's type is uninformative (it is the Agent class); name + # it `self` so the model reproduces the parameter instead of guessing. + if method and names: + names[0] = "self" + params_str = ", ".join(names) else: params_str = str(param_types) @@ -643,19 +690,172 @@ def _create_typed_synthesized_function( else: type_signature = str(callable_type) + doc = f"Python function with signature {type_signature}" + if method: + doc += ( + "\n\nThis implements an instance method: the first parameter is the " + "instance receiver `self`. Include it as the first parameter; you may " + "leave it unannotated." + ) + return pydantic.create_model( "TypedSynthesizedFunction", __base__=cls, - __doc__=f"""Python function with signature {type_signature}""", + __doc__=doc, + ) + + +class SynthesizedTemplateBody(SynthesizedFunction): + """Structured output for synthesizing a `Template`'s body (`submit_solution`). + + Decoded through `_pydantic_template_body`: the function is type-checked against + the enclosing Template's source and its doctests are run with self/recursive + calls routed to the synthesized implementation. + + Unlike `SynthesizedFunction`, the parameter and return *annotations* are not + required: a Template body is type-checked against the Template's own signature + (see `splice_template_body`), so the model may omit or vary them -- in + particular it need not annotate the ``self`` receiver of an instance-method + Template. + """ + + module_code: str = pydantic.Field( + ..., + description=textwrap.dedent(""" + The complete Python source for the function implementing the Template. + Write it as a drop-in implementation with the Template's signature (shown + in ... and in the Template spec). The code MUST + satisfy the following constraints, or it will fail validation: + + + 1. The code MUST be one complete syntactically valid Python module. + 2. The code MUST NOT use star imports or ``__future__`` imports. + 3. The function definition MUST be the LAST statement - do not add any code after it. + 4. Write the function with the Template's signature (see the Template spec); + parameter and return annotations are optional. + 5. You may include doctest examples (lines starting with >>>) inside the function's + docstring to demonstrate and verify its behavior; these examples are run as tests, + with calls to the Template routed to this implementation. + + """), + ) + + @pydantic.field_validator("module_code") + @classmethod + def _validate_module_code(cls, value: str) -> str: + # Structural checks only. Parameter/return annotations are intentionally NOT + # required: a TemplateBody is type-checked against the Template's real + # signature (`splice_template_body`), which already carries them -- so the + # model may omit them, and need not annotate the `self` receiver. + module: ast.AST = ast.parse(value) + if not isinstance(module, ast.Module) or not module.body: + raise ValueError( + "decode() requires module code with at least one statement." + ) + last_stmt = module.body[-1] + if not isinstance(last_stmt, ast.FunctionDef): + raise ValueError( + f"decode() requires the last statement to be a function definition, " + f"got {type(last_stmt).__name__}" + ) + for stmt in module.body: + if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__": + raise ValueError( + "decode() does not allow __future__ imports in the module code" + ) + if isinstance(stmt, ast.ImportFrom) and any( + alias.name == "*" for alias in stmt.names + ): + raise ValueError( + "decode() does not allow star imports in the module code" + ) + return value + + +def _serialize_synthesized( + value: Callable, typed_enc: type[SynthesizedFunction] +) -> dict: + """Encode a callable back to its ``module_code`` form (source, or a stub).""" + try: + source = inspect.getsource(value) + except (OSError, TypeError): + source = None + + if source: + return typed_enc(module_code=textwrap.dedent(source)).model_dump() + + name = getattr(value, "__name__", None) + docstring = inspect.getdoc(value) + if name is None or docstring is None: + raise ValueError( + f"Cannot encode callable {value}: no source code and no __name__ or docstring" ) + try: + sig_str = str(inspect.signature(value)) + except (ValueError, TypeError): + sig_str = "(...)" + + stub_code = f'''def {name}{sig_str}: + """{docstring}""" + ... +''' + return typed_enc(module_code=stub_code).model_dump() + + +def _synthesize_callable( + module_code: str, + ctx: Mapping, + *, + template_body: bool, +) -> tuple[Callable, dict[str, typing.Any]]: + """Parse, type-check, compile and exec a synthesized module, returning the + function it defines and the exec namespace. + + The code is type-checked against the enclosing Template's source when an + ``anchor`` is present in ``ctx``. ``template_body`` selects the splice: a + `TemplateBody` (submit_solution) is spliced as the Template's own body; a + general `Callable` uses the strict result splice (`splice_into_source`) when it + is a structured-output result, else the lenient REPL splice. + """ + filename = f"" + module: ast.Module = evaluation.parse(module_code, filename) + + if template_body: + anchor = ctx.get(TYPE_CHECK_ANCHOR_KEY) or ctx.get(REPL_ANCHOR_KEY) + if anchor is not None: + # Check the synthesized function *as the Template's body*, strictly: it + # is the final answer, so -- unlike incrementally-built REPL code -- it + # must honor the Template's declared types and gets no redefinition slack + # (no name reuse with a new type, no duplicate definitions). + spliced = evaluation.splice_template_body(module, anchor) + if spliced is not None: + evaluation.type_check(*spliced) + elif ctx.get(TYPE_CHECK_ANCHOR_KEY) is not None: + spliced = evaluation.splice_into_source(module, ctx[TYPE_CHECK_ANCHOR_KEY]) + if spliced is not None: + evaluation.type_check(*spliced) + elif ctx.get(REPL_ANCHOR_KEY) is not None: + spliced = evaluation.splice_repl_code_into_body(module, ctx[REPL_ANCHOR_KEY]) + if spliced is not None: + evaluation.type_check(*spliced, lenient=True) + + bytecode: types.CodeType = evaluation.compile(module, filename) + g: dict[str, typing.Any] = {k: v for k, v in ctx.items() if k.isidentifier()} + evaluation.exec(bytecode, g) + result = g[module.body[-1].name] # type: ignore + return result, g + @TypeToPydanticType.register(Callable) -def _pydantic_callable( - ty: typing.Any, metadata: _SynthesisSpec | None = None -) -> typing.Any: - """Create a Pydantic-compatible Annotated type for a parameterized Callable.""" +def _pydantic_callable(ty: typing.Any) -> typing.Any: + """Pydantic-compatible Annotated type for a parameterized `Callable` value. + The model *produces* a function (as ``module_code``); it is synthesized, + type-checked in the enclosing Template's scope, and its own doctests are run. + Template-body synthesis (`submit_solution`) has its own encoding, + `_pydantic_template_body`. + """ typed_enc = SynthesizedFunction._create_typed_synthesized_function( Callable[..., typing.Any] if not typing.get_args(ty) else ty # type: ignore[arg-type] ) @@ -665,114 +865,144 @@ def _validate( ) -> Callable: if isinstance(value, dict): value = typed_enc.model_validate(value) - ctx = info.context or {} - filename = f"" - module: ast.Module = evaluation.parse(value.module_code, filename) + result, g = _synthesize_callable( + value.module_code, info.context or {}, template_body=False + ) + evaluation.run_doctests(result, g) + return result - if ctx.get(TYPE_CHECK_ANCHOR_KEY) is not None: - spliced = evaluation.splice_into_source(module, ctx[TYPE_CHECK_ANCHOR_KEY]) - if spliced is not None: - evaluation.type_check(*spliced) - elif ctx.get(REPL_ANCHOR_KEY) is not None: - spliced = evaluation._splice_repl( - [], value.module_code, ctx[REPL_ANCHOR_KEY] - ) - if spliced is not None: - evaluation.type_check(*spliced, lenient=True) - - bytecode: types.CodeType = evaluation.compile(module, filename) - - g: dict[str, typing.Any] = {k: v for k, v in ctx.items() if k.isidentifier()} - evaluation.exec(bytecode, g) - result = g[module.body[-1].name] # type: ignore - - if metadata is not None: - if metadata._class_template is not None: - # Agent-method template: doctests build their own instances, so the - # method must route to `synth` on *any* instance (not just the one - # that triggered synthesis). A fresh instance's call dispatches - # through `Template.__apply__`, which we intercept here. - result = functools.wraps(metadata._class_template)(result) - - def _doctest_apply(op, *args, **kwargs): - instance = metadata._method_instance(op) - if instance is None: - return fwd() - return metadata._class_template(instance, *args, **kwargs) - - with handler( - { - Template.__apply__: _doctest_apply, - metadata._class_template: result, - } - ): - evaluation.run_doctests(result, g) - return result - else: - # Free-function template: shadow the global name the doctest calls, - # and route the template op back into `synth` for recursion. - result = functools.wraps(metadata.template)(result) - g.update({metadata.template.__name__: result}) - with handler({metadata.template: result}): - evaluation.run_doctests(result, g) - return result - else: + # Distinct schemas per direction: validation (the model *produces* a function) + # carries the synthesis instructions; serialization (the model *reads* an + # encoded function) shows only the `module_code` shape `_serialize_synthesized` + # emits, with no synthesis prose. + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate), + pydantic.PlainSerializer( + lambda value: _serialize_synthesized(value, typed_enc) + ), + pydantic.WithJsonSchema( + _inline_refs(pydantic.TypeAdapter(typed_enc).json_schema()), + mode="validation", + ), + pydantic.WithJsonSchema( + EncodedFunction.model_json_schema(), mode="serialization" + ), + ] + + +@TypeToPydanticType.register(TemplateBody) +def _pydantic_template_body(ty: typing.Any) -> typing.Any: + """`TypeToPydanticType` case for a free-function `Template` body. + + Like `_pydantic_callable`, but the synthesized function is checked against the + enclosing Template's source (the ``anchor`` in the decode context) and its + doctests are run with the Template's own name/op routed back to the synthesized + implementation, so a doctest that calls the Template (including for recursion) + exercises the freshly synthesized code rather than re-invoking the model. + """ + typed_enc = SynthesizedTemplateBody._create_typed_synthesized_function( + ty if typing.get_args(ty) else Callable[..., typing.Any], # type: ignore[arg-type] + method=False, + ) + + def _validate( + value: SynthesizedTemplateBody | dict, info: pydantic.ValidationInfo + ) -> Callable: + if isinstance(value, dict): + value = typed_enc.model_validate(value) + ctx = info.context or {} + result, g = _synthesize_callable(value.module_code, ctx, template_body=True) + anchor = ctx.get(TYPE_CHECK_ANCHOR_KEY) or ctx.get(REPL_ANCHOR_KEY) + if anchor is None: evaluation.run_doctests(result, g) return result + # Shadow the global name the doctests call and route the Template op back + # into the synthesized function. + result = functools.wraps(anchor)(result) + g.update({anchor.__name__: result}) + with handler({anchor: result}): + evaluation.run_doctests(result, g) + return result - def _serialize(value: Callable) -> dict: - try: - source = inspect.getsource(value) - except (OSError, TypeError): - source = None + # Distinct schemas per direction: validation (the model *produces* a function) + # carries the synthesis instructions; serialization (the model *reads* an + # encoded function) shows only the `module_code` shape `_serialize_synthesized` + # emits, with no synthesis prose. + return typing.Annotated[ + ty, + pydantic.PlainValidator(_validate), + pydantic.PlainSerializer( + lambda value: _serialize_synthesized(value, typed_enc) + ), + pydantic.WithJsonSchema( + _inline_refs(pydantic.TypeAdapter(typed_enc).json_schema()), + mode="validation", + ), + pydantic.WithJsonSchema( + EncodedFunction.model_json_schema(), mode="serialization" + ), + ] - if source: - return typed_enc(module_code=textwrap.dedent(source)).model_dump() - name = getattr(value, "__name__", None) - docstring = inspect.getdoc(value) - if name is None or docstring is None: - raise ValueError( - f"Cannot encode callable {value}: no source code and no __name__ or docstring" - ) +@TypeToPydanticType.register(MethodTemplateBody) +def _pydantic_method_template_body(ty: typing.Any) -> typing.Any: + """`TypeToPydanticType` case for an instance-method `Template` body. - try: - sig = inspect.signature(value) - sig_str = str(sig) - except (ValueError, TypeError): - sig_str = "(...)" + Registered separately from `TemplateBody` (rather than reached via subclass + MRO) so the method/free distinction is an explicit dispatch: it surfaces the + leading ``self`` receiver in the signature hint, and its doctests -- which build + their own instances -- route ``agent.method(...)`` on *any* instance to the + synthesized implementation. + """ + typed_enc = SynthesizedTemplateBody._create_typed_synthesized_function( + ty if typing.get_args(ty) else Callable[..., typing.Any], # type: ignore[arg-type] + method=True, + ) - stub_code = f'''def {name}{sig_str}: - """{docstring}""" - ... -''' - return typed_enc(module_code=stub_code).model_dump() + def _validate( + value: SynthesizedTemplateBody | dict, info: pydantic.ValidationInfo + ) -> Callable: + if isinstance(value, dict): + value = typed_enc.model_validate(value) + ctx = info.context or {} + result, g = _synthesize_callable(value.module_code, ctx, template_body=True) + anchor = ctx.get(TYPE_CHECK_ANCHOR_KEY) or ctx.get(REPL_ANCHOR_KEY) + class_template = _class_template_of(anchor) if anchor is not None else None + if class_template is None: + evaluation.run_doctests(result, g) + return result + # A fresh instance's `agent.method(...)` dispatches through + # `Template.__apply__`, which we intercept and redirect to the synthesized + # implementation. + result = functools.wraps(class_template)(result) + + def _doctest_apply(op, *args, **kwargs): + instance = _method_instance(op, class_template) + if instance is None: + return fwd() + return class_template(instance, *args, **kwargs) + + with handler({Template.__apply__: _doctest_apply, class_template: result}): + evaluation.run_doctests(result, g) + return result + # Distinct schemas per direction: validation (the model *produces* a function) + # carries the synthesis instructions; serialization (the model *reads* an + # encoded function) shows only the `module_code` shape `_serialize_synthesized` + # emits, with no synthesis prose. return typing.Annotated[ ty, pydantic.PlainValidator(_validate), - pydantic.PlainSerializer(_serialize), - # Distinct schemas per direction. Validation (the model *produces* a - # function -- tool arguments, response_format) carries the synthesis - # instructions. Serialization (the model *reads* an encoded function -- - # e.g. a tool's output) shows only the shape `_serialize` emits, with no - # synthesis prose. + pydantic.PlainSerializer( + lambda value: _serialize_synthesized(value, typed_enc) + ), pydantic.WithJsonSchema( _inline_refs(pydantic.TypeAdapter(typed_enc).json_schema()), mode="validation", ), pydantic.WithJsonSchema( - { - "type": "object", - "required": ["module_code"], - "properties": { - "module_code": { - "type": "string", - "description": "Python source defining the function.", - } - }, - }, - mode="serialization", + EncodedFunction.model_json_schema(), mode="serialization" ), ] diff --git a/effectful/handlers/llm/evaluation.py b/effectful/handlers/llm/evaluation.py index c39cde71a..ad0edbb41 100644 --- a/effectful/handlers/llm/evaluation.py +++ b/effectful/handlers/llm/evaluation.py @@ -65,7 +65,8 @@ def type_check( lenient: when True, relax mypy for incrementally-built REPL code spliced into a Template body -- allow redefinition (a cell may rebind or redefine a name) and don't require the body to satisfy the Template's return type. Off (strict) - for synthesized ``Callable`` bodies, which must honor their signature. + for a synthesized ``Callable`` or ``TemplateBody``, which must honor its + signature and gets no redefinition slack. Returns None, raises TypeError on an in-region failure. """ @@ -131,6 +132,14 @@ def exec( logger = logging.getLogger(__name__) +# The shared output of the three splicers (`splice_into_source`, +# `splice_template_body`, `splice_repl_code_into_body`): the module ``source`` to +# type-check and the inclusive ``[lo, hi]`` line span within it to report +# diagnostics from -- exactly the leading arguments of `type_check`. ``None`` (not +# this type) is returned when the anchor's source can't be recovered. +type SplicedRegion = tuple[str, int, int] + + def scan_non_nestable(generated: ast.Module) -> None: """Reject constructs legal at module level but illegal once nested in a function. @@ -213,8 +222,8 @@ def _region_errors( def splice_into_source( - generated: ast.Module, anchor: typing.Any -) -> tuple[str, int, int] | None: + generated: ast.Module, anchor: collections.abc.Callable[..., typing.Any] +) -> SplicedRegion | None: """Splice `generated` into the anchor Template's own function body, in its real module source. @@ -228,6 +237,33 @@ def splice_into_source( body of the Template's own function at its real (possibly nested) position, so the generated code is checked in its real lexical scope with no synthesized type stubs. + + This is the splice for a Template whose *return type* is a callable (the model + writes a function and the Template returns it). Example. For the Template :: + + @Template.define + def make_adder(n: int) -> Callable[[int], int]: + '''Return a function that adds {n}.''' + + a model that submits this ``generated`` (its last statement is the function to + return) :: + + def adder(x: int) -> int: + return x + n + + becomes the whole Template body followed by ``return `` :: + + @Template.define + def make_adder(n: int) -> Callable[[int], int]: + def adder(x: int) -> int: + return x + n + return adder + + so mypy checks that ``adder`` satisfies ``Callable[[int], int]`` and that its + body may reference the Template's ``n``. Contrast `splice_template_body`, which + grafts the model's function *body* under the Template's own header (for a + Template whose body -- not return value -- is synthesized). The returned + ``[lo, hi]`` spans the generated statements only, not the ``def`` header. """ if not generated.body: raise TypeError("splice: generated module is empty") @@ -276,8 +312,99 @@ def splice_into_source( return checked_source, lo, hi +def splice_template_body( + generated: ast.Module, anchor: collections.abc.Callable[..., typing.Any] +) -> SplicedRegion | None: + """Splice a synthesized function in as the anchor Template's *own body*. + + Unlike `splice_into_source` (which appends ``return `` and checks that the + Template returns the synthesized *function*), this treats the synthesized + function as the Template's implementation: the Template keeps its own + authoritative signature and its body becomes ``[, *]``. mypy then checks that body + against the Template's declared parameter and return types -- so a body that + fails to return the declared type is rejected. The synthesized function's own + parameter list (including any ``self``) is intentionally discarded: the + Template's real signature is the contract. + + ``generated`` is the model's whole ``module_code`` parsed to a module; its + *last* statement is the implementation, any earlier statements are helper + definitions/imports. For example, given the Template :: + + @Template.define + def parity(numbers: Sequence[int]) -> bool: + '''True iff the sum of {numbers} is odd. + >>> parity([1, 2]) + True + ''' + + a model that submits this ``generated`` (note the header on its final ``def`` + -- ``numbers: list`` -- is discarded) :: + + import math + def _odd(n: int) -> bool: + return n % 2 == 1 + def parity(numbers: list) -> bool: + return _odd(sum(numbers)) + + is spliced into the Template's real source as :: + + @Template.define + def parity(numbers: Sequence[int]) -> bool: # authoritative header kept + import math + def _odd(n: int) -> bool: + return n % 2 == 1 + return _odd(sum(numbers)) # from the final def's body + + so mypy checks the grafted body against ``numbers: Sequence[int]`` and + ``-> bool``. The helper ``_odd`` and ``import math`` (everything before the + final ``def``) become locals at the top of the body; only the final ``def``'s + *body* is taken, under the Template's own header. + + Returns the modified module source and the ``[lo, hi]`` line span from the + ``def`` line through the last body line, or ``None`` when the anchor's source + can't be recovered (REPL/notebook template -- the caller skips rather than + guesses). Raises ``RuntimeError`` on source drift, via `_recover_template_def`. + """ + if not generated.body: + raise TypeError("splice: generated module is empty") + last = generated.body[-1] + if not isinstance(last, ast.FunctionDef | ast.AsyncFunctionDef): + raise TypeError( + f"splice: last statement must be a function definition, " + f"got {type(last).__name__}" + ) + + recovered = _recover_template_def(anchor) + if recovered is None: + return None + module_ast, template_def = recovered + + # Keep the Template's real header (authoritative annotations, `self` for + # methods); replace only its body with the model's helpers/imports followed by + # the synthesized function's body statements, so the declared return type is + # enforced. Any docstring/doctests in the recovered source are dropped. + template_def.body = [*generated.body[:-1], *last.body] + + # Report the def line through the end of the body. Unlike `splice_into_source`, + # the region starts at the `def` line (not the first body statement): mypy + # anchors "Missing return statement"/"empty-body" there, and a body that doesn't + # return the Template's declared type is a real defect we want to catch. The + # header is the Template's own (recovered, resolvable) signature -- sourceless + # templates return `None` above and skip -- so including it adds no spurious + # signature diagnostics. Decorator lines sit above `spliced.lineno` and stay out. + # `template_def` is still a node in `module_ast` (only its body changed), so its + # walk-order index is stable across the unparse round-trip. + def_index = _def_nodes(module_ast).index(template_def) + checked_source = ast.unparse(ast.fix_missing_locations(module_ast)) + spliced = _def_nodes(ast.parse(checked_source))[def_index] + lo = spliced.lineno + hi = spliced.body[-1].end_lineno or lo + return checked_source, lo, hi + + def _recover_template_def( - anchor: typing.Any, + anchor: collections.abc.Callable[..., typing.Any], ) -> tuple[ast.Module, ast.FunctionDef | ast.AsyncFunctionDef] | None: """Locate the anchor Template's own ``def`` in its real module source. @@ -286,7 +413,11 @@ def _recover_template_def( skips rather than guesses). Raises ``RuntimeError`` on source drift (source recovered but the def no longer sits where ``fn`` was compiled from). """ - fn = inspect.unwrap(anchor) # staticmethod/classmethod -> underlying function + # `anchor` is the enclosing `Template` (an `Operation`), a bound method, or a + # plain function; `inspect.unwrap` follows the `__wrapped__` chain that + # `Operation`/method binding sets up, resolving all of them to the original + # source-backed function (staticmethod/classmethod included). + fn = inspect.unwrap(anchor) # Recover the module source via fn's own filename -- a real path or a # linecache-registered synthetic name (e.g. ) for REPL/exec/ # notebook templates; linecache.getlines reads real files from disk too. @@ -308,32 +439,53 @@ def _recover_template_def( return module_ast, template_def -def _splice_repl( - prior: list[str], snippet: str, anchor: typing.Any -) -> tuple[str, int, int] | None: - """Splice the cumulative REPL code -- ``prior`` snippets followed by the current - ``snippet`` -- into the anchor Template's body, in its real module source, and return - the modified source with the ``[lo, hi]`` line span of the *current* snippet. +def splice_repl_code_into_body( + generated: ast.Module, anchor: collections.abc.Callable[..., typing.Any] +) -> SplicedRegion | None: + """Splice REPL code -- ``generated`` -- into the anchor Template's body, in its + real module source, and return the modified source with the ``[lo, hi]`` line + span of the spliced statements. + + ``generated`` is the cumulative session code (any already-run snippets followed + by the current one; the caller prepends them). It becomes the Template function's + body at its real (possibly nested) position, so the Template's parameters and + enclosing scope -- i.e. the session's seed env -- are in scope and each statement + sees the ones before it (they are function locals). No ``return`` is appended; + the REPL code doesn't produce the Template's declared type, and that contract is + waived by ``lenient`` type checking. The whole spliced body is reported, but the + already-run snippets are type-clean (they passed this same check when *they* were + the current one), so only the new statements can raise. + + Example. For the Template :: + + @Template.define + def analyze(data: list[int]) -> str: + '''Analyze {data}.''' + + a ``generated`` module of accumulated session statements :: - The REPL code becomes the Template function's body at its real (possibly nested) - position, so the Template's parameters and enclosing scope -- i.e. the session's seed - env -- are in scope, and each snippet sees the ones before it (they are function - locals). No ``return`` is appended; the REPL code doesn't produce the Template's - declared type, and that contract is waived by ``lenient`` type checking. Every prior - snippet stays in the body so its bindings resolve (matching the runtime, which ran - them), but only the current snippet's lines are reported, so an earlier cell's error - isn't re-reported on every later call. + total = sum(data) + print(total / len(data)) - Returns ``None`` when the current snippet has no statements to check, or when the + becomes the Template's body :: + + @Template.define + def analyze(data: list[int]) -> str: + total = sum(data) + print(total / len(data)) + + so each statement sees the Template's ``data`` and the earlier statements' + bindings (here ``total``). + + Returns ``None`` when ``generated`` has no statements to check, or when the Template's source can't be recovered -- a Template defined at a REPL, in a notebook, or via ``exec()`` is sourceless, so we skip the check and run the code unchecked, exactly as ``splice_into_source`` does for a sourceless Callable anchor. Raises ``RuntimeError`` only on source *drift* (source recovered but the def no longer sits where it was compiled from), which ``_recover_template_def`` surfaces. """ - # An empty or comment-only snippet parses to zero statements: nothing to check. - n_current = len(ast.parse(snippet).body) - if n_current == 0: + # An empty or comment-only module parses to zero statements: nothing to check. + if not generated.body: return None # None means the Template's source can't be recovered (REPL/exec/notebook-defined) -- # skip, like the Callable path, rather than break the tool; `_recover_template_def` @@ -342,16 +494,14 @@ def _splice_repl( if recovered is None: return None module_ast, template_def = recovered - cumulative = "".join(s if s.endswith("\n") else s + "\n" for s in [*prior, snippet]) - template_def.body = ast.parse(cumulative).body + template_def.body = list(generated.body) - # mypy reports line numbers in the coordinates of the unparsed source; the current - # snippet is the last `n_current` statements of the spliced body. ast.unparse keeps def - # order, so the template def is at the same walk index after the round-trip. + # `template_def` is still a node in `module_ast` (only its body changed), so its + # walk-order index is stable across the unparse round-trip. def_index = _def_nodes(module_ast).index(template_def) checked_source = ast.unparse(ast.fix_missing_locations(module_ast)) spliced = _def_nodes(ast.parse(checked_source))[def_index] - lo = spliced.body[-n_current].lineno + lo = spliced.body[0].lineno hi = spliced.body[-1].end_lineno or lo return checked_source, lo, hi diff --git a/tests/test_handlers_llm_evaluation.py b/tests/test_handlers_llm_evaluation.py index 9b8917c9a..5751cf84f 100644 --- a/tests/test_handlers_llm_evaluation.py +++ b/tests/test_handlers_llm_evaluation.py @@ -24,10 +24,10 @@ ReplSession, RestrictedEvalProvider, UnsafeEvalProvider, - _splice_repl, run_doctests, scan_non_nestable, splice_into_source, + splice_repl_code_into_body, type_check, ) from effectful.handlers.llm.evaluation import compile as compile_op @@ -800,8 +800,8 @@ def test_decode_restricted_provider(self): # REPL code type-checking (issue #690) # # `exec_code` type-checks the cumulative session code -- prior snippets plus the -# current one -- spliced into the enclosing Template's body (`_splice_repl` + the -# `type_check` op, `lenient=True`), so names resolve in their real execution +# current one -- spliced into the enclosing Template's body (`splice_repl_code_into_body` +# + the `type_check` op, `lenient=True`), so names resolve in their real execution # context. These tests drive that pipeline against a real anchor and assert the # contract by exception type / runtime effect -- never by matching a mypy message # or a filename. @@ -818,7 +818,10 @@ def _repl_anchor(readings: list[int]) -> int: def _repl_raises(prior: list[str], snippet: str) -> bool: """type_check the cumulative REPL code spliced into `_repl_anchor`'s body; True if it reports an in-region error.""" - checked = _splice_repl(prior, snippet, _repl_anchor) + # Prepend the already-run snippets to the current one (as the production caller + # does) and splice the whole cumulative module. + prior_src = "".join(s if s.endswith("\n") else s + "\n" for s in prior) + checked = splice_repl_code_into_body(ast.parse(prior_src + snippet), _repl_anchor) assert checked is not None with handler(UnsafeEvalProvider()): try: @@ -881,13 +884,17 @@ def test_repl_illtyped_but_runnable_snippet_is_caught(): assert _repl_raises([], "n: int = 'oops'\nprint(n)") -def test_repl_check_reports_only_the_current_snippet(): - """Only the current snippet's lines are reported: an error in the current cell raises, - but the same error confined to an earlier cell is out of region (not re-reported) -- - while that earlier cell stays in the body so its bindings still resolve.""" - assert _repl_raises([], "bad: int = 'x'\nprint(bad)") # current cell -> reported - assert not _repl_raises(["bad: int = 'x'"], "ok = 1\nprint(ok)") # earlier -> not - assert not _repl_raises(["c = 3"], "print(c + 1)") # earlier binding still resolves +def test_repl_checks_the_cumulative_body(): + """The whole cumulative body is checked. A production caller prepends only + already-run, type-clean snippets (from `repl_history`), so an error can only + come from the current cell; earlier cells stay in the body so their bindings + still resolve.""" + assert _repl_raises( + [], "bad: int = 'x'\nprint(bad)" + ) # current cell error -> raised + assert not _repl_raises( + ["c = 3"], "print(c + 1)" + ) # earlier binding resolves, clean def test_repl_splice_skips_sourceless_anchor(): @@ -896,7 +903,7 @@ def test_repl_splice_skips_sourceless_anchor(): Callable anchor. Only source *drift* raises.""" ns: dict[str, Any] = {} exec("def t(readings):\n raise NotImplementedError", ns) - assert _splice_repl([], "x = 1", ns["t"]) is None + assert splice_repl_code_into_body(ast.parse("x = 1"), ns["t"]) is None # --- decode-time type-checking: a decode gate, exactly like Callable synthesis --- From 7ecf823b488ed5cd9e2484a2cdb97ad9ae4581ef Mon Sep 17 00:00:00 2001 From: Eli Date: Thu, 23 Jul 2026 23:42:46 -0400 Subject: [PATCH 055/155] fix no-tool-call error --- effectful/handlers/llm/completions.py | 39 ++++++++++++++++++++++++--- effectful/handlers/llm/encoding.py | 12 ++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index ee84d70ac..c4312a6f8 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -244,6 +244,7 @@ def call_assistant[T]( response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), anchor: "Template | None" = None, + force_tool: bool = False, ) -> AssistantResult[T]: """Low-level LLM request. Handlers may log/modify requests and delegate via fwd(). @@ -256,6 +257,13 @@ def call_assistant[T]( model-visible name is derived from its `__name__`, so collection and decoding agree on a single naming scheme. + `force_tool` is set when the request requires the model to call a tool (the + provider derives it from a ``tool_choice="required"`` config) so that a + response which nonetheless comes back with no tool call — some + OpenAI-compatible servers treat ``tool_choice`` as advisory — is reported as + the protocol violation it is, rather than being misdecoded as a bare + structured result. + Raises: ToolCallDecodingError: If a tool call cannot be decoded. The error includes the raw assistant message for retry handling. @@ -297,6 +305,15 @@ def call_assistant[T]( append_message(raw_message) raw_tool_calls = message.get("tool_calls") or [] + if force_tool and not raw_tool_calls: + raise ResultDecodingError( + ValueError( + "tool_choice='required' but the model returned no tool call." + "**IMPORTANT: YOU MUST GENERATE A TOOL CALL IN YOUR NEXT RESPONSE.**" + ), + raw_message=raw_message, + ) + tool_calls: list[DecodedToolCall] = [] encoding: pydantic.TypeAdapter[DecodedToolCall] = pydantic.TypeAdapter( Encodable[DecodedToolCall] @@ -694,6 +711,7 @@ def _call_assistant[T]( response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), anchor: "Template | None" = None, + force_tool: bool = False, ) -> AssistantResult[T]: readers: set[Tool] = set(tools) taken = {t.__name__ for t in tools} @@ -710,7 +728,7 @@ def _call_assistant[T]( taken.add(name) except Exception: continue - return fwd(env, response_type, readers, anchor=anchor) + return fwd(env, response_type, readers, anchor=anchor, force_tool=force_tool) class SynthesizeAndCall(ObjectInterpretation): @@ -770,6 +788,13 @@ class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): doctests: a solution whose doctests fail (or that errors when applied) is rejected and fed back to you to revise, so the answer only stands once the function's own doctests pass. Calling this tool terminates the completion. + + This answers the *current* call only. Each call is a fresh, independent + task: even if you already submitted a working solution earlier in this + conversation, a prior submission is not a standing answer — you must call + `submit_solution` again to answer the current call. Never end a turn with + a prose summary in place of the answer; a plain message is not a valid + response and will be rejected. """ __toolname__: typing.ClassVar[typing.Literal["submit_solution"]] = ( @@ -822,10 +847,14 @@ def _apply[**P, T]( bound_args.apply_defaults() tool = self._SynthesisFinalTool.define(template, bound_args) - def _add_synthesis_tool(env, response_type, tools=frozenset(), anchor=None): + def _add_synthesis_tool( + env, response_type, tools=frozenset(), anchor=None, force_tool=False + ): if any(isinstance(t, self._SynthesisFinalTool) for t in tools): return fwd() - return fwd(env, response_type, tools | {tool}, anchor=anchor) + return fwd( + env, response_type, tools | {tool}, anchor=anchor, force_tool=force_tool + ) with handler({call_assistant: _add_synthesis_tool}): return fwd() @@ -915,12 +944,14 @@ def _call_assistant[T]( response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), anchor: "Template | None" = None, + force_tool: bool = False, ) -> AssistantResult[T]: return fwd( env, response_type, tools | {self.exec_code, self.read_lexical_variable}, anchor=anchor, + force_tool=force_tool, ) @@ -990,6 +1021,7 @@ def _call_assistant[T]( response_type: type[T], tools: collections.abc.Set[Tool] = frozenset(), anchor: "Template | None" = None, + force_tool: bool = False, ) -> AssistantResult[T]: _message_sequence = _get_history().copy() @@ -1521,6 +1553,7 @@ def _call[**P, T]( template.__signature__.return_annotation, _tools_in_scope(env) - {template}, anchor=template, + force_tool=self.config.get("tool_choice") == "required", ) if tool_calls: for tool_call in tool_calls: diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index a14545710..1d4652192 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -861,8 +861,10 @@ def _pydantic_callable(ty: typing.Any) -> typing.Any: ) def _validate( - value: SynthesizedFunction | dict, info: pydantic.ValidationInfo + value: SynthesizedFunction | dict | str, info: pydantic.ValidationInfo ) -> Callable: + if isinstance(value, str): + value = typed_enc.model_validate_json(value) if isinstance(value, dict): value = typed_enc.model_validate(value) result, g = _synthesize_callable( @@ -907,8 +909,10 @@ def _pydantic_template_body(ty: typing.Any) -> typing.Any: ) def _validate( - value: SynthesizedTemplateBody | dict, info: pydantic.ValidationInfo + value: SynthesizedTemplateBody | dict | str, info: pydantic.ValidationInfo ) -> Callable: + if isinstance(value, str): + value = typed_enc.model_validate_json(value) if isinstance(value, dict): value = typed_enc.model_validate(value) ctx = info.context or {} @@ -961,8 +965,10 @@ def _pydantic_method_template_body(ty: typing.Any) -> typing.Any: ) def _validate( - value: SynthesizedTemplateBody | dict, info: pydantic.ValidationInfo + value: SynthesizedTemplateBody | dict | str, info: pydantic.ValidationInfo ) -> Callable: + if isinstance(value, str): + value = typed_enc.model_validate_json(value) if isinstance(value, dict): value = typed_enc.model_validate(value) ctx = info.context or {} From 998c59a37815d64698928dc6e43e7ab6aa34a5b9 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 24 Jul 2026 01:07:05 -0400 Subject: [PATCH 056/155] fix prompts --- docs/source/llm_examples/lexical_scope.py | 4 ++-- effectful/handlers/llm/__init__.py | 9 --------- effectful/handlers/llm/completions.py | 23 +++++++++++++++-------- effectful/handlers/llm/encoding.py | 14 +++++--------- effectful/handlers/llm/template.py | 16 ++++------------ 5 files changed, 26 insertions(+), 40 deletions(-) diff --git a/docs/source/llm_examples/lexical_scope.py b/docs/source/llm_examples/lexical_scope.py index e61f23d01..697205218 100644 --- a/docs/source/llm_examples/lexical_scope.py +++ b/docs/source/llm_examples/lexical_scope.py @@ -20,12 +20,12 @@ @Template.define def story_with_moral(topic: str) -> str: - """Write a short story about {topic} and end with a moral lesson.""" + """Write a short story about {topic} and end with a moral lesson. Do not use any tools.""" @Template.define def story_funny(topic: str) -> str: - """Write a funny, humorous story about {topic}.""" + """Write a funny, humorous story about {topic}. Do not use any tools.""" class TripPlanner(Agent): diff --git a/effectful/handlers/llm/__init__.py b/effectful/handlers/llm/__init__.py index 72ebf2d09..f6e92498d 100644 --- a/effectful/handlers/llm/__init__.py +++ b/effectful/handlers/llm/__init__.py @@ -36,15 +36,6 @@ dataclasses, etc.) come back as real Python values. A `FinalTool` lets the model "answer" by calling a tool whose return value becomes the result and terminates the loop. - -## Providers and handlers - -Execution is controlled by composing handlers with -`effectful.ops.semantics.handler(...)`: a provider such as -`effectful.handlers.llm.completions.LiteLLMProvider` implements the model calls, -and helpers like `RetryLLMHandler` add reliability behavior. Because everything -is an algebraic effect, behavior (model requests, tool dispatch, history) can be -observed, logged, or overridden by installing additional handlers. """ from .template import Agent, Encodable, Template, Tool diff --git a/effectful/handlers/llm/completions.py b/effectful/handlers/llm/completions.py index c4312a6f8..49de6d563 100644 --- a/effectful/handlers/llm/completions.py +++ b/effectful/handlers/llm/completions.py @@ -783,11 +783,14 @@ class _SynthesisFinalTool[T](FinalTool[[collections.abc.Callable[..., T]], T]): as a drop-in implementation of the Template. The function may reference names from the lexical scope (see the *Lexical scope* table). - Give the function a docstring containing `>>>` doctests that demonstrate - its intended behavior on examples. On submission the harness runs those - doctests: a solution whose doctests fail (or that errors when applied) is - rejected and fed back to you to revise, so the answer only stands once the - function's own doctests pass. Calling this tool terminates the completion. + You do not need to write a docstring or doctests: on submission the harness + attaches the Template's own docstring to your function and runs *its* + doctests (with recursive calls to the Template routed to your + implementation). A solution whose doctests fail — or that errors when + applied — is rejected and fed back to you to revise, so the answer only + stands once the Template's doctests pass. Write just the implementation; + any docstring you add is replaced and ignored. Calling this tool terminates + the completion. This answers the *current* call only. Each call is a fresh, independent task: even if you already submitted a working solution earlier in this @@ -827,9 +830,9 @@ def define( def submit_solution(implementation: body_type) -> return_type: # type: ignore """ - Submit your final answer as a Python function implementing the task. - The function must have the required signature; it is applied to the - original inputs and its return value is your final answer. + Answer this Template by submitting a Python function that implements + it (see the "Code synthesis" section); its return value on the + original arguments becomes the answer. """ return implementation(*args, **kwargs) # type: ignore @@ -891,6 +894,10 @@ class _ReplInteractionTool[**P, T](Tool[P, T]): across turns, so you may define variables, functions, and classes that are used in later turns. The return value of the code is returned to you as the result of the tool call. + + Use the REPL only when running code actually helps — computing or verifying + a result, exploring data, or calling a tool. If you can answer directly, just + answer; do not route a plain text answer through `print(...)`. """ @typing.final diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 1d4652192..d53716b4d 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -722,20 +722,16 @@ class SynthesizedTemplateBody(SynthesizedFunction): module_code: str = pydantic.Field( ..., description=textwrap.dedent(""" - The complete Python source for the function implementing the Template. - Write it as a drop-in implementation with the Template's signature (shown - in ... and in the Template spec). The code MUST - satisfy the following constraints, or it will fail validation: + The complete Python source implementing the Template shown in its spec. + The code MUST satisfy the following constraints, or it will fail validation: 1. The code MUST be one complete syntactically valid Python module. 2. The code MUST NOT use star imports or ``__future__`` imports. 3. The function definition MUST be the LAST statement - do not add any code after it. - 4. Write the function with the Template's signature (see the Template spec); - parameter and return annotations are optional. - 5. You may include doctest examples (lines starting with >>>) inside the function's - docstring to demonstrate and verify its behavior; these examples are run as tests, - with calls to the Template routed to this implementation. + 4. Write the function with the Template's signature; parameter and return + annotations are optional. + 5. Do not include a docstring or doctests; the Template's are supplied automatically. """), ) diff --git a/effectful/handlers/llm/template.py b/effectful/handlers/llm/template.py index b3a1439f0..b34f6c105 100644 --- a/effectful/handlers/llm/template.py +++ b/effectful/handlers/llm/template.py @@ -373,24 +373,16 @@ class Agent(abc.ABC): Example: ```python - import dataclasses - from effectful.handlers.llm import Agent, Template - from effectful.handlers.llm.completions import LiteLLMProvider - from effectful.ops.semantics import handler - from effectful.ops.types import NotHandled - - @dataclasses.dataclass + @dataclass class ChatBot(Agent): - bot_name: str = dataclasses.field(default="ChatBot") + bot_name: str @Template.define def send(self, user_input: str) -> str: \"""Friendly bot named {self.bot_name}. User writes: {user_input}\""" - provider = LiteLLMProvider() - chatbot = ChatBot() - - with handler(provider): + def main(): + chatbot = ChatBot() chatbot.send("Hi! How are you? I am in France.") chatbot.send("Remind me again, where am I?") # sees prior context ``` From 2537af03536b97134d5cb1a0b1a5780a8bdd4b50 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 24 Jul 2026 11:21:30 -0400 Subject: [PATCH 057/155] fix bug in method templates that synthesize code --- effectful/handlers/llm/encoding.py | 196 ++++++++++++++++------------- 1 file changed, 109 insertions(+), 87 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index d53716b4d..5e316925e 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -608,6 +608,12 @@ class SynthesizedFunction(EncodedFunction): """), ) + # A general `Callable` is type-checked against the requested signature, so it must + # be fully annotated. A Template *body* is instead checked against the enclosing + # Template's own signature (`splice_template_body`), which already carries the + # annotations -- so its subclasses waive this and may omit the `self` receiver. + _require_annotations: typing.ClassVar[bool] = True + @pydantic.field_validator("module_code") @classmethod def _validate_module_code(cls, value: str) -> str: @@ -625,18 +631,18 @@ def _validate_module_code(cls, value: str) -> str: f"got {type(last_stmt).__name__}" ) - for arg in last_stmt.args.args: - if arg.annotation is None: + if cls._require_annotations: + for arg in last_stmt.args.args: + if arg.annotation is None: + raise ValueError( + f"decode() requires all parameters to have type annotations, " + f"parameter '{arg.arg}' is missing an annotation" + ) + if last_stmt.returns is None: raise ValueError( - f"decode() requires all parameters to have type annotations, " - f"parameter '{arg.arg}' is missing an annotation" + "decode() requires the function to have a return type annotation" ) - if last_stmt.returns is None: - raise ValueError( - "decode() requires the function to have a return type annotation" - ) - for stmt in module.body: if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__": raise ValueError( @@ -654,56 +660,49 @@ def _validate_module_code(cls, value: str) -> str: return value @classmethod - def _create_typed_synthesized_function( - cls, callable_type: type[Callable], *, method: bool = False - ) -> type[typing.Self]: - """Create a SynthesizedFunction subclass with type signature in the model description. - - Uses pydantic.create_model to ensure the description is included in the JSON schema - sent to the LLM, informing it of the expected function signature. - - When ``method``, the leading parameter is the instance receiver: it is - rendered as ``self`` (rather than by its type) so the model writes it - explicitly, and a note records that it may be left unannotated. + def _create_model_from_callable_type(cls, typ: type[Callable]) -> type[typing.Self]: + """Create a SynthesizedFunction subclass carrying the requested signature in + the model-facing description. + + Uses ``pydantic.create_model`` so the rendered signature (and any + subclass-specific instructions) ride in the JSON schema ``description`` sent + to the model. Subclasses customize the receiver rendering via `_param_names` + and add guidance via `_extra_instructions`. """ - if not typing.get_args(callable_type): - type_signature = "Callable" - # Callable[[arg1, arg2, ...], return_type] - elif len(typing.get_args(callable_type)) >= 2: - param_types = typing.get_args(callable_type)[0] - return_type = typing.get_args(callable_type)[-1] - - if param_types is ...: - params_str = "..." - elif isinstance(param_types, list | tuple): - names = [getattr(t, "__name__", str(t)) for t in param_types] - # The receiver's type is uninformative (it is the Agent class); name - # it `self` so the model reproduces the parameter instead of guessing. - if method and names: - names[0] = "self" - params_str = ", ".join(names) - else: - params_str = str(param_types) - - return_str = getattr(return_type, "__name__", str(return_type)) - type_signature = f"Callable[[{params_str}], {return_str}]" - else: - type_signature = str(callable_type) - - doc = f"Python function with signature {type_signature}" - if method: - doc += ( - "\n\nThis implements an instance method: the first parameter is the " - "instance receiver `self`. Include it as the first parameter; you may " - "leave it unannotated." - ) - + doc = ( + f"Python function with signature " + f"{cls._signature_str(typ)}" + f"{cls._extra_instructions()}" + ) return pydantic.create_model( "TypedSynthesizedFunction", __base__=cls, __doc__=doc, ) + @classmethod + def _signature_str(cls, typ: type[Callable]) -> str: + """Render a ``Callable[[...], ...]`` signature by type *name* (not its + fully-qualified ``repr``), so the model sees ``Callable[[State], int]`` rather + than ``collections.abc.Callable[[pkg.mod.State], builtins.int]``.""" + args = typing.get_args(typ) + if not args: + return "Callable" + param_types, return_type = args + params_str = ( + "..." if param_types is ... else ", ".join(cls._param_names(param_types)) + ) + return_str = getattr(return_type, "__name__", str(return_type)) + return f"Callable[[{params_str}], {return_str}]" + + @classmethod + def _param_names(cls, param_types: typing.Iterable[typing.Any]) -> list[str]: + return [getattr(t, "__name__", str(t)) for t in param_types] + + @classmethod + def _extra_instructions(cls) -> str: + return "" + class SynthesizedTemplateBody(SynthesizedFunction): """Structured output for synthesizing a `Template`'s body (`submit_solution`). @@ -736,36 +735,61 @@ class SynthesizedTemplateBody(SynthesizedFunction): """), ) - @pydantic.field_validator("module_code") + # A Template body is checked against the Template's own (already-annotated) + # signature, so the synthesized body's annotations are optional. + _require_annotations: typing.ClassVar[bool] = False + + +class SynthesizedMethodTemplateBody(SynthesizedTemplateBody): + """Structured output for synthesizing an *instance-method* `Template`'s body. + + Decoded through `_pydantic_template_body`: the function is type-checked against + the enclosing Template's source and its doctests are run with self/recursive + calls routed to the synthesized implementation. + + Unlike `SynthesizedFunction`, the parameter and return *annotations* are not + required: a Template body is type-checked against the Template's own signature + (see `splice_template_body`), so the model may omit or vary them -- in + particular it need not annotate the ``self`` receiver of an instance-method + Template. + """ + + module_code: str = pydantic.Field( + ..., + description=textwrap.dedent(""" + The complete Python source implementing the instance-method Template shown in + its spec. The code MUST satisfy the following constraints, or it will fail + validation: + + + 1. The code MUST be one complete syntactically valid Python module. + 2. The code MUST NOT use star imports or ``__future__`` imports. + 3. The function definition MUST be the LAST statement - do not add any code after it. + 4. Write the function with the Template's signature: its FIRST parameter is the + instance receiver ``self`` (which you may leave unannotated); all other parameter + and return annotations are optional too. + 5. Do not include a docstring or doctests; the Template's are supplied automatically. + + """), + ) + @classmethod - def _validate_module_code(cls, value: str) -> str: - # Structural checks only. Parameter/return annotations are intentionally NOT - # required: a TemplateBody is type-checked against the Template's real - # signature (`splice_template_body`), which already carries them -- so the - # model may omit them, and need not annotate the `self` receiver. - module: ast.AST = ast.parse(value) - if not isinstance(module, ast.Module) or not module.body: - raise ValueError( - "decode() requires module code with at least one statement." - ) - last_stmt = module.body[-1] - if not isinstance(last_stmt, ast.FunctionDef): - raise ValueError( - f"decode() requires the last statement to be a function definition, " - f"got {type(last_stmt).__name__}" - ) - for stmt in module.body: - if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__": - raise ValueError( - "decode() does not allow __future__ imports in the module code" - ) - if isinstance(stmt, ast.ImportFrom) and any( - alias.name == "*" for alias in stmt.names - ): - raise ValueError( - "decode() does not allow star imports in the module code" - ) - return value + def _param_names(cls, param_types: typing.Iterable[typing.Any]) -> list[str]: + # The method's callable type already carries the receiver as its first + # parameter (with an uninformative Agent-class type); relabel it ``self`` so + # the model reproduces it rather than inventing one -- do NOT prepend a receiver. + names = super()._param_names(param_types) + if names: + names[0] = "self" + return names + + @classmethod + def _extra_instructions(cls) -> str: + return ( + "\n\nThis implements an instance method: the first parameter is the " + "instance receiver `self`. Include it as the first parameter; you may " + "leave it unannotated." + ) def _serialize_synthesized( @@ -852,7 +876,7 @@ def _pydantic_callable(ty: typing.Any) -> typing.Any: Template-body synthesis (`submit_solution`) has its own encoding, `_pydantic_template_body`. """ - typed_enc = SynthesizedFunction._create_typed_synthesized_function( + typed_enc = SynthesizedFunction._create_model_from_callable_type( Callable[..., typing.Any] if not typing.get_args(ty) else ty # type: ignore[arg-type] ) @@ -899,9 +923,8 @@ def _pydantic_template_body(ty: typing.Any) -> typing.Any: implementation, so a doctest that calls the Template (including for recursion) exercises the freshly synthesized code rather than re-invoking the model. """ - typed_enc = SynthesizedTemplateBody._create_typed_synthesized_function( + typed_enc = SynthesizedTemplateBody._create_model_from_callable_type( ty if typing.get_args(ty) else Callable[..., typing.Any], # type: ignore[arg-type] - method=False, ) def _validate( @@ -955,13 +978,12 @@ def _pydantic_method_template_body(ty: typing.Any) -> typing.Any: their own instances -- route ``agent.method(...)`` on *any* instance to the synthesized implementation. """ - typed_enc = SynthesizedTemplateBody._create_typed_synthesized_function( + typed_enc = SynthesizedMethodTemplateBody._create_model_from_callable_type( ty if typing.get_args(ty) else Callable[..., typing.Any], # type: ignore[arg-type] - method=True, ) def _validate( - value: SynthesizedTemplateBody | dict | str, info: pydantic.ValidationInfo + value: SynthesizedMethodTemplateBody | dict | str, info: pydantic.ValidationInfo ) -> Callable: if isinstance(value, str): value = typed_enc.model_validate_json(value) From 4717f2f8c8eb0eefe9b49f3129ab83d0b10a26b6 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 24 Jul 2026 12:14:21 -0400 Subject: [PATCH 058/155] fix test bugs --- effectful/handlers/llm/encoding.py | 28 ++++++++++++++++++++++++++++ effectful/handlers/llm/harness.py | 5 +++++ tests/test_handlers_llm_template.py | 26 ++++++++++++-------------- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/effectful/handlers/llm/encoding.py b/effectful/handlers/llm/encoding.py index 5e316925e..49a7b4033 100644 --- a/effectful/handlers/llm/encoding.py +++ b/effectful/handlers/llm/encoding.py @@ -867,6 +867,33 @@ def _synthesize_callable( return result, g +def _reject_param_count_mismatch(fn: Callable, ty: typing.Any) -> None: + """Raise ``ValueError`` if the synthesized ``fn``'s positional arity does not + match the expected ``Callable[[...], ret]`` type. + + The mypy signature check only runs when a type-check anchor is in scope; this + structural check runs unconditionally, so a wrong parameter count is still + rejected on the anchorless argument-decoding path. + """ + args = typing.get_args(ty) + if not args or args[0] is ...: + return # bare ``Callable`` or ``Callable[..., R]``: any arity is acceptable + expected = len(args[0]) + params = list(inspect.signature(fn).parameters.values()) + if any(p.kind is inspect.Parameter.VAR_POSITIONAL for p in params): + return # ``*args`` accepts any number of positional arguments + positional = sum( + p.kind + in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + for p in params + ) + if positional != expected: + raise ValueError( + f"synthesized function takes {positional} positional parameter(s), " + f"but the expected signature has {expected}" + ) + + @TypeToPydanticType.register(Callable) def _pydantic_callable(ty: typing.Any) -> typing.Any: """Pydantic-compatible Annotated type for a parameterized `Callable` value. @@ -890,6 +917,7 @@ def _validate( result, g = _synthesize_callable( value.module_code, info.context or {}, template_body=False ) + _reject_param_count_mismatch(result, ty) evaluation.run_doctests(result, g) return result diff --git a/effectful/handlers/llm/harness.py b/effectful/handlers/llm/harness.py index 5fde40d11..ac60595c3 100644 --- a/effectful/handlers/llm/harness.py +++ b/effectful/handlers/llm/harness.py @@ -185,6 +185,11 @@ def main(argv: list[str] | None = None) -> None: ns, script_args = _parse_args(sys.argv[1:] if argv is None else argv) # The script should see only its own flags, under its own name. sys.argv = [ns.script, *script_args] + # Mirror `python