From f3cff01aba9bda5e1a4a01e0178e77f16a5f4c3b Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 09:55:32 -0400 Subject: [PATCH 1/9] Unify Operation with Callable --- effectful/internals/unification.py | 11 +++++ tests/test_internals_unification.py | 66 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/effectful/internals/unification.py b/effectful/internals/unification.py index ba770ad3d..110b4d872 100644 --- a/effectful/internals/unification.py +++ b/effectful/internals/unification.py @@ -539,6 +539,17 @@ def _unify_generic(typ, subtyp, subs: Substitutions) -> Substitutions: typing.get_origin(typ), collections.abc.Generator ): return unify(typing.get_args(typ)[0], typing.get_args(subtyp)[0], subs) + elif typing.get_origin(subtyp) is effectful.ops.types.Operation and not ( + isinstance(typing.get_origin(typ), type) + and issubclass(typing.get_origin(typ), effectful.ops.types.Operation) + ): + # An Operation[P, R] is a Callable[P, R] (gh #669): unify the pattern + # against the operation's parameter/return signature. ``Operation``'s + # args are (params, return) just like ``Callable``'s, except params is + # a tuple (or ``...``) rather than a list. + op_params, op_ret = typing.get_args(subtyp) + callable_params = op_params if op_params is ... else list(op_params) + return unify(typ, collections.abc.Callable[callable_params, op_ret], subs) # type: ignore elif typing.get_origin(typ) == typing.get_origin(subtyp): return unify(typing.get_args(typ), typing.get_args(subtyp), subs) elif types.get_original_bases(typing.get_origin(subtyp)): diff --git a/tests/test_internals_unification.py b/tests/test_internals_unification.py index fc1a4ba14..fe3a9ed06 100644 --- a/tests/test_internals_unification.py +++ b/tests/test_internals_unification.py @@ -1967,3 +1967,69 @@ class Info(typing.TypedDict): subs = unify(collections.abc.Mapping, Info) assert subs == {} + + +def test_unify_jax_array_iterable(): + import jax + + subs = unify(collections.abc.Iterable[T], jax.Array) + assert subs == {T: jax.Array} + + +def test_unify_operation_callable(): + """An ``Operation[P, R]`` unifies as a ``Callable[P, R]`` (gh #669).""" + from effectful.ops.types import Operation + + # TypeVar params bind to the operation's parameter/return types + assert unify(collections.abc.Callable[[T], V], Operation[[int], int]) == { + T: int, + V: int, + } + # a repeated TypeVar binds consistently + assert unify(collections.abc.Callable[[T], T], Operation[[int], int]) == {T: int} + # multiple parameters + assert unify(collections.abc.Callable[[T, U], V], Operation[[int, str], bool]) == { + T: int, + U: str, + V: bool, + } + # ``...`` parameters in the pattern ignore the operation's parameter types + assert unify(collections.abc.Callable[..., V], Operation[[int], int]) == {V: int} + # fully concrete: nothing to bind + assert unify(collections.abc.Callable[[int], int], Operation[[int], int]) == {} + # nested: an operation-valued argument + assert unify( + collections.abc.Callable[[T], list[V]], Operation[[int], list[str]] + ) == {T: int, V: str} + + +def test_unify_operation_callable_failure(): + """An arity mismatch between the Callable pattern and the Operation fails.""" + from effectful.ops.types import Operation + + with pytest.raises(TypeError): + unify(collections.abc.Callable[[T, U], V], Operation[[int], int]) + with pytest.raises(TypeError): + unify(collections.abc.Callable[[T], V], Operation[[int, str], bool]) + + +def test_operation_unifies_with_callable_param_gh669(): + """An Operation passed where a ``Callable`` is expected infers correctly. + + Regression test for gh #669: calling an operation whose parameter is typed + ``Callable[[S], T]`` with another operation should unify and infer the return + type, rather than raising ``Cannot unify generic type ...``. + """ + from effectful.ops.semantics import typeof + from effectful.ops.types import NotHandled, Operation + + @Operation.define + def f(x: int) -> int: + raise NotHandled + + @Operation.define + def g[S, R](x: collections.abc.Callable[[S], R]) -> R: + raise NotHandled + + term = g(f) + assert typeof(term) is int From f92213dc66602fe760a7900ac8fca2332b75fce5 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 10:26:20 -0400 Subject: [PATCH 2/9] grab fix from 656 --- effectful/internals/unification.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/effectful/internals/unification.py b/effectful/internals/unification.py index 110b4d872..ad022a63b 100644 --- a/effectful/internals/unification.py +++ b/effectful/internals/unification.py @@ -567,6 +567,17 @@ def _unify_generic(typ, subtyp, subs: Substitutions) -> Substitutions: and issubclass(subtyp, typing.get_origin(typ)) ): return subs # implicit expansion to subtyp[Any] + elif isinstance(typ, GenericAlias): + # Special case for treating arrays as iterables of arrays + try: + import jax + + if typing.get_origin(typ) is collections.abc.Iterable and issubclass( + subtyp, jax.Array + ): + return unify(typing.get_args(typ)[0], jax.Array, subs) + except ImportError: + pass raise TypeError(f"Cannot unify generic type {typ} with {subtyp} given {subs}.") From 379956d7c724014d9b674343fe22cfb2d4eb9175 Mon Sep 17 00:00:00 2001 From: Eli Date: Tue, 9 Jun 2026 18:34:03 -0400 Subject: [PATCH 3/9] evaluate patterns for builtin iterator types --- effectful/ops/semantics.py | 79 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/effectful/ops/semantics.py b/effectful/ops/semantics.py index f7678fd24..15d15a309 100644 --- a/effectful/ops/semantics.py +++ b/effectful/ops/semantics.py @@ -1,7 +1,10 @@ +import builtins import collections.abc import contextlib import dataclasses import functools +import inspect +import itertools import operator import types import typing @@ -224,6 +227,13 @@ def _evaluate_object[T](expr: T, **kwargs) -> T: return expr +@evaluate.register(builtins.range) +@evaluate.register(str | bytes | bytearray) +@evaluate.register(int | float | complex | bool | type(None)) +def _evaluate_atomic(expr: Any, **kwargs): + return expr + + @evaluate.register(Term) def _evaluate_term(expr: Term, **kwargs): args = tuple(evaluate(arg) for arg in expr.args) @@ -284,6 +294,75 @@ def _evaluate_list_view(expr, **kwargs): return [evaluate(item) for item in expr] +@evaluate.register(collections.abc.Set) +def _evaluate_set(expr, **kwargs): + return type(expr)(evaluate(item) for item in expr) + + +@evaluate.register(collections.abc.Generator) +def _evaluate_generator(expr, **kwargs): + if inspect.getgeneratorstate(expr) != inspect.GEN_CREATED: + return expr # cannot introspect in-progress generator + else: + from effectful.internals.runtime import get_interpretation + + intp = get_interpretation() + return (evaluate(item, intp=intp) for item in expr) + + +@evaluate.register(collections.abc.Iterator) +def _evaluate_iterator(expr, **kwargs): + return iter(evaluate(expr.__reduce__()[1])) + + +@evaluate.register(type(iter((1,)))) +def _evaluate_tuple_iterator(expr, **kwargs): + return iter(evaluate(expr.__reduce__()[1])) + + +@evaluate.register(builtins.slice) +def _evaluate_slice(expr, **kwargs): + return builtins.slice(*evaluate((expr.start, expr.stop, expr.step))) + + +@evaluate.register(builtins.map) +def _evaluate_map(expr, **kwargs): + _, (fn, iterator) = expr.__reduce__() + return builtins.map(evaluate(fn), evaluate(iterator)) + + +@evaluate.register(builtins.filter) +def _evaluate_filter(expr, **kwargs): + _, (fn_or_none, iterator) = expr.__reduce__() + return builtins.filter(evaluate(fn_or_none), evaluate(iterator)) + + +@evaluate.register(builtins.zip) +def _evaluate_zip(expr, **kwargs): + iterators_strict = expr.__reduce__()[1:] + strict = len(iterators_strict) == 2 and iterators_strict[1] is True + values = iterators_strict[0] + return builtins.zip(*[evaluate(v) for v in values], strict=strict) + + +@evaluate.register(builtins.enumerate) +def _evaluate_enumerate(expr, **kwargs): + _, (iterator, start) = expr.__reduce__() + return builtins.enumerate(evaluate(iterator), start=start) + + +@evaluate.register(builtins.reversed) +def _evaluate_reversed(expr, **kwargs): + _, (seq,) = expr.__reduce__() + return builtins.reversed(evaluate(seq)) + + +@evaluate.register(itertools.product) +def _evaluate_product(expr: itertools.product, **kwargs): + _, (iterables, repeat) = expr.__reduce__() + return itertools.product(*[evaluate(it) for it in iterables], repeat=repeat) + + def _simple_type(tp: type) -> type: """Convert a type object into a type that can be dispatched on.""" if isinstance(tp, typing.TypeVar): From 7eafd731d31c1aeeede82066f94c47f4b1da62a9 Mon Sep 17 00:00:00 2001 From: Eli Date: Tue, 9 Jun 2026 21:57:10 -0400 Subject: [PATCH 4/9] simplify --- effectful/ops/semantics.py | 69 +++------------ tests/test_ops_semantics.py | 164 ++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 59 deletions(-) diff --git a/effectful/ops/semantics.py b/effectful/ops/semantics.py index 15d15a309..75d607a72 100644 --- a/effectful/ops/semantics.py +++ b/effectful/ops/semantics.py @@ -3,8 +3,6 @@ import contextlib import dataclasses import functools -import inspect -import itertools import operator import types import typing @@ -210,8 +208,6 @@ def evaluate[T]( @evaluate.register(object) -@evaluate.register(str) -@evaluate.register(bytes) def _evaluate_object[T](expr: T, **kwargs) -> T: if dataclasses.is_dataclass(expr) and not isinstance(expr, type): return typing.cast( @@ -299,25 +295,18 @@ def _evaluate_set(expr, **kwargs): return type(expr)(evaluate(item) for item in expr) -@evaluate.register(collections.abc.Generator) -def _evaluate_generator(expr, **kwargs): - if inspect.getgeneratorstate(expr) != inspect.GEN_CREATED: - return expr # cannot introspect in-progress generator - else: - from effectful.internals.runtime import get_interpretation - - intp = get_interpretation() - return (evaluate(item, intp=intp) for item in expr) - - @evaluate.register(collections.abc.Iterator) def _evaluate_iterator(expr, **kwargs): - return iter(evaluate(expr.__reduce__()[1])) - - -@evaluate.register(type(iter((1,)))) -def _evaluate_tuple_iterator(expr, **kwargs): - return iter(evaluate(expr.__reduce__()[1])) + try: + ctor, args, *state = expr.__reduce__() + except (TypeError, AttributeError): + return expr # un-reducible iterators are opaque, like any object we can't recurse into + result = ctor(*evaluate(args)) + if state and state[0] is not None and hasattr(result, "__setstate__"): + result.__setstate__( + state[0] + ) # preserve position so advanced iterators don't reset + return result @evaluate.register(builtins.slice) @@ -325,44 +314,6 @@ def _evaluate_slice(expr, **kwargs): return builtins.slice(*evaluate((expr.start, expr.stop, expr.step))) -@evaluate.register(builtins.map) -def _evaluate_map(expr, **kwargs): - _, (fn, iterator) = expr.__reduce__() - return builtins.map(evaluate(fn), evaluate(iterator)) - - -@evaluate.register(builtins.filter) -def _evaluate_filter(expr, **kwargs): - _, (fn_or_none, iterator) = expr.__reduce__() - return builtins.filter(evaluate(fn_or_none), evaluate(iterator)) - - -@evaluate.register(builtins.zip) -def _evaluate_zip(expr, **kwargs): - iterators_strict = expr.__reduce__()[1:] - strict = len(iterators_strict) == 2 and iterators_strict[1] is True - values = iterators_strict[0] - return builtins.zip(*[evaluate(v) for v in values], strict=strict) - - -@evaluate.register(builtins.enumerate) -def _evaluate_enumerate(expr, **kwargs): - _, (iterator, start) = expr.__reduce__() - return builtins.enumerate(evaluate(iterator), start=start) - - -@evaluate.register(builtins.reversed) -def _evaluate_reversed(expr, **kwargs): - _, (seq,) = expr.__reduce__() - return builtins.reversed(evaluate(seq)) - - -@evaluate.register(itertools.product) -def _evaluate_product(expr: itertools.product, **kwargs): - _, (iterables, repeat) = expr.__reduce__() - return itertools.product(*[evaluate(it) for it in iterables], repeat=repeat) - - def _simple_type(tp: type) -> type: """Convert a type object into a type that can be dispatched on.""" if isinstance(tp, typing.TypeVar): diff --git a/tests/test_ops_semantics.py b/tests/test_ops_semantics.py index 78179d33b..63dfe91ca 100644 --- a/tests/test_ops_semantics.py +++ b/tests/test_ops_semantics.py @@ -908,3 +908,167 @@ def f(self): b = B() with handler({b.f: lambda: "*B*"}): assert b.f() == "*B*" + + +# --- evaluate over built-in iterators --------------------------------------- + +# Free variables shared by the parametrized iterator tests. Iterators are built +# *outside* a handler (via the lambdas below) so their elements are Terms, then +# substituted under ``_IT_INTP`` during ``evaluate``. +_itx = defop(int, name="itx") +_ity = defop(int, name="ity") +_itz = defop(int, name="itz") +_IT_INTP = {_itx: lambda: 10, _ity: lambda: 20, _itz: lambda: 30} + + +class _CustomSeq: + def __len__(self): + return 3 + + def __getitem__(self, i): + return [1, 2, 3][i] + + +# (make_iterator, expected_items, expected_type) — iterator type is preserved. +ITERATOR_CASES = [ + pytest.param(lambda: iter([_itx(), _ity()]), [10, 20], type(iter([])), id="list"), + pytest.param(lambda: iter((_itx(), _ity())), [10, 20], type(iter(())), id="tuple"), + pytest.param(lambda: iter("ab"), ["a", "b"], type(iter("")), id="str"), + pytest.param(lambda: iter(b"ab"), [97, 98], type(iter(b"")), id="bytes"), + pytest.param(lambda: iter(range(3)), [0, 1, 2], type(iter(range(0))), id="range"), + pytest.param( + lambda: reversed([_itx(), _ity()]), + [20, 10], + type(reversed([])), + id="reversed-list", + ), + pytest.param( + lambda: reversed(range(3)), + [2, 1, 0], + type(reversed(range(0))), + id="reversed-range", + ), + pytest.param( + lambda: reversed(_CustomSeq()), [3, 2, 1], reversed, id="reversed-seq" + ), + pytest.param(lambda: map(lambda v: v, [_itx(), _ity()]), [10, 20], map, id="map"), + pytest.param( + lambda: map(lambda a, b: a + b, [_itx()], [_ity()]), [30], map, id="map-multi" + ), + pytest.param( + lambda: filter(lambda v: True, [_itx(), _ity()]), [10, 20], filter, id="filter" + ), + pytest.param(lambda: zip([_itx()], [_ity()]), [(10, 20)], zip, id="zip"), + pytest.param( + lambda: zip([_itx()], [_ity()], strict=True), [(10, 20)], zip, id="zip-strict" + ), + pytest.param(lambda: zip(), [], zip, id="zip-empty"), + pytest.param( + lambda: enumerate([_itx(), _ity()]), + [(0, 10), (1, 20)], + enumerate, + id="enumerate", + ), + pytest.param( + lambda: enumerate([_itx()], start=5), [(5, 10)], enumerate, id="enumerate-start" + ), +] + + +@pytest.mark.parametrize("make,expected,expected_type", ITERATOR_CASES) +def test_evaluate_iterator(make, expected, expected_type): + """evaluate substitutes free vars inside a fresh built-in iterator and + preserves the iterator type (laziness).""" + it = make() + with handler(_IT_INTP): + result = evaluate(it) + assert type(result) is expected_type + assert list(result) == expected + + +# set/dict iterators substitute their elements but do not preserve the exact +# iterator type (they reduce to a list iterator), and set order is arbitrary. +SET_DICT_CASES = [ + pytest.param(lambda: iter({_itx(), _ity()}), [10, 20], id="set"), + pytest.param(lambda: iter({_itx(): 1, _ity(): 2}.keys()), [10, 20], id="dict-keys"), + pytest.param( + lambda: iter({1: _itx(), 2: _ity()}.values()), [10, 20], id="dict-values" + ), + pytest.param(lambda: iter({_itx(): _ity()}.items()), [(10, 20)], id="dict-items"), +] + + +@pytest.mark.parametrize("make,expected", SET_DICT_CASES) +def test_evaluate_set_dict_iterator(make, expected): + """evaluate substitutes the elements of set/dict iterators.""" + it = make() + with handler(_IT_INTP): + assert sorted(evaluate(it)) == expected + + +# (make_iterator, expected_remaining) after advancing the iterator by one. +ADVANCED_CASES = [ + pytest.param(lambda: iter([_itx(), _ity(), _itz()]), [20, 30], id="list"), + pytest.param( + lambda: map(lambda v: v, [_itx(), _ity(), _itz()]), [20, 30], id="map" + ), + pytest.param( + lambda: filter(lambda v: True, [_itx(), _ity(), _itz()]), [20, 30], id="filter" + ), + pytest.param( + lambda: enumerate([_itx(), _ity(), _itz()]), [(1, 20), (2, 30)], id="enumerate" + ), + pytest.param(lambda: reversed([_itz(), _ity(), _itx()]), [20, 30], id="reversed"), + pytest.param( + lambda: zip([_itx(), _ity(), _itz()], [_itz(), _ity(), _itx()]), + [(20, 20), (30, 10)], + id="zip", + ), +] + + +@pytest.mark.parametrize("make,expected", ADVANCED_CASES) +def test_evaluate_advanced_iterator(make, expected): + """Advanced iterators preserve their position: evaluate yields the + substituted remaining items, it does not reset to the start.""" + it = make() + next(it) # advance past the first element + with handler(_IT_INTP): + assert list(evaluate(it)) == expected + + +FVSOF_CASES = [ + pytest.param(lambda: iter([_itx(), _ity()]), id="list"), + pytest.param(lambda: iter((_itx(), _ity())), id="tuple"), + pytest.param(lambda: iter({_itx(), _ity()}), id="set"), + pytest.param(lambda: iter({_itx(): 1, _ity(): 2}.keys()), id="dict-keys"), + pytest.param(lambda: reversed([_itx(), _ity()]), id="reversed"), + pytest.param(lambda: map(lambda v: v, [_itx(), _ity()]), id="map"), + pytest.param(lambda: map(lambda a, b: (a, b), [_itx()], [_ity()]), id="map-multi"), + pytest.param(lambda: filter(lambda v: True, [_itx(), _ity()]), id="filter"), + pytest.param(lambda: zip([_itx()], [_ity()]), id="zip"), + pytest.param(lambda: enumerate([_itx(), _ity()]), id="enumerate"), +] + + +@pytest.mark.parametrize("make", FVSOF_CASES) +def test_fvsof_iterator(make): + """fvsof finds the free variables inside a fresh built-in iterator.""" + assert fvsof(make()) >= {_itx, _ity} + + +TYPEOF_CASES = [ + pytest.param(lambda: iter([_itx()]), type(iter([])), id="list"), + pytest.param(lambda: iter((_itx(),)), type(iter(())), id="tuple"), + pytest.param(lambda: map(lambda v: v, [_itx()]), map, id="map"), + pytest.param(lambda: filter(lambda v: True, [_itx()]), filter, id="filter"), + pytest.param(lambda: zip([_itx()], [_itx()]), zip, id="zip"), + pytest.param(lambda: enumerate([_itx()]), enumerate, id="enumerate"), + pytest.param(lambda: reversed([_itx()]), type(reversed([])), id="reversed"), +] + + +@pytest.mark.parametrize("make,expected_type", TYPEOF_CASES) +def test_typeof_iterator(make, expected_type): + """typeof of a built-in iterator is its (reconstructed) iterator type.""" + assert typeof(make()) is expected_type From b80ea472059c192f71d6619be604f9e816123a2f Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 10:18:19 -0400 Subject: [PATCH 5/9] add failing test --- tests/test_internals_product_n.py | 43 ++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/test_internals_product_n.py b/tests/test_internals_product_n.py index 331019824..ba1f489c5 100644 --- a/tests/test_internals_product_n.py +++ b/tests/test_internals_product_n.py @@ -1,4 +1,6 @@ -from effectful.internals.product_n import argsof, productN +from collections.abc import Iterable + +from effectful.internals.product_n import Product, argsof, productN from effectful.internals.unification import Box from effectful.ops.semantics import apply, coproduct, evaluate, handler from effectful.ops.syntax import defop @@ -158,3 +160,42 @@ def add[T](x: T, y: T) -> T: assert result1.values(i) == result2.values(i) == 2 assert result1.values(s) == result2.values(s) == "aa" + + +def test_evaluate_iterator_under_product(): + """Evaluating a builtin iterator under a ``productN`` analysis must not crash. + + ``productN`` that binds the universal ``apply`` operation (as the type/cast + analysis in ``defdata`` does) intercepts *every* operation application and + returns its result wrapped in a :class:`Product`. So under such an + interpretation any sub-term evaluates to a ``Product``. + + ``evaluate`` reconstructs a builtin iterator by calling its constructor on + its evaluated source iterables (``ctor(*evaluate(args))`` in + ``_evaluate_iterator``). A ``map`` eagerly stores ``iter(s())`` -- an + iterator *term* -- as its source. Reconstruction therefore evaluates that + inner term to a ``Product`` and calls ``map(f, Product)``; since ``Product`` + is not iterable, this raises ``TypeError: 'Product' object is not iterable``. + + This reproduces the bug in isolation: a builtin iterator wrapping a term, + evaluated under a product interpretation, should evaluate successfully + rather than crashing in iterator reconstruction. + """ + + @defop + def s() -> Iterable[int]: + raise NotHandled + + def apply_type(op, *args, **kwargs): + return Box(op.__type_rule__(*args, **kwargs)) + + typ = defop(object, name="typ") + cast = defop(object, name="cast") + analysis = productN({typ: {apply: apply_type}, cast: {apply: apply_type}}) + + # ``map`` eagerly calls ``iter(s())``, storing an iterator term as its source. + m = map(lambda v: v, s()) + + # Currently raises ``TypeError: 'Product' object is not iterable``. + result = evaluate(m, intp=analysis) + assert isinstance(result, Product) From 7d08f4750ac2732e249ab112c25f1fb21a79fad8 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 12:07:31 -0400 Subject: [PATCH 6/9] pass through op --- effectful/ops/semantics.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/effectful/ops/semantics.py b/effectful/ops/semantics.py index 75d607a72..6956f829f 100644 --- a/effectful/ops/semantics.py +++ b/effectful/ops/semantics.py @@ -301,10 +301,23 @@ def _evaluate_iterator(expr, **kwargs): ctor, args, *state = expr.__reduce__() except (TypeError, AttributeError): return expr # un-reducible iterators are opaque, like any object we can't recurse into - result = ctor(*evaluate(args)) + + if ctor is iter: + result = ctor(*evaluate(args)) + else: + from effectful.internals.unification import nested_type + + ExprType = nested_type(expr).value + + @Operation.define + def ctor_op(*args) -> ExprType: + return ctor(*args) + + result = ctor_op(*evaluate(args)) + if state and state[0] is not None and hasattr(result, "__setstate__"): result.__setstate__( - state[0] + evaluate(state[0]) ) # preserve position so advanced iterators don't reset return result From bb48c04305ca8fe2feeffee74ff89d6dd2ddc452 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 14:27:20 -0400 Subject: [PATCH 7/9] nested_type --- effectful/internals/unification.py | 102 ++++++++++++++++++++++++++++ tests/test_internals_unification.py | 63 +++++++++++++++++ tests/test_ops_semantics.py | 11 ++- 3 files changed, 173 insertions(+), 3 deletions(-) diff --git a/effectful/internals/unification.py b/effectful/internals/unification.py index ad022a63b..50ffeb0aa 100644 --- a/effectful/internals/unification.py +++ b/effectful/internals/unification.py @@ -1099,6 +1099,108 @@ def _(value: str | bytes | range | None): return Box(type(value)) +def _iterable_element_type(value: collections.abc.Iterable) -> TypeExpression: + """Element type of an iterable *value*. + + Infers the value's type with :func:`nested_type` and unifies it against + ``Iterable[E]`` to recover the element type ``E``. Raises ``TypeError`` (the + element is itself a :class:`~effectful.ops.types.Term`) or ``KeyError`` (the + iterable is empty/bare so ``E`` is unbound) when the element type cannot be + determined; callers fall back to the bare iterator type in that case. + """ + if isinstance(value, str | bytes): + # str/bytes are atomic to nested_type; their elements share their type. + return type(value) + E = typing.TypeVar("E") + try: + return unify(collections.abc.Iterable[E], nested_type(value).value)[E] # type: ignore[return-value, valid-type] + except KeyError: + raise TypeError("Could not resolve concrete element type") + + +@nested_type.register +def _(value: collections.abc.Iterator): + try: + ctor, args, *state = value.__reduce__() # type: ignore[misc] + _ = [nested_type(arg).value for arg in args] + except (TypeError, AttributeError): + return Box(type(value)) # un-reducible iterators are opaque + + if ctor is iter or ctor is reversed: + # ``args[0]`` is the underlying iterable. ``reversed([...])`` is a + # ``list_reverseiterator`` -- not a ``reversed`` instance -- so it is + # dispatched here by ctor rather than by a type-keyed registration. + try: + return Box(collections.abc.Iterator[_iterable_element_type(args[0])]) # type: ignore[misc] + except TypeError: + return Box(type(value)) + else: + return Box(type(value)) + + +@nested_type.register(map) +def _(value): + # ``map(f, *iterables)`` yields ``f(*items)``, so the element type is the + # return type of ``f`` rather than the source element types. + _ctor, (func, *sources), *_state = value.__reduce__() + try: + if typing.get_args(nested_type(func).value) and sources: + Xs = [typing.TypeVar(f"X{i}") for i in range(len(sources))] + Y = typing.TypeVar("Y") + typ = ( + collections.abc.Callable[Xs, Y], + *[collections.abc.Iterable[Xi] for Xi in Xs], + ) + subtyp = ( + nested_type(func).value, + *[nested_type(source).value for source in sources], + ) + subs = unify(typ, subtyp) + if Y not in subs: + raise TypeError("Could not resolve concrete return type") + return Box(collections.abc.Iterator[subs[Y]]) + else: # un-annotated function: fall back to the bare iterator type + return nested_type.dispatch(collections.abc.Iterator)(value) + except TypeError: + return nested_type.dispatch(collections.abc.Iterator)(value) + + +@nested_type.register(filter) +def _(value): + # ``filter`` preserves the elements of its source iterable. + _ctor, (_func, source), *_state = value.__reduce__() + try: + return Box(collections.abc.Iterator[_iterable_element_type(source)]) + except TypeError: + return Box(type(value)) + + +@nested_type.register(zip) +def _(value): + # __reduce__() is (sources,) or (sources, strict); () if empty + _ctor, *rest = value.__reduce__() + sources = rest[0] if rest else () + if not sources: + return Box(collections.abc.Iterator[tuple]) + + try: + elt_type = tuple[tuple(_iterable_element_type(s) for s in sources)] + except TypeError: + elt_type = tuple[tuple(typing.Any for _ in sources)] + return Box(collections.abc.Iterator[elt_type]) + + +@nested_type.register(enumerate) +def _(value): + # ``enumerate`` yields ``(index, element)`` pairs. + _ctor, (source, _start), *_state = value.__reduce__() + try: + elt_type = tuple[int, _iterable_element_type(source)] + except TypeError: + elt_type = tuple[int, typing.Any] + return Box(collections.abc.Iterator[elt_type]) + + def freetypevars(typ) -> collections.abc.Set[TypeVariable]: """ Return a set of free type variables in the given type expression. diff --git a/tests/test_internals_unification.py b/tests/test_internals_unification.py index fe3a9ed06..cd7388514 100644 --- a/tests/test_internals_unification.py +++ b/tests/test_internals_unification.py @@ -788,6 +788,69 @@ def test_nested_type(value, expected): assert canonicalize(result) == canonicalize(expected) +def _annotated_to_str(x: int) -> str: + return str(x) + + +def _annotated_add(a: int, b: int) -> float: + return float(a + b) + + +@pytest.mark.parametrize( + "make,expected", + [ + # iter() infers the element type from the underlying iterable. + (lambda: iter([1, 2, 3]), collections.abc.Iterator[int]), + (lambda: iter((1, 2)), collections.abc.Iterator[int]), + # str/bytes are atomic: their elements share their own type. + (lambda: iter("ab"), collections.abc.Iterator[str]), + # Empty/bare iterables can't infer an element type -> bare iterator type. + (lambda: iter([]), type(iter([]))), + # map() element type is the *return* type of the mapped function. + (lambda: map(_annotated_to_str, [1, 2]), collections.abc.Iterator[str]), + ( + lambda: map(_annotated_add, [1], [2]), + collections.abc.Iterator[float], + ), + # An un-annotated function gives no return type -> bare iterator type. + (lambda: map(lambda v: v, [1, 2]), map), + # filter() preserves the source element type. + (lambda: filter(None, [1, 2, 3]), collections.abc.Iterator[int]), + (lambda: filter(lambda v: True, ["a", "b"]), collections.abc.Iterator[str]), + # zip() pairs the element types of each source. + (lambda: zip([1], ["a"]), collections.abc.Iterator[tuple[int, str]]), + ( + lambda: zip([1], [2], strict=True), + collections.abc.Iterator[tuple[int, int]], + ), + (lambda: zip(), collections.abc.Iterator[tuple]), + # enumerate() yields (int, element) pairs. + (lambda: enumerate([1, 2]), collections.abc.Iterator[tuple[int, int]]), + ( + lambda: enumerate(["a"], start=5), + collections.abc.Iterator[tuple[int, str]], + ), + # reversed([...]) is a list_reverseiterator (not a `reversed` instance), + # so it is recognized by its __reduce__ ctor; reversed(range(...)) + # reduces with ctor `iter`. + (lambda: reversed([1, 2]), collections.abc.Iterator[int]), + (lambda: reversed(range(3)), collections.abc.Iterator[int]), + ], +) +def test_nested_type_iterator(make, expected): + """nested_type infers element types for builtin iterator wrappers.""" + assert canonicalize(nested_type(make()).value) == canonicalize(expected) + + +def test_nested_type_iterator_advanced_preserves_element_type(): + """Advancing an iterator does not change its inferred element type.""" + it = map(_annotated_to_str, [1, 2, 3]) + next(it) + assert canonicalize(nested_type(it).value) == canonicalize( + collections.abc.Iterator[str] + ) + + def test_nested_type_typeddict_str_keys_mixed_values(): """Dicts with str keys and heterogeneous value types produce a TypedDict.""" value = {"name": "Alice", "age": 30} diff --git a/tests/test_ops_semantics.py b/tests/test_ops_semantics.py index 63dfe91ca..4543a8e45 100644 --- a/tests/test_ops_semantics.py +++ b/tests/test_ops_semantics.py @@ -3,7 +3,7 @@ import functools import itertools import logging -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping from typing import Annotated, Any, Literal, Union import pytest @@ -1057,13 +1057,18 @@ def test_fvsof_iterator(make): assert fvsof(make()) >= {_itx, _ity} +# These iterators wrap Term elements, whose element type cannot be inferred. +# For most builtins ``nested_type`` falls back to the bare iterator type, so +# typeof reports the exact reconstructed iterator class. ``zip``/``enumerate`` +# instead keep their structurally-known shape (tuple arity / the int index), so +# ``nested_type`` yields ``Iterator[tuple[...]]`` and typeof reports ``Iterator``. TYPEOF_CASES = [ pytest.param(lambda: iter([_itx()]), type(iter([])), id="list"), pytest.param(lambda: iter((_itx(),)), type(iter(())), id="tuple"), pytest.param(lambda: map(lambda v: v, [_itx()]), map, id="map"), pytest.param(lambda: filter(lambda v: True, [_itx()]), filter, id="filter"), - pytest.param(lambda: zip([_itx()], [_itx()]), zip, id="zip"), - pytest.param(lambda: enumerate([_itx()]), enumerate, id="enumerate"), + pytest.param(lambda: zip([_itx()], [_itx()]), Iterator, id="zip"), + pytest.param(lambda: enumerate([_itx()]), Iterator, id="enumerate"), pytest.param(lambda: reversed([_itx()]), type(reversed([])), id="reversed"), ] From 1edd65712936ab3d7f1d6d94b31a9f76fd5c1532 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 15:10:25 -0400 Subject: [PATCH 8/9] fix bug --- effectful/ops/semantics.py | 28 ++++++++++++++++++---------- tests/test_ops_syntax.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/effectful/ops/semantics.py b/effectful/ops/semantics.py index 6956f829f..2064146bd 100644 --- a/effectful/ops/semantics.py +++ b/effectful/ops/semantics.py @@ -302,20 +302,28 @@ def _evaluate_iterator(expr, **kwargs): except (TypeError, AttributeError): return expr # un-reducible iterators are opaque, like any object we can't recurse into - if ctor is iter: - result = ctor(*evaluate(args)) - else: - from effectful.internals.unification import nested_type + from effectful.internals.unification import nested_type - ExprType = nested_type(expr).value + ExprType = nested_type(expr).value - @Operation.define - def ctor_op(*args) -> ExprType: - return ctor(*args) + @Operation.define + def ctor_op(*args) -> ExprType: + return ctor(*args) - result = ctor_op(*evaluate(args)) + # Reify through ``ctor_op`` rather than calling the live constructor: when + # the evaluated args contain a ``Term`` the result is a structural ``Term`` + # node whose source iterables stay traversable (so ``fvsof`` finds their + # free variables and term-reconstruction interpretations can rewrite them). + # For fully concrete args ``ctor_op``'s default rule rebuilds the live + # iterator, preserving laziness. + result = ctor_op(*evaluate(args)) - if state and state[0] is not None and hasattr(result, "__setstate__"): + if ( + not isinstance(result, Term) + and state + and state[0] is not None + and hasattr(result, "__setstate__") + ): result.__setstate__( evaluate(state[0]) ) # preserve position so advanced iterators don't reset diff --git a/tests/test_ops_syntax.py b/tests/test_ops_syntax.py index 185b6132e..d8b16a06d 100644 --- a/tests/test_ops_syntax.py +++ b/tests/test_ops_syntax.py @@ -525,6 +525,36 @@ def cons_iterable(*args: int) -> Iterable[int]: assert list(tm.args) == [1, 2, 3] +def test_defdata_preserves_free_vars_in_iterator_arg(): + """Free variables inside a builtin iterator argument must survive term + construction. + + A ``map`` object is a lazy iterator that can close over free variables -- + here ``x`` via the element ``x()``. When such an iterator is passed as an + argument to an operation, ``defdata`` reconstructs the term and should + preserve those free variables. Currently the iterator is reconstructed as an + opaque iterator that no longer references ``x``, so ``x`` is silently dropped + from the constructed term's free variables (and can no longer be + substituted). + """ + + @defop + def g(xs: Iterable[int]) -> int: + raise NotHandled + + x = defop(int, name="x") + + def keep(v: int) -> int: + return v + + # Sanity: the raw map closes over ``x``. + assert x in fvsof(map(keep, [x()])) + + # Passing it to ``g`` must not drop ``x`` from the term's free variables. + term = g(map(keep, [x()])) + assert x in fvsof(term) + + def test_defstream_1(): x = defop(int, name="x") y = defop(int, name="y") From 88c96bee458bcd92a6415b76ee79742c85128a9b Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 10 Jun 2026 22:12:08 -0400 Subject: [PATCH 9/9] lint --- effectful/internals/unification.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/effectful/internals/unification.py b/effectful/internals/unification.py index 50ffeb0aa..968712e02 100644 --- a/effectful/internals/unification.py +++ b/effectful/internals/unification.py @@ -1121,7 +1121,10 @@ def _iterable_element_type(value: collections.abc.Iterable) -> TypeExpression: @nested_type.register def _(value: collections.abc.Iterator): try: - ctor, args, *state = value.__reduce__() # type: ignore[misc] + reduced = value.__reduce__() + if isinstance(reduced, str): + return Box(type(value)) # reduced to a global name; opaque + ctor, args, *state = reduced _ = [nested_type(arg).value for arg in args] except (TypeError, AttributeError): return Box(type(value)) # un-reducible iterators are opaque