Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion python/tvm/relax/frontend/onnx/onnx_frontend.py
Original file line number Diff line number Diff line change
Expand Up @@ -7247,7 +7247,14 @@ def _construct_nodes(self, graph: onnx.onnx_ml_pb2.GraphProto):
and isinstance(inp.ty, relax.ShapeType)
and op_name not in shape_compatible_ops
):
raise ValueError(f"Node {node.name} cannot handle ShapeExpr inputs.")
if op_name == "Add":
tensor = self.bb.normalize(relax.op.shape_to_tensor(inp))
if inp.ty.ndim < 0:
rank = tirx.Var("shape_rank", "int64")
tensor = self.bb.match_cast(tensor, relax.TensorType([rank], "int64"))
inputs[i] = tensor
else:
raise ValueError(f"Node {node.name} cannot handle ShapeExpr inputs.")
try:
op = self._convert_operator(op_name, inputs, attr, self.opset)
except Exception as err: # pylint: disable=broad-exception-caught
Expand Down
3 changes: 2 additions & 1 deletion python/tvm/relax/op/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

from collections.abc import Callable

import numpy as np
import tvm_ffi

import tvm
Expand Down Expand Up @@ -459,7 +460,7 @@ def relax_shape_to_tensor(shape_tuple: tvm_ffi.Shape) -> tvm_tirx.runtime.Tensor
shape_tuple: tvm_ffi.Shape
Shape tuple that we want to convert to Tensor at runtime
"""
return tvm_tirx.runtime.tensor([int(v) for v in shape_tuple])
return tvm_tirx.runtime.tensor(np.asarray([int(v) for v in shape_tuple], dtype="int64"))


@tvm_ffi.register_global_func("relax.run.print")
Expand Down
28 changes: 27 additions & 1 deletion python/tvm/relax/transform/legalize_ops/binary.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@
# pylint: disable=invalid-name
"""Default legalization function for binary operators."""

from tvm import topi
from tvm import te, topi
from tvm_tirx import tirx
from tvm_tirx.ir import Call
from tvm_tirx.script.ir_builder import IRBuilder
from tvm_tirx.tirx.script import ir_builder as T

from ...block_builder import BlockBuilder
from ...expr import Expr
Expand All @@ -31,6 +34,27 @@
)


def _dynamic_vector_add(a, b):
"""Add vectors with runtime broadcast checks for their lengths."""
a_len, b_len = a.shape[0], b.shape[0]
out_len = tirx.Select(a_len == 1, b_len, a_len)
out_dtype = (a(0) + b(0)).ty.dtype

def compute(ins, outs):
with IRBuilder() as ib:
T.assert_(
tirx.any(a_len == b_len, a_len == 1, b_len == 1),
("ValueError", ["incompatible broadcast lengths"]),
)
with T.serial(0, out_len) as i:
a_index = tirx.if_then_else(a_len == 1, 0, i)
b_index = tirx.if_then_else(b_len == 1, 0, i)
T.tensor_store(outs[0], (i,), ins[0][a_index] + ins[1][b_index])
return ib.get()

return te.extern((out_len,), [a, b], compute, dtype=str(out_dtype), name="dynamic_vector_add")


def _binary(te_func: TEFunc) -> LegalizeFunc:
"""A common wrapper util for the legalization of binary operators.

Expand All @@ -41,6 +65,8 @@ def _binary(te_func: TEFunc) -> LegalizeFunc:
def binary_call_te(bb: BlockBuilder, call: Call) -> Expr:
# To simplify the created Function, we first check if arg1 is a constant scalar.
# If it is not, we then check if arg0 is a constant scalar.
if te_func is topi.add and call.ty.shape is None:
return bb.call_te(_dynamic_vector_add, *call.args)
arg0 = call.args[0]
arg1 = _try_convert_to_scalar_const(call.args[1])
if _is_relax_expr(arg1):
Expand Down
3 changes: 3 additions & 0 deletions src/relax/op/op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1508,6 +1508,9 @@ Type ReturnShapeToTensorType(const CallObj* call_node) {
const auto* ty = GetTypeAs<ShapeTypeObj>(call->args[0]);
TVM_FFI_ICHECK(ty);
int32_t ndim = ty->ndim;
if (ndim == kUnknownNDim) {
return TensorType(PrimType::Int(64), /*ndim=*/1);
}
return TensorType(ShapeExpr({PrimExpr(ndim)}), PrimType::Int(64));
}

Expand Down
9 changes: 9 additions & 0 deletions src/relax/transform/legalize_ops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,15 @@ class LegalizeMutator : public ExprMutator {
}();
bool ret_shape_defined = KnowAllShapeValues(GetType(visited_call));
if (!is_data_dependent_op && !ret_shape_defined) {
// The dynamic one-dimensional add legalization computes and checks
// its broadcast shape at runtime.
if (op.same_as(Op::Get("relax.add")) && visited_call->args.size() == 2) {
const auto* lhs = GetType(visited_call->args[0]).as<TensorTypeObj>();
const auto* rhs = GetType(visited_call->args[1]).as<TensorTypeObj>();
if (lhs && rhs && lhs->ndim == 1 && rhs->ndim == 1) {
return true;
}
}
// This operator cannot be legalized, because legalization by
// default requires the output shape. The exception is
// data-dependent operators (e.g. `R.dynamic_strided_slice`),
Expand Down
96 changes: 96 additions & 0 deletions tests/python/relax/test_frontend_onnx.py
Original file line number Diff line number Diff line change
Expand Up @@ -4052,6 +4052,102 @@ def main(data: R.Tensor((3, 4, 5, 6), dtype="float32")) -> R.Shape([3, 4, 5, 6])
tvm_tirx.ir.assert_structural_equal(tvm_model, Expected)


@pytest.mark.parametrize("symbolic", [False, True], ids=["static", "symbolic"])
@pytest.mark.parametrize("shape_input", ["left", "right", "both"])
def test_shape_output_as_add_input(symbolic, shape_input):
input_shape = ["n", 3] if symbolic else [2, 3]
add_inputs = {
"left": ["shape", "offset"],
"right": ["offset", "shape"],
"both": ["shape", "shape"],
}[shape_input]
graph = helper.make_graph(
[
helper.make_node("Shape", ["data"], ["shape"]),
helper.make_node("Add", add_inputs, ["output"]),
],
"shape_add_test",
inputs=[helper.make_tensor_value_info("data", TensorProto.FLOAT, input_shape)],
outputs=[helper.make_tensor_value_info("output", TensorProto.INT64, [2])],
initializer=[numpy_helper.from_array(np.array([1, 1], dtype="int64"), name="offset")],
)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
model.ir_version = 10

mod = from_onnx(model, keep_params_in_input=False)
vm = relax.VirtualMachine(relax.build(mod, target="llvm"), tvm_tirx.cpu())
for rows in [2, 5] if symbolic else [2]:
data = tvm_tirx.runtime.tensor(np.zeros((rows, 3), dtype="float32"))
actual = vm["main"](data).numpy()
expected = [rows * 2, 6] if shape_input == "both" else [rows + 1, 4]
np.testing.assert_array_equal(actual, np.array(expected, dtype="int64"))


def test_unknown_rank_shape_output_as_add_input():
graph = helper.make_graph(
[
helper.make_node("Shape", ["data"], ["shape"]),
helper.make_node("Add", ["shape", "offset"], ["output"]),
],
"unknown_rank_shape_add_test",
inputs=[helper.make_tensor_value_info("data", TensorProto.FLOAT, None)],
outputs=[helper.make_tensor_value_info("output", TensorProto.INT64, [None])],
initializer=[numpy_helper.from_array(np.array([1], dtype="int64"), name="offset")],
)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
model.ir_version = 10

mod = from_onnx(model, keep_params_in_input=False)
ops = collect_relax_call_ops(mod["main"])
assert "relax.shape_of" in ops
assert "relax.shape_to_tensor" in ops
assert "relax.add" in ops
vm = relax.VirtualMachine(relax.build(mod, target="llvm"), tvm_tirx.cpu())
for shape in [(), (3,), (2, 3)]:
data = tvm_tirx.runtime.tensor(np.zeros(shape, dtype="float32"))
actual = vm["main"](data).numpy()
np.testing.assert_array_equal(actual, np.array(shape, dtype="int64") + 1)


def test_unknown_rank_shape_outputs_as_add_inputs():
graph = helper.make_graph(
[
helper.make_node("Shape", ["a"], ["a_shape"]),
helper.make_node("Shape", ["b"], ["b_shape"]),
helper.make_node("Add", ["a_shape", "b_shape"], ["output"]),
],
"unknown_rank_shapes_add_test",
inputs=[
helper.make_tensor_value_info("a", TensorProto.FLOAT, None),
helper.make_tensor_value_info("b", TensorProto.FLOAT, None),
],
outputs=[helper.make_tensor_value_info("output", TensorProto.INT64, [None])],
)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
model.ir_version = 10

mod = from_onnx(model, keep_params_in_input=False)
vm = relax.VirtualMachine(relax.build(mod, target="llvm"), tvm_tirx.cpu())
for shape_a, shape_b in [
((), ()),
((), (4,)),
((4,), ()),
((2, 3), (4,)),
((4,), (2, 3)),
((2, 3), (4, 5)),
]:
a = tvm_tirx.runtime.tensor(np.zeros(shape_a, dtype="float32"))
b = tvm_tirx.runtime.tensor(np.zeros(shape_b, dtype="float32"))
actual = vm["main"](a, b).numpy()
expected = np.array(shape_a, dtype="int64") + np.array(shape_b, dtype="int64")
np.testing.assert_array_equal(actual, expected)

a = tvm_tirx.runtime.tensor(np.zeros((2, 3, 4), dtype="float32"))
b = tvm_tirx.runtime.tensor(np.zeros((4, 5), dtype="float32"))
with pytest.raises(ValueError, match="broadcast"):
vm["main"](a, b)


def test_shape_scalar_input():
# A rank-0 input has a known, empty static shape. It used to be imported as
# a runtime R.shape_of because an empty ShapeExpr is falsy, which made the
Expand Down
90 changes: 90 additions & 0 deletions tests/python/relax/test_relax_operators.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,96 @@ def test_op_shape_to_tensor():
assert np.array_equal(outs.numpy(), np.array([3, 2]))


def test_shape_to_tensor_unknown_rank_has_unknown_length():
result_type = ShapeToTensorTest["const_shape"].body.ty
assert result_type.ndim == 1
assert result_type.shape is None


def test_shape_to_tensor_empty_shape_dtype():
result = run_cpu(ShapeToTensorTest, "const_shape", tvm_ffi.Shape([])).numpy()
assert result.shape == (0,)
assert result.dtype == np.dtype("int64")


@pytest.mark.parametrize(
"a_values,b_values",
[
([2, 3], [4]),
([4], [2, 3]),
([2, 3], [4, 5]),
([], [4]),
([4], []),
([], []),
],
)
def test_add_independent_symbolic_lengths(a_values, b_values):
@I.ir_module
class Module:
@R.function
def main(a: R.Tensor((m,), "int64"), b: R.Tensor((n,), "int64")):
return R.add(a, b)

a_np = np.array(a_values, dtype="int64")
b_np = np.array(b_values, dtype="int64")
a = tvm_tirx.runtime.tensor(a_np)
b = tvm_tirx.runtime.tensor(b_np)
np.testing.assert_array_equal(run_cpu(Module, "main", a, b).numpy(), a_np + b_np)


@pytest.mark.parametrize("lengths", [(2, 3), (0, 2), (2, 0)])
def test_add_independent_symbolic_lengths_incompatible(lengths):
@I.ir_module
class Module:
@R.function
def main(a: R.Tensor((m,), "int64"), b: R.Tensor((n,), "int64")):
return R.add(a, b)

ex = relax.build(Module, target="llvm")
vm = relax.VirtualMachine(ex, tvm_tirx.cpu())
a = tvm_tirx.runtime.tensor(np.arange(lengths[0], dtype="int64"))
b = tvm_tirx.runtime.tensor(np.arange(lengths[1], dtype="int64"))
with pytest.raises(ValueError, match="broadcast"):
vm["main"](a, b)


@pytest.mark.parametrize("dtype", ["float16", "float32", "int32"])
def test_add_independent_symbolic_lengths_dtype(dtype):
@I.ir_module
class Module:
@R.function
def main(a: R.Tensor((m,), dtype), b: R.Tensor((n,), dtype)):
return R.add(a, b)

a_np = np.array([2, -3], dtype=dtype)
b_np = np.array([1], dtype=dtype)
a = tvm_tirx.runtime.tensor(a_np)
b = tvm_tirx.runtime.tensor(b_np)
np.testing.assert_array_equal(run_cpu(Module, "main", a, b).numpy(), a_np + b_np)


@pytest.mark.parametrize("left_dtype,right_dtype", [("bool", "float32"), ("float32", "bool")])
def test_add_independent_symbolic_lengths_mixed_bool(left_dtype, right_dtype):
@I.ir_module
class Dynamic:
@R.function
def main(a: R.Tensor((m,), left_dtype), b: R.Tensor((n,), right_dtype)):
return R.add(a, b)

@I.ir_module
class Static:
@R.function
def main(a: R.Tensor((2,), left_dtype), b: R.Tensor((1,), right_dtype)):
return R.add(a, b)

a = tvm_tirx.runtime.tensor(np.array([0, 1], dtype=left_dtype))
b = tvm_tirx.runtime.tensor(np.array([1], dtype=right_dtype))
expected = run_cpu(Static, "main", a, b).numpy()
actual = run_cpu(Dynamic, "main", a, b).numpy()
np.testing.assert_array_equal(actual, expected)
assert actual.dtype == expected.dtype


def test_op_call_pure_packed():
@tvm_tirx.script.ir_module
class CallPureTest:
Expand Down
Loading