Skip to content

Commit 8f01d0d

Browse files
[Relax][Frontend][ONNX] Add QLinearMatMul operator converter (#20951)
This PR adds support for the ONNX `QLinearMatMul` operator in the Relax ONNX frontend, resolving #20958. ### Key Changes - In `python/tvm/relax/frontend/onnx/onnx_frontend.py`: - Implemented `QLinearMatMul` op converter supporting opset 10 and 21. - Widens inputs to `int32` before zero-point subtraction to prevent arithmetic overflow. - Computes matrix multiplication via `relax.op.matmul(..., out_dtype="int32")` for 32-bit integer accumulation. - Supports per-row and per-column scale and zero-point broadcasting. - Handles requantization: `round(scaled) + y_zero_point` and clamps to output data type range (`uint8`, `int8`, `uint16`, `int16`). - Registered `"QLinearMatMul": QLinearMatMul` in `_get_convert_map()`. - In `tests/python/relax/test_frontend_onnx.py`: - Added `test_qlinearmatmul` testing 2D uint8, 2D int8, 3D batched int8/uint8, and opset 10/21, validated against ONNX Runtime. Closes #20958. --------- Co-authored-by: Masahiro Hiramori <contact@mshr-h.com>
1 parent 46f7c8e commit 8f01d0d

2 files changed

Lines changed: 663 additions & 0 deletions

File tree

‎python/tvm/relax/frontend/onnx/onnx_frontend.py‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6206,6 +6206,221 @@ def _impl_v10(cls, bb, inputs, attr, params):
62066206
return relax.op.matmul(a, b, out_dtype="int32") # Output is int32 per ONNX spec
62076207

62086208

6209+
class QLinearMatMul(OnnxOpConverter):
6210+
"""Converts an onnx QLinearMatMul node into an equivalent Relax expression."""
6211+
6212+
@classmethod
6213+
def _squeeze_singleton(cls, bb, param):
6214+
"""Squeeze singleton tensors (all dimensions equal to 1, e.g. [1], [1, 1]) to 0D scalars.
6215+
6216+
Per-tensor scales and zero points in ONNX can be provided as scalars (0D) or singleton
6217+
tensors (1D [1], 2D [1, 1], etc.). Normalizing them to scalars prevents them from
6218+
accidentally expanding output ranks during matrix-vector and vector-matrix multiplications.
6219+
"""
6220+
if param is None:
6221+
return None
6222+
param_norm = bb.normalize(param)
6223+
if not hasattr(param_norm.ty, "shape"):
6224+
return param
6225+
p_shape = param_norm.ty.shape
6226+
if len(p_shape) > 0 and all(
6227+
isinstance(d, int | tvm_tirx.ir.IntImm) and int(d) == 1 for d in p_shape
6228+
):
6229+
return relax.op.squeeze(param)
6230+
return param
6231+
6232+
@classmethod
6233+
def _align_output_param(cls, bb, param, a_norm, b_norm):
6234+
if param is None:
6235+
return None
6236+
param = cls._squeeze_singleton(bb, param)
6237+
param_norm = bb.normalize(param)
6238+
if not hasattr(param_norm.ty, "shape"):
6239+
return param
6240+
p_shape = param_norm.ty.shape
6241+
p_ndim = len(p_shape)
6242+
if p_ndim == 0:
6243+
return param
6244+
6245+
a_shape = a_norm.ty.shape if hasattr(a_norm.ty, "shape") else None
6246+
b_shape = b_norm.ty.shape if hasattr(b_norm.ty, "shape") else None
6247+
a_rank = len(a_shape) if a_shape is not None else 2
6248+
b_rank = len(b_shape) if b_shape is not None else 2
6249+
6250+
# When both a and b are 1D vectors, matmul output is scalar (0D)
6251+
if a_rank == 1 and b_rank == 1:
6252+
return relax.op.squeeze(param)
6253+
6254+
# When b_rank == 1, output of matmul is [..., M], no trailing column axis
6255+
if b_rank == 1:
6256+
if (
6257+
p_ndim >= 2
6258+
and isinstance(p_shape[-1], int | tvm_tirx.ir.IntImm)
6259+
and int(p_shape[-1]) == 1
6260+
):
6261+
return relax.op.squeeze(param, axis=[-1])
6262+
return param
6263+
6264+
# When a_rank == 1, output of matmul is [..., N], no leading row axis
6265+
if a_rank == 1:
6266+
if (
6267+
p_ndim >= 2
6268+
and isinstance(p_shape[-2], int | tvm_tirx.ir.IntImm)
6269+
and int(p_shape[-2]) == 1
6270+
):
6271+
return relax.op.squeeze(param, axis=[-2])
6272+
return param
6273+
6274+
# Both a and b have rank >= 2, output is [..., M, N]
6275+
if p_ndim == 1:
6276+
# Check if it corresponds to row dimension M
6277+
if a_shape is not None and a_rank >= 2:
6278+
row_dim = a_shape[-2]
6279+
if isinstance(row_dim, int | tvm_tirx.ir.IntImm) and int(row_dim) == int(
6280+
p_shape[0]
6281+
):
6282+
return relax.op.expand_dims(param, axis=-1)
6283+
# Check if it corresponds to col dimension N
6284+
if b_shape is not None and b_rank >= 2:
6285+
col_dim = b_shape[-1]
6286+
if isinstance(col_dim, int | tvm_tirx.ir.IntImm) and int(col_dim) == int(
6287+
p_shape[0]
6288+
):
6289+
return relax.op.expand_dims(param, axis=0)
6290+
6291+
return param
6292+
6293+
@classmethod
6294+
def _impl_v10(cls, bb, inputs, attr, params):
6295+
a = inputs[0]
6296+
a_scale = inputs[1]
6297+
a_zero_point = inputs[2] if len(inputs) > 2 and inputs[2] is not None else None
6298+
b = inputs[3]
6299+
b_scale = inputs[4]
6300+
b_zero_point = inputs[5] if len(inputs) > 5 and inputs[5] is not None else None
6301+
y_scale = inputs[6]
6302+
y_zero_point = inputs[7] if len(inputs) > 7 and inputs[7] is not None else None
6303+
6304+
if y_zero_point is None:
6305+
y_zero_point = relax.const(0, "uint8")
6306+
out_dtype = y_zero_point.ty.dtype.dtype
6307+
6308+
# Reject float8 input and output dtypes
6309+
for name, val in [("A", a), ("B", b), ("Y", y_zero_point)]:
6310+
if val is not None and hasattr(val, "ty") and hasattr(val.ty, "dtype"):
6311+
dtype_str = str(val.ty.dtype.dtype)
6312+
if dtype_str.startswith("float8") or "e4m3" in dtype_str or "e5m2" in dtype_str:
6313+
raise NotImplementedError(
6314+
f"Float8 operands and outputs ({name} has dtype {dtype_str}) "
6315+
"are currently not supported in QLinearMatMul."
6316+
)
6317+
6318+
# Normalize singleton [1]-shape and [1, 1]-shape parameters to 0D scalars
6319+
# BEFORE any rank expansion or alignment, so they do not accidentally
6320+
# increase the output rank in vector-matrix, matrix-vector, or dot products.
6321+
a_scale = cls._squeeze_singleton(bb, a_scale)
6322+
a_zero_point = cls._squeeze_singleton(bb, a_zero_point)
6323+
b_scale = cls._squeeze_singleton(bb, b_scale)
6324+
b_zero_point = cls._squeeze_singleton(bb, b_zero_point)
6325+
y_scale = cls._squeeze_singleton(bb, y_scale)
6326+
y_zero_point = cls._squeeze_singleton(bb, y_zero_point)
6327+
6328+
a_norm = bb.normalize(a)
6329+
b_norm = bb.normalize(b)
6330+
a_shape = a_norm.ty.shape if hasattr(a_norm.ty, "shape") else None
6331+
b_shape = b_norm.ty.shape if hasattr(b_norm.ty, "shape") else None
6332+
a_rank = len(a_shape) if a_shape is not None else 2
6333+
b_rank = len(b_shape) if b_shape is not None else 2
6334+
6335+
# Widen to int32 before arithmetic to prevent overflow
6336+
a = relax.op.astype(a, "int32")
6337+
b = relax.op.astype(b, "int32")
6338+
6339+
if a_zero_point is not None:
6340+
a_zp = relax.op.astype(a_zero_point, "int32")
6341+
a_zp = bb.normalize(a_zp)
6342+
if a_rank >= 2 and len(a_zp.ty.shape) == 1:
6343+
a_zp = relax.op.expand_dims(a_zp, axis=-1)
6344+
a = relax.op.subtract(a, a_zp)
6345+
6346+
if b_zero_point is not None:
6347+
b_zp = relax.op.astype(b_zero_point, "int32")
6348+
b_zp = bb.normalize(b_zp)
6349+
if b_rank >= 2 and len(b_zp.ty.shape) == 1:
6350+
b_zp = relax.op.expand_dims(b_zp, axis=0)
6351+
b = relax.op.subtract(b, b_zp)
6352+
6353+
# 32-bit integer accumulation
6354+
matmul_int32 = relax.op.matmul(a, b, out_dtype="int32")
6355+
6356+
# Broadcast a_scale: when both a and b are rank >= 2, expand row dimension to broadcast over columns
6357+
a_scale_norm = bb.normalize(a_scale)
6358+
if hasattr(a_scale_norm.ty, "shape"):
6359+
as_shape = a_scale_norm.ty.shape
6360+
if a_rank >= 2 and b_rank >= 2:
6361+
if len(as_shape) == 1:
6362+
a_scale = relax.op.expand_dims(a_scale, axis=-1)
6363+
elif b_rank == 1:
6364+
if (
6365+
len(as_shape) >= 2
6366+
and isinstance(as_shape[-1], int | tvm_tirx.ir.IntImm)
6367+
and int(as_shape[-1]) == 1
6368+
):
6369+
a_scale = relax.op.squeeze(a_scale, axis=[-1])
6370+
6371+
# Broadcast b_scale: when both a and b are rank >= 2, expand column dimension to broadcast over rows
6372+
b_scale_norm = bb.normalize(b_scale)
6373+
if hasattr(b_scale_norm.ty, "shape"):
6374+
bs_shape = b_scale_norm.ty.shape
6375+
if a_rank >= 2 and b_rank >= 2:
6376+
if len(bs_shape) == 1:
6377+
b_scale = relax.op.expand_dims(b_scale, axis=0)
6378+
elif a_rank == 1:
6379+
if (
6380+
len(bs_shape) >= 2
6381+
and isinstance(bs_shape[-2], int | tvm_tirx.ir.IntImm)
6382+
and int(bs_shape[-2]) == 1
6383+
):
6384+
b_scale = relax.op.squeeze(b_scale, axis=[-2])
6385+
6386+
# Ensure any resulting singleton scale or output parameter is squeezed
6387+
a_scale = cls._squeeze_singleton(bb, a_scale)
6388+
b_scale = cls._squeeze_singleton(bb, b_scale)
6389+
y_scale = cls._squeeze_singleton(bb, y_scale)
6390+
y_zero_point = cls._squeeze_singleton(bb, y_zero_point)
6391+
6392+
y_scale = cls._align_output_param(bb, y_scale, a_norm, b_norm)
6393+
y_zero_point = cls._align_output_param(bb, y_zero_point, a_norm, b_norm)
6394+
6395+
matmul_float = relax.op.astype(matmul_int32, "float32")
6396+
a_scale_f = relax.op.astype(a_scale, "float32")
6397+
b_scale_f = relax.op.astype(b_scale, "float32")
6398+
y_scale_f = relax.op.astype(y_scale, "float32")
6399+
6400+
scale_mult = relax.op.divide(relax.op.multiply(a_scale_f, b_scale_f), y_scale_f)
6401+
scaled = relax.op.multiply(matmul_float, scale_mult)
6402+
rounded = relax.op.round(scaled)
6403+
y_val = relax.op.add(rounded, relax.op.astype(y_zero_point, "float32"))
6404+
6405+
if out_dtype == "uint8":
6406+
min_val, max_val = 0, 255
6407+
elif out_dtype == "int8":
6408+
min_val, max_val = -128, 127
6409+
elif out_dtype == "uint16":
6410+
min_val, max_val = 0, 65535
6411+
elif out_dtype == "int16":
6412+
min_val, max_val = -32768, 32767
6413+
else:
6414+
min_val, max_val = 0, 255
6415+
6416+
clipped = relax.op.clip(y_val, min_val, max_val)
6417+
return relax.op.astype(clipped, out_dtype)
6418+
6419+
@classmethod
6420+
def _impl_v21(cls, bb, inputs, attr, params):
6421+
return cls._impl_v10(bb, inputs, attr, params)
6422+
6423+
62096424
class DFT(OnnxOpConverter):
62106425
"""Converts an onnx DFT node into an equivalent Relax expression."""
62116426

@@ -6457,6 +6672,7 @@ def _get_convert_map():
64576672
"MatMul": MatMul,
64586673
"MatMulInteger": MatMulInteger,
64596674
"MatMulInteger16": MatMulInteger16,
6675+
"QLinearMatMul": QLinearMatMul,
64606676
"Reshape": Reshape,
64616677
"Sigmoid": Sigmoid,
64626678
"Softmax": Softmax,

0 commit comments

Comments
 (0)