Skip to content

Commit 46f7c8e

Browse files
dotcom07Copilot
andauthored
[Perf][Relax] Skip future KV tiles in causal ragged prefill (#20903)
The generic TIR ragged prefill kernel traverses the full KV range even when causal masking makes later tiles invisible to every query in the current tile. These iterations still load K/V and compute QK, online softmax, and PV. Bound the KV loop by the last valid query in each tile, accounting for GQA head packing and partial query tiles. Keep the diagonal tile and its elementwise mask. Tests check the loop bound against an independent derivation of the last visible key, compile the kernel for WebGPU, and compare the full kernel with a NumPy reference on CUDA and Metal for sequences spanning several query and KV tiles. Locally, these tests pass on an Apple M4 Pro except the CUDA cases, which were not run because no NVIDIA GPU was available. The existing prefill and paged KV cache tests and pre-commit checks also pass. Causal kernel time on Metal (Apple M4 Pro, fp16, head_dim 128, 32 query heads; median of three interleaved runs): | Case | main (ms) | this PR (ms) | Speedup | |---|---|---|---| | GQA (8 KV heads), seq 128 | 0.29 | 0.21 | 1.36x | | GQA, seq 512 | 3.87 | 2.41 | 1.60x | | GQA, seq 1024 | 12.88 | 7.87 | 1.64x | | GQA, seq 2048 | 52.42 | 28.86 | 1.82x | | GQA, seq 4096 | 212.45 | 116.09 | 1.83x | | MHA, seq 2048 | 45.97 | 26.04 | 1.77x | | GQA, batch 4 x seq 1024 | 52.85 | 29.36 | 1.80x | --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent 62a5528 commit 46f7c8e

2 files changed

Lines changed: 152 additions & 1 deletion

File tree

‎python/tvm/relax/frontend/nn/llm/_prefill_kernels.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -838,7 +838,16 @@ def batch_prefill_ragged_kv( # pylint: disable=too-many-branches
838838
Q_smem[i, j] = 0.0
839839
T.gpu_storage_sync("shared")
840840

841-
for iterator in T.serial(T.ceildiv(kv_chunk_len[0], tile_z)):
841+
# Query rows are packed with GQA heads. Keep the diagonal tile
842+
# and let the elementwise mask handle its partially visible keys.
843+
q_len: T.let[T.int32] = q_indptr[b_idx + 1] - q_indptr[b_idx]
844+
q_end: T.let[T.int32] = T.min(q_len, T.ceildiv(LH_start + tile_x, group_size))
845+
kv_stop: T.let[T.int32] = T.if_then_else(
846+
causal > 0,
847+
T.max(0, T.min(kv_chunk_len[0], kv_chunk_len[0] - q_len + q_end)),
848+
kv_chunk_len[0],
849+
)
850+
for iterator in T.serial(T.ceildiv(kv_stop, tile_z)):
842851
L_kv_start: T.let[T.int32] = iterator * tile_z
843852
L_kv_base: T.let[T.int32] = kv_indptr[b_idx]
844853
for lz, ly in T.grid(tile_z, tile_y):
Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
import itertools
18+
19+
import numpy as np
20+
import pytest
21+
import tvm_ffi
22+
23+
import tvm
24+
import tvm_tirx
25+
import tvm_tirx.testing
26+
from tvm.relax.frontend.nn.llm._kernel_common import _get_prefill_kernel_config
27+
from tvm.relax.frontend.nn.llm._prefill_kernels import _attention_prefill_ragged
28+
from tvm.testing import device_enabled
29+
from tvm_tirx import ir
30+
31+
32+
@pytest.mark.parametrize("group_size", [1, 3, 8])
33+
@pytest.mark.parametrize("dtype", ["float16", "float32"])
34+
def test_ragged_causal_loop_bound(group_size, dtype):
35+
target = tvm_tirx.target.Target("webgpu")
36+
config = _get_prefill_kernel_config(1, group_size, 64, dtype, target)
37+
tile_x, _, tile_z = config[-3:]
38+
func = _attention_prefill_ragged(1, group_size, 64, 64, dtype, {}, target)
39+
bindings = {}
40+
loops = []
41+
42+
def collect(node):
43+
if isinstance(node, ir.Bind):
44+
bindings[node.var.name] = node.value
45+
if isinstance(node, ir.For) and node.var.name == "iterator":
46+
loops.append(node)
47+
48+
tvm_ffi.structural_walk(func.body, collect)
49+
assert len(loops) == 1
50+
analyzer = tvm_tirx.sym.Analyzer()
51+
# PagedKVCache passes the same indptr for Q and KV, so each sequence has q_len == kv_len.
52+
# Also cover longer KV sequences, supported by the kernel's separate indptr arrays.
53+
for seq_len, kv_extra, causal in itertools.product((1, 7, 33, 65, 129), (0, 37), (0, 1)):
54+
kv_len = seq_len + kv_extra
55+
for start in range(0, seq_len * group_size, tile_x):
56+
values = {"q_len": seq_len, "LH_start": start, "causal": causal}
57+
58+
def substitute(node):
59+
# q_len is replaced below, leaving only kv_chunk_len[0] as a tensor load.
60+
if isinstance(node, ir.TensorLoad):
61+
return ir.IntImm("int32", kv_len)
62+
if isinstance(node, ir.Var):
63+
if node.name in values:
64+
return ir.IntImm("int32", values[node.name])
65+
if node.name in ("q_end", "kv_stop"):
66+
return bindings[node.name]
67+
return node
68+
69+
end = loops[0].end
70+
for _ in range(3):
71+
end = tvm_ffi.structural_map(end, substitute)
72+
actual = int(analyzer.simplify(end))
73+
# Derive the last visible key independently of the kernel's bound expression.
74+
last_row = min(start + tile_x, seq_len * group_size) - 1
75+
last_key = kv_len - seq_len + last_row // group_size if causal else kv_len - 1
76+
assert actual == last_key // tile_z + 1, (seq_len, kv_len, causal, start, group_size)
77+
78+
79+
def test_ragged_prefill_webgpu_codegen():
80+
target = tvm_tirx.target.Target("webgpu")
81+
func = _attention_prefill_ragged(1, 3, 64, 32, "float32", {}, target)
82+
compiled = tvm_tirx.tirx.compile(func.with_attr("global_symbol", "main"), target=target)
83+
assert compiled.mod.imports[0].inspect_source()
84+
85+
86+
@pytest.mark.gpu
87+
@pytest.mark.parametrize("device_type", ["cuda", "metal"])
88+
@pytest.mark.parametrize("dtype", ["float16", "float32"])
89+
def test_ragged_causal_runtime(device_type, dtype):
90+
if not device_enabled(device_type):
91+
pytest.skip(f"{device_type} not enabled")
92+
device = tvm_ffi.device(device_type)
93+
target = tvm_tirx.target.Target.from_device(device)
94+
# Lengths span several query and KV tiles so later KV tiles are actually skipped.
95+
lengths = [1, 5, 37, 129]
96+
indptr = np.array([0, *np.cumsum(lengths)], dtype="int32")
97+
rng = np.random.default_rng(127)
98+
q = rng.normal(0, 0.3, (indptr[-1], 8, 32)).astype(dtype)
99+
k = rng.normal(0, 0.3, (indptr[-1], 2, 32)).astype(dtype)
100+
v = rng.normal(size=(indptr[-1], 2, 64)).astype(dtype)
101+
scale = 32**-0.5
102+
reference = np.zeros((indptr[-1], 8, 64))
103+
reference_lse = np.zeros((indptr[-1], 8))
104+
for b, n in enumerate(lengths):
105+
rows = slice(indptr[b], indptr[b + 1])
106+
for head in range(8):
107+
# The kernel works in base 2: LSE is log2(sum(exp2(score))).
108+
scores = q[rows, head].astype("float64") @ k[rows, head // 4].astype("float64").T
109+
scores = scores * scale / np.log(2)
110+
scores[np.triu_indices(n, 1)] = -np.inf
111+
maximum = scores.max(axis=1, keepdims=True)
112+
weights = np.exp2(scores - maximum)
113+
reference[rows, head] = (
114+
weights @ v[rows, head // 4] / weights.sum(axis=1, keepdims=True)
115+
)
116+
reference_lse[rows, head] = maximum[:, 0] + np.log2(weights.sum(axis=1))
117+
func = _attention_prefill_ragged(2, 8, 32, 64, dtype, {}, target)
118+
module = tvm.compile(func.with_attr("global_symbol", "main"), target=target)
119+
120+
def run_and_check():
121+
arrays = [
122+
q,
123+
indptr,
124+
k,
125+
v,
126+
indptr,
127+
np.arange(indptr[-1], dtype="int32"),
128+
np.zeros(len(lengths), dtype="int32"),
129+
]
130+
args = [tvm_tirx.runtime.tensor(a, device) for a in arrays]
131+
output = tvm_tirx.runtime.tensor(np.full(reference.shape, np.nan, dtype=dtype), device)
132+
lse = tvm_tirx.runtime.tensor(np.full(reference_lse.shape, np.nan, dtype="float32"), device)
133+
module(*args, output, lse, 1, 0, 1.0, 10000.0, scale)
134+
rtol, atol = (2e-3, 2e-3) if dtype == "float16" else (2e-4, 2e-5)
135+
np.testing.assert_allclose(output.numpy(), reference, rtol=rtol, atol=atol)
136+
np.testing.assert_allclose(lse.numpy(), reference_lse, rtol=0, atol=2e-5)
137+
138+
tvm_tirx.testing.run_with_gpu_lock(run_and_check)
139+
140+
141+
if __name__ == "__main__":
142+
tvm_tirx.testing.main()

0 commit comments

Comments
 (0)